def run_torture_tests(): print '\n[ checking torture testcases... ]\n' unexpected_result_count = 0 import test.waterfall.src.link_assembly_files as link_assembly_files s2wasm_torture_out = os.path.abspath( os.path.join(options.binaryen_test, 's2wasm-torture-out')) if os.path.isdir(s2wasm_torture_out): shutil.rmtree(s2wasm_torture_out) os.mkdir(s2wasm_torture_out) unexpected_result_count += link_assembly_files.run( linker=os.path.abspath(S2WASM_EXE), files=os.path.abspath( os.path.join(options.binaryen_test, 'torture-s', '*.s')), fails=os.path.abspath( os.path.join(options.binaryen_test, 's2wasm_known_gcc_test_failures.txt')), out=s2wasm_torture_out) assert os.path.isdir(s2wasm_torture_out ), 'Expected output directory %s' % s2wasm_torture_out import test.waterfall.src.execute_files as execute_files unexpected_result_count += execute_files.run( runner=os.path.abspath(WASM_SHELL_EXE), files=os.path.abspath(os.path.join(s2wasm_torture_out, '*.wast')), fails=os.path.abspath( os.path.join(options.binaryen_test, 's2wasm_known_binaryen_shell_test_failures.txt')), out='', wasmjs='') shutil.rmtree(s2wasm_torture_out) if unexpected_result_count: fail('%s failures' % unexpected_result_count, '0 failures')
def run_binaryen_js_tests(): if not MOZJS and not NODEJS: return print '\n[ checking binaryen.js testcases... ]\n' for s in sorted( os.listdir(os.path.join(options.binaryen_test, 'binaryen.js'))): if not s.endswith('.js'): continue print s f = open('a.js', 'w') f.write(open(os.path.join(options.binaryen_bin, 'binaryen.js')).read()) # node test support f.write( '\nif (typeof require === "function") var Binaryen = module.exports;\n' ) test_path = os.path.join(options.binaryen_test, 'binaryen.js', s) test = open(test_path).read() need_wasm = 'WebAssembly.' in test # some tests use wasm support in the VM if MOZJS: cmd = [MOZJS] elif NODEJS and not need_wasm: # TODO: check if node is new and has wasm support cmd = [NODEJS] else: continue # we can't run it cmd += ['a.js'] f.write(test) f.close() out = run_command(cmd, stderr=subprocess.STDOUT) expected = open( os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt')).read() if expected not in out: fail(out, expected)
def check_expected(actual, expected): if expected and os.path.exists(expected): expected = open(expected).read() # fix it up, our pretty (i32.const 83) must become compared to a homely 83 : i32 def fix_expected(x): x = x.strip() if not x: return x v, t = x.split(' : ') if v.endswith('.'): v = v[:-1] # remove trailing '.' return '(' + t + '.const ' + v + ')' def fix_actual(x): if '[trap ' in x: return '' return x expected = '\n'.join( map(fix_expected, expected.split('\n'))) actual = '\n'.join(map(fix_actual, actual.split('\n'))) print(' (using expected output)') actual = actual.strip() expected = expected.strip() if actual != expected: fail(actual, expected)
def test(engine): cmd = [engine, 'a.js'] out = run_command(cmd, stderr=subprocess.STDOUT) expected = open( os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt')).read() if expected not in out: fail(out, expected)
def check_expected(actual, expected): if expected and os.path.exists(expected): expected = open(expected).read() print(' (using expected output)') actual = actual.strip() expected = expected.strip() if actual != expected: shared.fail(actual, expected)
def run_vanilla_tests(): if not (has_vanilla_emcc and has_vanilla_llvm and 0): print('\n[ skipping emcc WASM_BACKEND testcases...]\n') return print('\n[ checking emcc WASM_BACKEND testcases...]\n') try: if has_vanilla_llvm: os.environ['LLVM'] = BIN_DIR # use the vanilla LLVM else: # if we did not set vanilla llvm, then we must set this env var to make emcc use the wasm backend. # (if we are using vanilla llvm, things should just work) print( '(not using vanilla llvm, so setting env var to tell emcc to use wasm backend)' ) os.environ['EMCC_WASM_BACKEND'] = '1' VANILLA_EMCC = os.path.join(options.binaryen_test, 'emscripten', 'emcc') # run emcc to make sure it sets itself up properly, if it was never run before command = [VANILLA_EMCC, '-v'] print('____' + ' '.join(command)) subprocess.check_call(command) for c in sorted( os.listdir(os.path.join(options.binaryen_test, 'wasm_backend'))): if not c.endswith('cpp'): continue print('..', c) base = c.replace('.cpp', '').replace('.c', '') expected = open( os.path.join(options.binaryen_test, 'wasm_backend', base + '.txt')).read() for opts in [[], ['-O1'], ['-O2']]: # only my code is a hack we used early in wasm backend dev, which somehow worked, but only with -O1 only = [] if opts != ['-O1'] or '_only' not in base else [ '-s', 'ONLY_MY_CODE=1' ] command = [ VANILLA_EMCC, '-o', 'a.wasm.js', os.path.join(options.binaryen_test, 'wasm_backend', c) ] + opts + only print('....' + ' '.join(command)) if os.path.exists('a.wasm.js'): os.unlink('a.wasm.js') subprocess.check_call(command) if NODEJS: print(' (check in node)') cmd = [NODEJS, 'a.wasm.js'] out = run_command(cmd) if out.strip() != expected.strip(): fail(out, expected) finally: if has_vanilla_llvm: del os.environ['LLVM'] else: del os.environ['EMCC_WASM_BACKEND']
def test(engine): cmd = [engine, 'a.js'] if 'fatal' not in s: out = run_command(cmd, stderr=subprocess.STDOUT) else: # expect an error - the specific error code will depend on the vm out = run_command(cmd, stderr=subprocess.STDOUT, expected_status=None) expected = open(os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt')).read() if expected not in out: fail(out, expected)
def run_torture_tests(): print '\n[ checking torture testcases... ]\n' # torture tests are parallel anyhow, don't create multiple threads in each child old_cores = os.environ.get('BINARYEN_CORES') try: os.environ['BINARYEN_CORES'] = '1' unexpected_result_count = 0 import test.waterfall.src.link_assembly_files as link_assembly_files s2wasm_torture_out = os.path.abspath( os.path.join(options.binaryen_test, 's2wasm-torture-out')) if os.path.isdir(s2wasm_torture_out): shutil.rmtree(s2wasm_torture_out) os.mkdir(s2wasm_torture_out) unexpected_result_count += link_assembly_files.run( linker=os.path.abspath(S2WASM_EXE), files=os.path.abspath( os.path.join(options.binaryen_test, 'torture-s', '*.s')), fails=[ os.path.abspath( os.path.join(options.binaryen_test, 's2wasm_known_gcc_test_failures.txt')) ], attributes=['O2'], out=s2wasm_torture_out, args=None) assert os.path.isdir( s2wasm_torture_out ), 'Expected output directory %s' % s2wasm_torture_out import test.waterfall.src.execute_files as execute_files unexpected_result_count += execute_files.run( runner=os.path.abspath(WASM_SHELL_EXE), files=os.path.abspath(os.path.join(s2wasm_torture_out, '*.wast')), fails=[ os.path.abspath( os.path.join( options.binaryen_test, 's2wasm_known_binaryen_shell_test_failures.txt')) ], attributes=['O2'], out='', wasmjs='') shutil.rmtree(s2wasm_torture_out) if unexpected_result_count: fail('%s failures' % unexpected_result_count, '0 failures') finally: if old_cores: os.environ['BINARYEN_CORES'] = old_cores else: del os.environ['BINARYEN_CORES']
def do_asm2wasm_test(): actual = run_command(cmd) # verify output if not os.path.exists(wasm): fail_with_error('output .wast file %s does not exist' % wasm) expected = open(wasm, 'rb').read() if actual != expected: fail(actual, expected) binary_format_check(wasm, verify_final_result=False)
def run_gcc_torture_tests(): print '\n[ checking native gcc testcases...]\n' if not NATIVECC or not NATIVEXX: fail_with_error('Native compiler (e.g. gcc/g++) was not found in PATH!') else: for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'example'))): output_file = os.path.join(options.binaryen_bin, 'example') cmd = ['-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-lasmjs', '-lsupport', '-L' + os.path.join(options.binaryen_bin, '..', 'lib'), '-pthread', '-o', output_file] if t.endswith('.txt'): # check if there is a trace in the file, if so, we should build it out = subprocess.Popen([os.path.join('scripts', 'clean_c_api_trace.py'), os.path.join(options.binaryen_test, 'example', t)], stdout=subprocess.PIPE).communicate()[0] if len(out) == 0: print ' (no trace in ', t, ')' continue print ' (will check trace in ', t, ')' src = 'trace.cpp' with open(src, 'w') as o: o.write(out) expected = os.path.join(options.binaryen_test, 'example', t + '.txt') else: src = os.path.join(options.binaryen_test, 'example', t) expected = os.path.join(options.binaryen_test, 'example', '.'.join(t.split('.')[:-1]) + '.txt') if src.endswith(('.c', '.cpp')): # build the C file separately extra = [NATIVECC, src, '-c', '-o', 'example.o', '-I' + os.path.join(options.binaryen_root, 'src'), '-g', '-L' + os.path.join(options.binaryen_bin, '..', 'lib'), '-pthread'] print 'build: ', ' '.join(extra) subprocess.check_call(extra) # Link against the binaryen C library DSO, using an executable-relative rpath cmd = ['example.o', '-lbinaryen'] + cmd + ['-Wl,-rpath=$ORIGIN/../lib'] else: continue print ' ', t, src, expected if os.environ.get('COMPILER_FLAGS'): for f in os.environ.get('COMPILER_FLAGS').split(' '): cmd.append(f) cmd = [NATIVEXX, '-std=c++11'] + cmd try: print 'link: ', ' '.join(cmd) subprocess.check_call(cmd) print 'run...', output_file proc = subprocess.Popen([output_file], stdout=subprocess.PIPE, stderr=subprocess.PIPE) actual, err = proc.communicate() assert proc.returncode == 0, [proc.returncode, actual, err] finally: os.remove(output_file) if sys.platform == 'darwin': # Also removes debug directory produced on Mac OS shutil.rmtree(output_file + '.dSYM') expected = open(expected).read() if actual != expected: fail(actual, expected)
def check_expected(actual, expected): if expected and os.path.exists(expected): expected = open(expected).read() # fix it up, our pretty (i32.const 83) must become compared to a homely 83 : i32 def fix(x): x = x.strip() if not x: return x v, t = x.split(' : ') if v.endswith('.'): v = v[:-1] # remove trailing '.' return '(' + t + '.const ' + v + ')' expected = '\n'.join(map(fix, expected.split('\n'))) print ' (using expected output)' actual = actual.strip() expected = expected.strip() if actual != expected: fail(actual, expected)
def run_wasm_dis_tests(): print '\n[ checking wasm-dis on provided binaries... ]\n' for t in tests: if t.endswith('.wasm') and not t.startswith('spec'): print '..', t t = os.path.join(options.binaryen_test, t) cmd = WASM_DIS + [t] if os.path.isfile(t + '.map'): cmd += ['--source-map', t + '.map'] actual = run_command(cmd) with open(t + '.fromBinary') as f: expected = f.read() if actual != expected: fail(actual, expected)
def run_binaryen_js_tests(): print '\n[ checking binaryen.js testcases... ]\n' for s in sorted( os.listdir(os.path.join(options.binaryen_test, 'binaryen.js'))): if not s.endswith('.js'): continue print s f = open('a.js', 'w') f.write(open(os.path.join(options.binaryen_bin, 'binaryen.js')).read()) f.write( open(os.path.join(options.binaryen_test, 'binaryen.js', s)).read()) f.close() cmd = [MOZJS, 'a.js'] out = run_command(cmd, stderr=subprocess.STDOUT) expected = open( os.path.join(options.binaryen_test, 'binaryen.js', s + '.txt')).read() if expected not in out: fail(out, expected)
open('a.mem', 'wb').write(asm) cmd += ['--mem-init=a.mem'] if asm[0] == 'e': cmd += ['--mem-base=1024'] if 'i64' in asm or 'wasm-only' in asm: cmd += ['--wasm-only'] wasm = os.path.join(options.binaryen_test, wasm) print '..', asm, wasm actual = run_command(cmd) # verify output if not os.path.exists(wasm): fail_with_error('output .wast file %s does not exist' % wasm) expected = open(wasm, 'rb').read() if actual != expected: fail(actual, expected) binary_format_check(wasm, verify_final_result=False) # verify in wasm if options.interpreter: # remove imports, spec interpreter doesn't know what to do with them subprocess.check_call(WASM_OPT + ['--remove-imports', wasm], stdout=open('ztemp.wast', 'w'), stderr=subprocess.PIPE) proc = subprocess.Popen([options.interpreter, 'ztemp.wast'], stderr=subprocess.PIPE) out, err = proc.communicate() if proc.returncode != 0: try: # to parse the error reported = err.split(':')[1] start, end = reported.split('-') start_line, start_col = map(int, start.split('.')) lines = open('ztemp.wast').read().split('\n')
def execute(): if NODEJS: cmd = [NODEJS, 'a.' + which + '.js'] + args out = run_command(cmd) if out.strip() != expected.strip(): fail(out, expected)
def run_wasm_opt_tests(): print '\n[ checking wasm-opt -o notation... ]\n' wast = os.path.join(options.binaryen_test, 'hello_world.wast') delete_from_orbit('a.wast') cmd = WASM_OPT + [wast, '-o', 'a.wast', '-S'] run_command(cmd) fail_if_not_identical(open('a.wast').read(), open(wast).read()) print '\n[ checking wasm-opt binary reading/writing... ]\n' shutil.copyfile(os.path.join(options.binaryen_test, 'hello_world.wast'), 'a.wast') delete_from_orbit('a.wasm') delete_from_orbit('b.wast') run_command(WASM_OPT + ['a.wast', '-o', 'a.wasm']) assert open('a.wasm', 'rb').read()[0] == '\0', 'we emit binary by default' run_command(WASM_OPT + ['a.wasm', '-o', 'b.wast', '-S']) assert open('b.wast', 'rb').read()[0] != '\0', 'we emit text with -S' print '\n[ checking wasm-opt passes... ]\n' for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'passes'))): if t.endswith(('.wast', '.wasm')): print '..', t binary = '.wasm' in t base = os.path.basename(t).replace('.wast', '').replace('.wasm', '') passname = base if passname.isdigit(): passname = open( os.path.join(options.binaryen_test, 'passes', passname + '.passes')).read().strip() opts = [('--' + p if not p.startswith('O') else '-' + p) for p in passname.split('_')] t = os.path.join(options.binaryen_test, 'passes', t) actual = '' for module, asserts in split_wast(t): assert len(asserts) == 0 with open('split.wast', 'w') as o: o.write(module) cmd = WASM_OPT + opts + ['split.wast', '--print'] curr = run_command(cmd) actual += curr # also check debug mode output is valid debugged = run_command(cmd + ['--debug'], stderr=subprocess.PIPE) fail_if_not_contained(actual, debugged) # also check pass-debug mode old_pass_debug = os.environ.get('BINARYEN_PASS_DEBUG') try: os.environ['BINARYEN_PASS_DEBUG'] = '1' pass_debug = run_command(cmd) fail_if_not_identical(curr, pass_debug) finally: if old_pass_debug is not None: os.environ['BINARYEN_PASS_DEBUG'] = old_pass_debug else: if 'BINARYEN_PASS_DEBUG' in os.environ: del os.environ['BINARYEN_PASS_DEBUG'] fail_if_not_identical( actual, open( os.path.join(options.binaryen_test, 'passes', base + ('.bin' if binary else '') + '.txt'), 'rb').read()) if 'emit-js-wrapper' in t: with open('a.js') as actual: with open(t + '.js') as expected: fail_if_not_identical(actual.read(), expected.read()) if 'emit-spec-wrapper' in t: with open('a.wat') as actual: with open(t + '.wat') as expected: fail_if_not_identical(actual.read(), expected.read()) print '\n[ checking wasm-opt parsing & printing... ]\n' for t in sorted(os.listdir(os.path.join(options.binaryen_test, 'print'))): if t.endswith('.wast'): print '..', t wasm = os.path.basename(t).replace('.wast', '') cmd = WASM_OPT + [ os.path.join(options.binaryen_test, 'print', t), '--print' ] print ' ', ' '.join(cmd) actual, err = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() fail_if_not_identical( actual, open( os.path.join(options.binaryen_test, 'print', wasm + '.txt')).read()) cmd = WASM_OPT + [ os.path.join(options.binaryen_test, 'print', t), '--print-minified' ] print ' ', ' '.join(cmd) actual, err = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() fail_if_not_identical( actual.strip(), open( os.path.join(options.binaryen_test, 'print', wasm + '.minified.txt')).read().strip()) print '\n[ checking wasm-opt testcases... ]\n' for t in tests: if t.endswith('.wast') and not t.startswith('spec'): print '..', t t = os.path.join(options.binaryen_test, t) f = t + '.from-wast' cmd = WASM_OPT + [t, '--print'] actual = run_command(cmd) actual = actual.replace('printing before:\n', '') expected = open(f, 'rb').read() if actual != expected: fail(actual, expected) binary_format_check(t, wasm_as_args=['-g']) # test with debuginfo binary_format_check(t, wasm_as_args=[], binary_suffix='.fromBinary.noDebugInfo' ) # test without debuginfo minify_check(t)
cmd += ['--mem-init=a.mem'] if asm[0] == 'e': cmd += ['--mem-base=1024'] if 'i64' in asm or 'wasm-only' in asm or 'noffi' in asm: cmd += ['--wasm-only'] wasm = os.path.join(options.binaryen_test, wasm) print '..', asm, wasm actual = run_command(cmd) # verify output if not os.path.exists(wasm): fail_with_error('output .wast file %s does not exist' % wasm) expected = open(wasm, 'rb').read() if actual != expected: fail(actual, expected) binary_format_check(wasm, verify_final_result=False) # verify in wasm if options.interpreter: # remove imports, spec interpreter doesn't know what to do with them subprocess.check_call(WASM_OPT + ['--remove-imports', wasm], stdout=open('ztemp.wast', 'w'), stderr=subprocess.PIPE) proc = subprocess.Popen( [options.interpreter, 'ztemp.wast'], stderr=subprocess.PIPE) out, err = proc.communicate() if proc.returncode != 0: