def make_command(filename, engine, args=[]): if type(engine) is not list: engine = [engine] # Emscripten supports multiple javascript runtimes. The default is nodejs but # it can also use d8 (the v8 engine shell) or jsc (JavaScript Core aka # Safari). Both d8 and jsc require a '--' to delimit arguments to be passed # to the executed script from d8/jsc options. Node does not require a # delimeter--arguments after the filename are passed to the script. # # Check only the last part of the engine path to ensure we don't accidentally # label a path to nodejs containing a 'd8' as spidermonkey instead. jsengine = os.path.basename(engine[0]) # Use "'d8' in" because the name can vary, e.g. d8_g, d8, etc. is_d8 = 'd8' in jsengine or 'v8' in jsengine is_jsc = 'jsc' in jsengine is_wasmer = 'wasmer' in jsengine is_wasmtime = 'wasmtime' in jsengine is_clang = engine[0] == shared.CLANG_CC # Disable true async compilation (async apis will in fact be synchronous) for now # due to https://bugs.chromium.org/p/v8/issues/detail?id=6263 shell_option_flags = ['--no-wasm-async-compilation'] if is_d8 else [] command_flags = [] if is_wasmer: command_flags += ['run'] if is_wasmer or is_wasmtime: # in a wasm runtime, run the wasm, not the js filename = shared.unsuffixed(filename) + '.wasm' elif is_clang: # with wasm2c, the input is a c file, which we must compile first c = shared.unsuffixed(filename) + '.wasm.c' executable = shared.unsuffixed(filename) + '.exe' shared.run_process(engine + [c, '-o', executable]) # we can now run the executable directly, without an engine engine = [] filename = os.path.abspath(executable) # Separates engine flags from script flags flag_separator = ['--'] if is_d8 or is_jsc else [] return engine + command_flags + [ filename ] + shell_option_flags + flag_separator + args
def finalize_wasm(temp_files, infile, outfile, memfile, DEBUG): basename = shared.unsuffixed(outfile.name) wasm = basename + '.wasm' base_wasm = infile building.save_intermediate(infile, 'base.wasm') args = ['--detect-features', '--minimize-wasm-changes'] write_source_map = shared.Settings.DEBUG_LEVEL >= 4 if write_source_map: building.emit_wasm_source_map(base_wasm, base_wasm + '.map') building.save_intermediate(base_wasm + '.map', 'base_wasm.map') args += [ '--output-source-map-url=' + shared.Settings.SOURCE_MAP_BASE + os.path.basename(shared.Settings.WASM_BINARY_FILE) + '.map' ] # tell binaryen to look at the features section, and if there isn't one, to use MVP # (which matches what llvm+lld has given us) if shared.Settings.DEBUG_LEVEL >= 2 or shared.Settings.PROFILING_FUNCS or shared.Settings.EMIT_SYMBOL_MAP or shared.Settings.ASYNCIFY_ONLY or shared.Settings.ASYNCIFY_REMOVE or shared.Settings.ASYNCIFY_ADD: args.append('-g') if shared.Settings.WASM_BIGINT: args.append('--bigint') if not shared.Settings.USE_LEGACY_DYNCALLS: if shared.Settings.WASM_BIGINT: args.append('--no-dyncalls') else: args.append('--dyncalls-i64') if shared.Settings.LEGALIZE_JS_FFI != 1: args.append('--no-legalize-javascript-ffi') if not shared.Settings.MEM_INIT_IN_WASM: args.append('--separate-data-segments=' + memfile) if shared.Settings.SIDE_MODULE: args.append('--side-module') else: # --global-base is used by wasm-emscripten-finalize to calculate the size # of the static data used. The argument we supply here needs to match the # global based used by lld (see building.link_lld). For relocatable this is # zero for the global base although at runtime __memory_base is used. # For non-relocatable output we used shared.Settings.GLOBAL_BASE. # TODO(sbc): Can we remove this argument infer this from the segment # initializer? if shared.Settings.RELOCATABLE: args.append('--global-base=0') else: args.append('--global-base=%s' % shared.Settings.GLOBAL_BASE) if shared.Settings.STACK_OVERFLOW_CHECK >= 2: args.append('--check-stack-overflow') if shared.Settings.STANDALONE_WASM: args.append('--standalone-wasm') # When we dynamically link our JS loader adds functions from wasm modules to # the table. It must add the original versions of them, not legalized ones, # so that indirect calls have the right type, so export those. if shared.Settings.RELOCATABLE: args.append('--pass-arg=legalize-js-interface-export-originals') if shared.Settings.DEBUG_LEVEL >= 3: args.append('--dwarf') stdout = building.run_binaryen_command('wasm-emscripten-finalize', infile=base_wasm, outfile=wasm, args=args, stdout=subprocess.PIPE) if write_source_map: building.save_intermediate(wasm + '.map', 'post_finalize.map') building.save_intermediate(wasm, 'post_finalize.wasm') if not shared.Settings.MEM_INIT_IN_WASM: # we have a separate .mem file. binaryen did not strip any trailing zeros, # because it's an ABI question as to whether it is valid to do so or not. # we can do so here, since we make sure to zero out that memory (even in # the dynamic linking case, our loader zeros it out) remove_trailing_zeros(memfile) return load_metadata_wasm(stdout, DEBUG)
decrunchCallbacks[msg.data.callbackID] = null; }; function requestDecrunch(filename, data, callback) { decrunchWorker.postMessage({ filename: filename, data: new Uint8Array(data), callbackID: decrunchCallbacks.length }); decrunchCallbacks.push(callback); } ''' for file_ in data_files: if file_['dstpath'].endswith(CRUNCH_INPUT_SUFFIX): src_dds_name = file_['srcpath'] src_crunch_name = unsuffixed(src_dds_name) + CRUNCH_OUTPUT_SUFFIX # Preload/embed the .crn version instead of the .dds version, but use the .dds suffix for the target file in the virtual FS. file_['srcpath'] = src_crunch_name try: # Do not crunch if crunched version exists and is more recent than dds source crunch_time = os.stat(src_crunch_name).st_mtime dds_time = os.stat(src_dds_name).st_mtime if dds_time < crunch_time: continue except: pass # if one of them does not exist, continue on # guess at format. this lets us tell crunch to not try to be clever and use odd formats like DXT5_AGBR try: format = run_process(['file', file_['srcpath']],
def do_wasm2c(infile): assert Settings.STANDALONE_WASM WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] WASM2C_DIR = path_from_root('node_modules', 'wasm2c') c_file = unsuffixed(infile) + '.wasm.c' h_file = unsuffixed(infile) + '.wasm.h' cmd = WASM2C + [infile, '-o', c_file] check_call(cmd) total = '''\ /* * This file was generated by emcc+wasm2c. To compile it, use something like * * $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000 */ ''' SEP = '\n/* ==================================== */\n' def bundle_file(total, filename): with open(filename) as f: total += '// ' + filename + '\n' + f.read() + SEP return total # hermeticize the C file, by bundling in the wasm2c/ includes headers = [ (WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'), ('', h_file) ] for header in headers: total = bundle_file(total, os.path.join(header[0], header[1])) # add the wasm2c output with open(c_file) as read_c: c = read_c.read() total += c + SEP # add the wasm2c runtime total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) # add the support code support_files = ['base'] if Settings.AUTODEBUG: support_files.append('autodebug') if Settings.EXPECT_MAIN: # TODO: add an option for direct OS access. For now, do that when building # an executable with main, as opposed to a library support_files.append('os') support_files.append('main') else: support_files.append('os_sandboxed') support_files.append('reactor') # for a reactor, also append wasmbox_* API definitions with open(h_file, 'a') as f: f.write(''' // wasmbox_* API // TODO: optional prefixing extern void wasmbox_init(void); ''') for support_file in support_files: total = bundle_file(total, path_from_root('tools', 'wasm2c', support_file + '.c')) # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) # generate the necessary invokes invokes = [] for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total): all_func_types = get_func_types(total) def name(i): return 'a' + str(i) wabt_sig = sig[0] + 'i' + sig[1:] typed_args = [s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))] full_typed_args = ['u32 fptr'] + typed_args types = [s_to_c(sig[i]) for i in range(1, len(sig))] args = [name(i) for i in range(1, len(sig))] c_func_type = s_to_c(sig[0]) + ' (*)(' + (', '.join(types) if types else 'void') + ')' if sig not in all_func_types: exit_with_error('could not find signature ' + sig + ' in function types ' + str(all_func_types)) type_index = all_func_types[sig] invokes.append(r''' IMPORT_IMPL(%(return_type)s, Z_envZ_invoke_%(sig)sZ_%(wabt_sig)s, (%(full_typed_args)s), { VERBOSE_LOG("invoke\n"); // waka u32 sp = Z_stackSaveZ_iv(); if (next_setjmp >= MAX_SETJMP_STACK) { abort_with_message("too many nested setjmps"); } u32 id = next_setjmp++; int result = setjmp(setjmp_stack[id]); %(declare_return)s if (result == 0) { %(receive)sCALL_INDIRECT(w2c___indirect_function_table, %(c_func_type)s, %(type_index)s, fptr %(args)s); /* if we got here, no longjmp or exception happened, we returned normally */ } else { /* A longjmp or an exception took us here. */ Z_stackRestoreZ_vi(sp); Z_setThrewZ_vii(1, 0); } next_setjmp--; %(return)s }); ''' % { 'return_type': s_to_c(sig[0]) if sig[0] != 'v' else 'void', 'sig': sig, 'wabt_sig': wabt_sig, 'full_typed_args': ', '.join(full_typed_args), 'type_index': type_index, 'c_func_type': c_func_type, 'args': (', ' + ', '.join(args)) if args else '', 'declare_return': (s_to_c(sig[0]) + ' returned_value = 0;') if sig[0] != 'v' else '', 'receive': 'returned_value = ' if sig[0] != 'v' else '', 'return': 'return returned_value;' if sig[0] != 'v' else '' }) total += '\n'.join(invokes) # write out the final file with open(c_file, 'w') as out: out.write(total)
def do_wasm2c(infile): assert settings.STANDALONE_WASM WASM2C = config.NODE_JS + [path_from_root('node_modules/wasm2c/wasm2c.js')] WASM2C_DIR = path_from_root('node_modules/wasm2c') c_file = unsuffixed(infile) + '.wasm.c' h_file = unsuffixed(infile) + '.wasm.h' cmd = WASM2C + [infile, '-o', c_file] check_call(cmd) total = '''\ /* * This file was generated by emcc+wasm2c. To compile it, use something like * * $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000 */ ''' SEP = '\n/* ==================================== */\n' def bundle_file(filename): nonlocal total with open(filename) as f: total += '// ' + filename + '\n' + f.read() + SEP # hermeticize the C file, by bundling in the wasm2c/ includes headers = [(WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'), ('', h_file)] for header in headers: bundle_file(os.path.join(header[0], header[1])) # add the wasm2c output bundle_file(c_file) # add the wasm2c runtime bundle_file(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) # add the support code support_files = ['base.c'] if settings.AUTODEBUG: support_files.append('autodebug.c') if settings.EXPECT_MAIN: # TODO: add an option for direct OS access. For now, do that when building # an executable with main, as opposed to a library support_files.append('os.c') support_files.append('main.c') else: support_files.append('os_sandboxed.c') support_files.append('reactor.c') # for a reactor, also append wasmbox_* API definitions with open(h_file, 'a') as f: f.write(''' // wasmbox_* API // TODO: optional prefixing extern void wasmbox_init(void); ''') for support_file in support_files: bundle_file(path_from_root(f'tools/wasm2c/{support_file}')) # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) # generate the necessary invokes invokes = [] for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total): all_func_types = get_func_types(total) def name(i): return 'a' + str(i) wabt_sig = sig[0] + 'i' + sig[1:] typed_args = [ s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig)) ] full_typed_args = ['u32 fptr'] + typed_args types = [s_to_c(sig[i]) for i in range(1, len(sig))] args = [name(i) for i in range(1, len(sig))] c_func_type = s_to_c( sig[0]) + ' (*)(' + (', '.join(types) if types else 'void') + ')' if sig not in all_func_types: exit_with_error('could not find signature ' + sig + ' in function types ' + str(all_func_types)) type_index = all_func_types[sig] invokes.append( r''' IMPORT_IMPL(%(return_type)s, Z_envZ_invoke_%(sig)sZ_%(wabt_sig)s, (%(full_typed_args)s), { VERBOSE_LOG("invoke\n"); // waka u32 sp = WASM_RT_ADD_PREFIX(Z_stackSaveZ_iv)(); if (next_setjmp >= MAX_SETJMP_STACK) { abort_with_message("too many nested setjmps"); } u32 id = next_setjmp++; int result = setjmp(setjmp_stack[id]); %(declare_return)s if (result == 0) { %(receive)sCALL_INDIRECT(w2c___indirect_function_table, %(c_func_type)s, %(type_index)s, fptr %(args)s); /* if we got here, no longjmp or exception happened, we returned normally */ } else { /* A longjmp or an exception took us here. */ WASM_RT_ADD_PREFIX(Z_stackRestoreZ_vi)(sp); WASM_RT_ADD_PREFIX(Z_setThrewZ_vii)(1, 0); } next_setjmp--; %(return)s }); ''' % { 'return_type': s_to_c(sig[0]) if sig[0] != 'v' else 'void', 'sig': sig, 'wabt_sig': wabt_sig, 'full_typed_args': ', '.join(full_typed_args), 'type_index': type_index, 'c_func_type': c_func_type, 'args': (', ' + ', '.join(args)) if args else '', 'declare_return': (s_to_c(sig[0]) + ' returned_value = 0;') if sig[0] != 'v' else '', 'receive': 'returned_value = ' if sig[0] != 'v' else '', 'return': 'return returned_value;' if sig[0] != 'v' else '' }) total += '\n'.join(invokes) # adjust sandboxing TRAP_OOB = 'TRAP(OOB)' assert total.count(TRAP_OOB) == 2 if settings.WASM2C_SANDBOXING == 'full': pass # keep it elif settings.WASM2C_SANDBOXING == 'none': total = total.replace(TRAP_OOB, '{}') elif settings.WASM2C_SANDBOXING == 'mask': assert not settings.ALLOW_MEMORY_GROWTH assert (settings.INITIAL_MEMORY & (settings.INITIAL_MEMORY - 1)) == 0, 'poewr of 2' total = total.replace(TRAP_OOB, '{}') MEM_ACCESS = '[addr]' assert total.count(MEM_ACCESS) == 3, '2 from wasm2c, 1 from runtime' total = total.replace(MEM_ACCESS, '[addr & %d]' % (settings.INITIAL_MEMORY - 1)) else: exit_with_error('bad sandboxing') # adjust prefixing: emit simple output that works with multiple libraries, # each compiled into its own single .c file, by adding 'static' in some places # TODO: decide on the proper pattern for this in an upstream discussion in # wasm2c; another option would be to prefix all these things. for rep in [ 'uint32_t wasm_rt_register_func_type(', 'void wasm_rt_trap(', 'void wasm_rt_allocate_memory(', 'uint32_t wasm_rt_grow_memory(', 'void wasm_rt_allocate_table(', 'jmp_buf g_jmp_buf', 'uint32_t g_func_type_count', 'FuncType* g_func_types', 'uint32_t wasm_rt_call_stack_depth', 'uint32_t g_saved_call_stack_depth', ]: # remove 'extern' from declaration total = total.replace('extern ' + rep, rep) # add 'static' to implementation old = total total = total.replace(rep, 'static ' + rep) assert old != total, f'did not find "{rep}"' # write out the final file with open(c_file, 'w') as out: out.write(total)
def do_wasm2c(infile): assert Settings.STANDALONE_WASM WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')] WASM2C_DIR = path_from_root('node_modules', 'wasm2c') c_file = unsuffixed(infile) + '.wasm.c' h_file = unsuffixed(infile) + '.wasm.h' cmd = WASM2C + [infile, '-o', c_file] check_call(cmd) total = '''\ /* * This file was generated by emcc+wasm2c. To compile it, use something like * * $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000 */ ''' SEP = '\n/* ==================================== */\n' def bundle_file(total, filename): with open(filename) as f: total += '// ' + filename + '\n' + f.read() + SEP return total # hermeticize the C file, by bundling in the wasm2c/ includes headers = [(WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'), ('', h_file)] for header in headers: total = bundle_file(total, os.path.join(header[0], header[1])) # add the wasm2c output with open(c_file) as read_c: c = read_c.read() total += c + SEP # add the wasm2c runtime total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c')) # add the support code support_files = ['base'] if Settings.AUTODEBUG: support_files.append('autodebug') if Settings.EXPECT_MAIN: # TODO: add an option for direct OS access. For now, do that when building # an executable with main, as opposed to a library support_files.append('os') support_files.append('main') else: support_files.append('os_sandboxed') support_files.append('reactor') # for a reactor, also append wasmbox_* API definitions with open(h_file, 'a') as f: f.write(''' // wasmbox_* API // TODO: optional prefixing extern void wasmbox_init(void); ''') for support_file in support_files: total = bundle_file( total, path_from_root('tools', 'wasm2c', support_file + '.c')) # remove #includes of the headers we bundled for header in headers: total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1]) # generate the necessary invokes invokes = [] for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total): def s_to_c(s): if s == 'v': return 'void' elif s == 'i': return 'u32' elif s == 'j': return 'u64' elif s == 'f': return 'f32' elif s == 'd': return 'f64' else: exit_with_error('invalid sig element:' + str(s)) def name(i): return 'a' + str(i) wabt_sig = sig[0] + 'i' + sig[1:] typed_args = ['u32 fptr'] + [ s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig)) ] types = ['u32'] + [s_to_c(sig[i]) for i in range(1, len(sig))] args = ['fptr'] + [name(i) for i in range(1, len(sig))] invokes.append( '%s_INVOKE_IMPL(%sZ_envZ_invoke_%sZ_%s, (%s), (%s), (%s), Z_dynCall_%sZ_%s);' % ('VOID' if sig[0] == 'v' else 'RETURNING', (s_to_c(sig[0]) + ', ') if sig[0] != 'v' else '', sig, wabt_sig, ', '.join(typed_args), ', '.join(types), ', '.join(args), sig, wabt_sig)) total += '\n'.join(invokes) # write out the final file with open(c_file, 'w') as out: out.write(total)
decrunchCallbacks[msg.data.callbackID] = null; }; function requestDecrunch(filename, data, callback) { decrunchWorker.postMessage({ filename: filename, data: new Uint8Array(data), callbackID: decrunchCallbacks.length }); decrunchCallbacks.push(callback); } ''' for file_ in data_files: if file_['dstpath'].endswith(CRUNCH_INPUT_SUFFIX): src_dds_name = file_['srcpath'] src_crunch_name = unsuffixed(src_dds_name) + CRUNCH_OUTPUT_SUFFIX # Preload/embed the .crn version instead of the .dds version, but use the .dds suffix for the target file in the virtual FS. file_['srcpath'] = src_crunch_name try: # Do not crunch if crunched version exists and is more recent than dds source crunch_time = os.stat(src_crunch_name).st_mtime dds_time = os.stat(src_dds_name).st_mtime if dds_time < crunch_time: continue except: pass # if one of them does not exist, continue on # guess at format. this lets us tell crunch to not try to be clever and use odd formats like DXT5_AGBR try: format = run_process(['file', file_['srcpath']], stdout=PIPE).stdout