def make_command(filename, engine, args=[]):
if type(engine) is not list:
engine = [engine]
# Emscripten supports multiple javascript runtimes. The default is nodejs but
# it can also use d8 (the v8 engine shell) or jsc (JavaScript Core aka
# Safari). Both d8 and jsc require a '--' to delimit arguments to be passed
# to the executed script from d8/jsc options. Node does not require a
# delimeter--arguments after the filename are passed to the script.
#
# Check only the last part of the engine path to ensure we don't accidentally
# label a path to nodejs containing a 'd8' as spidermonkey instead.
jsengine = os.path.basename(engine[0])
# Use "'d8' in" because the name can vary, e.g. d8_g, d8, etc.
is_d8 = 'd8' in jsengine or 'v8' in jsengine
is_jsc = 'jsc' in jsengine
is_wasmer = 'wasmer' in jsengine
is_wasmtime = 'wasmtime' in jsengine
is_clang = engine[0] == shared.CLANG_CC
# Disable true async compilation (async apis will in fact be synchronous) for now
# due to https://bugs.chromium.org/p/v8/issues/detail?id=6263
shell_option_flags = ['--no-wasm-async-compilation'] if is_d8 else []
command_flags = []
if is_wasmer:
command_flags += ['run']
if is_wasmer or is_wasmtime:
# in a wasm runtime, run the wasm, not the js
filename = shared.unsuffixed(filename) + '.wasm'
elif is_clang:
# with wasm2c, the input is a c file, which we must compile first
c = shared.unsuffixed(filename) + '.wasm.c'
executable = shared.unsuffixed(filename) + '.exe'
shared.run_process(engine + [c, '-o', executable])
# we can now run the executable directly, without an engine
engine = []
filename = os.path.abspath(executable)
# Separates engine flags from script flags
flag_separator = ['--'] if is_d8 or is_jsc else []
return engine + command_flags + [
filename
] + shell_option_flags + flag_separator + args
def finalize_wasm(temp_files, infile, outfile, memfile, DEBUG):
basename = shared.unsuffixed(outfile.name)
wasm = basename + '.wasm'
base_wasm = infile
building.save_intermediate(infile, 'base.wasm')
args = ['--detect-features', '--minimize-wasm-changes']
write_source_map = shared.Settings.DEBUG_LEVEL >= 4
if write_source_map:
building.emit_wasm_source_map(base_wasm, base_wasm + '.map')
building.save_intermediate(base_wasm + '.map', 'base_wasm.map')
args += [
'--output-source-map-url=' + shared.Settings.SOURCE_MAP_BASE +
os.path.basename(shared.Settings.WASM_BINARY_FILE) + '.map'
]
# tell binaryen to look at the features section, and if there isn't one, to use MVP
# (which matches what llvm+lld has given us)
if shared.Settings.DEBUG_LEVEL >= 2 or shared.Settings.PROFILING_FUNCS or shared.Settings.EMIT_SYMBOL_MAP or shared.Settings.ASYNCIFY_ONLY or shared.Settings.ASYNCIFY_REMOVE or shared.Settings.ASYNCIFY_ADD:
args.append('-g')
if shared.Settings.WASM_BIGINT:
args.append('--bigint')
if not shared.Settings.USE_LEGACY_DYNCALLS:
if shared.Settings.WASM_BIGINT:
args.append('--no-dyncalls')
else:
args.append('--dyncalls-i64')
if shared.Settings.LEGALIZE_JS_FFI != 1:
args.append('--no-legalize-javascript-ffi')
if not shared.Settings.MEM_INIT_IN_WASM:
args.append('--separate-data-segments=' + memfile)
if shared.Settings.SIDE_MODULE:
args.append('--side-module')
else:
# --global-base is used by wasm-emscripten-finalize to calculate the size
# of the static data used. The argument we supply here needs to match the
# global based used by lld (see building.link_lld). For relocatable this is
# zero for the global base although at runtime __memory_base is used.
# For non-relocatable output we used shared.Settings.GLOBAL_BASE.
# TODO(sbc): Can we remove this argument infer this from the segment
# initializer?
if shared.Settings.RELOCATABLE:
args.append('--global-base=0')
else:
args.append('--global-base=%s' % shared.Settings.GLOBAL_BASE)
if shared.Settings.STACK_OVERFLOW_CHECK >= 2:
args.append('--check-stack-overflow')
if shared.Settings.STANDALONE_WASM:
args.append('--standalone-wasm')
# When we dynamically link our JS loader adds functions from wasm modules to
# the table. It must add the original versions of them, not legalized ones,
# so that indirect calls have the right type, so export those.
if shared.Settings.RELOCATABLE:
args.append('--pass-arg=legalize-js-interface-export-originals')
if shared.Settings.DEBUG_LEVEL >= 3:
args.append('--dwarf')
stdout = building.run_binaryen_command('wasm-emscripten-finalize',
infile=base_wasm,
outfile=wasm,
args=args,
stdout=subprocess.PIPE)
if write_source_map:
building.save_intermediate(wasm + '.map', 'post_finalize.map')
building.save_intermediate(wasm, 'post_finalize.wasm')
if not shared.Settings.MEM_INIT_IN_WASM:
# we have a separate .mem file. binaryen did not strip any trailing zeros,
# because it's an ABI question as to whether it is valid to do so or not.
# we can do so here, since we make sure to zero out that memory (even in
# the dynamic linking case, our loader zeros it out)
remove_trailing_zeros(memfile)
return load_metadata_wasm(stdout, DEBUG)
decrunchCallbacks[msg.data.callbackID] = null;
};
function requestDecrunch(filename, data, callback) {
decrunchWorker.postMessage({
filename: filename,
data: new Uint8Array(data),
callbackID: decrunchCallbacks.length
});
decrunchCallbacks.push(callback);
}
'''
for file_ in data_files:
if file_['dstpath'].endswith(CRUNCH_INPUT_SUFFIX):
src_dds_name = file_['srcpath']
src_crunch_name = unsuffixed(src_dds_name) + CRUNCH_OUTPUT_SUFFIX
# Preload/embed the .crn version instead of the .dds version, but use the .dds suffix for the target file in the virtual FS.
file_['srcpath'] = src_crunch_name
try:
# Do not crunch if crunched version exists and is more recent than dds source
crunch_time = os.stat(src_crunch_name).st_mtime
dds_time = os.stat(src_dds_name).st_mtime
if dds_time < crunch_time: continue
except:
pass # if one of them does not exist, continue on
# guess at format. this lets us tell crunch to not try to be clever and use odd formats like DXT5_AGBR
try:
format = run_process(['file', file_['srcpath']],
def do_wasm2c(infile):
assert Settings.STANDALONE_WASM
WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')]
WASM2C_DIR = path_from_root('node_modules', 'wasm2c')
c_file = unsuffixed(infile) + '.wasm.c'
h_file = unsuffixed(infile) + '.wasm.h'
cmd = WASM2C + [infile, '-o', c_file]
check_call(cmd)
total = '''\
/*
* This file was generated by emcc+wasm2c. To compile it, use something like
*
* $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000
*/
'''
SEP = '\n/* ==================================== */\n'
def bundle_file(total, filename):
with open(filename) as f:
total += '// ' + filename + '\n' + f.read() + SEP
return total
# hermeticize the C file, by bundling in the wasm2c/ includes
headers = [
(WASM2C_DIR, 'wasm-rt.h'),
(WASM2C_DIR, 'wasm-rt-impl.h'),
('', h_file)
]
for header in headers:
total = bundle_file(total, os.path.join(header[0], header[1]))
# add the wasm2c output
with open(c_file) as read_c:
c = read_c.read()
total += c + SEP
# add the wasm2c runtime
total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c'))
# add the support code
support_files = ['base']
if Settings.AUTODEBUG:
support_files.append('autodebug')
if Settings.EXPECT_MAIN:
# TODO: add an option for direct OS access. For now, do that when building
# an executable with main, as opposed to a library
support_files.append('os')
support_files.append('main')
else:
support_files.append('os_sandboxed')
support_files.append('reactor')
# for a reactor, also append wasmbox_* API definitions
with open(h_file, 'a') as f:
f.write('''
// wasmbox_* API
// TODO: optional prefixing
extern void wasmbox_init(void);
''')
for support_file in support_files:
total = bundle_file(total, path_from_root('tools', 'wasm2c', support_file + '.c'))
# remove #includes of the headers we bundled
for header in headers:
total = total.replace('#include "%s"\n' % header[1], '/* include of %s */\n' % header[1])
# generate the necessary invokes
invokes = []
for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total):
all_func_types = get_func_types(total)
def name(i):
return 'a' + str(i)
wabt_sig = sig[0] + 'i' + sig[1:]
typed_args = [s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))]
full_typed_args = ['u32 fptr'] + typed_args
types = [s_to_c(sig[i]) for i in range(1, len(sig))]
args = [name(i) for i in range(1, len(sig))]
c_func_type = s_to_c(sig[0]) + ' (*)(' + (', '.join(types) if types else 'void') + ')'
if sig not in all_func_types:
exit_with_error('could not find signature ' + sig + ' in function types ' + str(all_func_types))
type_index = all_func_types[sig]
invokes.append(r'''
IMPORT_IMPL(%(return_type)s, Z_envZ_invoke_%(sig)sZ_%(wabt_sig)s, (%(full_typed_args)s), {
VERBOSE_LOG("invoke\n"); // waka
u32 sp = Z_stackSaveZ_iv();
if (next_setjmp >= MAX_SETJMP_STACK) {
abort_with_message("too many nested setjmps");
}
u32 id = next_setjmp++;
int result = setjmp(setjmp_stack[id]);
%(declare_return)s
if (result == 0) {
%(receive)sCALL_INDIRECT(w2c___indirect_function_table, %(c_func_type)s, %(type_index)s, fptr %(args)s);
/* if we got here, no longjmp or exception happened, we returned normally */
} else {
/* A longjmp or an exception took us here. */
Z_stackRestoreZ_vi(sp);
Z_setThrewZ_vii(1, 0);
}
next_setjmp--;
%(return)s
});
''' % {
'return_type': s_to_c(sig[0]) if sig[0] != 'v' else 'void',
'sig': sig,
'wabt_sig': wabt_sig,
'full_typed_args': ', '.join(full_typed_args),
'type_index': type_index,
'c_func_type': c_func_type,
'args': (', ' + ', '.join(args)) if args else '',
'declare_return': (s_to_c(sig[0]) + ' returned_value = 0;') if sig[0] != 'v' else '',
'receive': 'returned_value = ' if sig[0] != 'v' else '',
'return': 'return returned_value;' if sig[0] != 'v' else ''
})
total += '\n'.join(invokes)
# write out the final file
with open(c_file, 'w') as out:
out.write(total)
def do_wasm2c(infile):
assert settings.STANDALONE_WASM
WASM2C = config.NODE_JS + [path_from_root('node_modules/wasm2c/wasm2c.js')]
WASM2C_DIR = path_from_root('node_modules/wasm2c')
c_file = unsuffixed(infile) + '.wasm.c'
h_file = unsuffixed(infile) + '.wasm.h'
cmd = WASM2C + [infile, '-o', c_file]
check_call(cmd)
total = '''\
/*
* This file was generated by emcc+wasm2c. To compile it, use something like
*
* $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000
*/
'''
SEP = '\n/* ==================================== */\n'
def bundle_file(filename):
nonlocal total
with open(filename) as f:
total += '// ' + filename + '\n' + f.read() + SEP
# hermeticize the C file, by bundling in the wasm2c/ includes
headers = [(WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'),
('', h_file)]
for header in headers:
bundle_file(os.path.join(header[0], header[1]))
# add the wasm2c output
bundle_file(c_file)
# add the wasm2c runtime
bundle_file(os.path.join(WASM2C_DIR, 'wasm-rt-impl.c'))
# add the support code
support_files = ['base.c']
if settings.AUTODEBUG:
support_files.append('autodebug.c')
if settings.EXPECT_MAIN:
# TODO: add an option for direct OS access. For now, do that when building
# an executable with main, as opposed to a library
support_files.append('os.c')
support_files.append('main.c')
else:
support_files.append('os_sandboxed.c')
support_files.append('reactor.c')
# for a reactor, also append wasmbox_* API definitions
with open(h_file, 'a') as f:
f.write('''
// wasmbox_* API
// TODO: optional prefixing
extern void wasmbox_init(void);
''')
for support_file in support_files:
bundle_file(path_from_root(f'tools/wasm2c/{support_file}'))
# remove #includes of the headers we bundled
for header in headers:
total = total.replace('#include "%s"\n' % header[1],
'/* include of %s */\n' % header[1])
# generate the necessary invokes
invokes = []
for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total):
all_func_types = get_func_types(total)
def name(i):
return 'a' + str(i)
wabt_sig = sig[0] + 'i' + sig[1:]
typed_args = [
s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))
]
full_typed_args = ['u32 fptr'] + typed_args
types = [s_to_c(sig[i]) for i in range(1, len(sig))]
args = [name(i) for i in range(1, len(sig))]
c_func_type = s_to_c(
sig[0]) + ' (*)(' + (', '.join(types) if types else 'void') + ')'
if sig not in all_func_types:
exit_with_error('could not find signature ' + sig +
' in function types ' + str(all_func_types))
type_index = all_func_types[sig]
invokes.append(
r'''
IMPORT_IMPL(%(return_type)s, Z_envZ_invoke_%(sig)sZ_%(wabt_sig)s, (%(full_typed_args)s), {
VERBOSE_LOG("invoke\n"); // waka
u32 sp = WASM_RT_ADD_PREFIX(Z_stackSaveZ_iv)();
if (next_setjmp >= MAX_SETJMP_STACK) {
abort_with_message("too many nested setjmps");
}
u32 id = next_setjmp++;
int result = setjmp(setjmp_stack[id]);
%(declare_return)s
if (result == 0) {
%(receive)sCALL_INDIRECT(w2c___indirect_function_table, %(c_func_type)s, %(type_index)s, fptr %(args)s);
/* if we got here, no longjmp or exception happened, we returned normally */
} else {
/* A longjmp or an exception took us here. */
WASM_RT_ADD_PREFIX(Z_stackRestoreZ_vi)(sp);
WASM_RT_ADD_PREFIX(Z_setThrewZ_vii)(1, 0);
}
next_setjmp--;
%(return)s
});
''' % {
'return_type':
s_to_c(sig[0]) if sig[0] != 'v' else 'void',
'sig':
sig,
'wabt_sig':
wabt_sig,
'full_typed_args':
', '.join(full_typed_args),
'type_index':
type_index,
'c_func_type':
c_func_type,
'args': (', ' + ', '.join(args)) if args else '',
'declare_return':
(s_to_c(sig[0]) +
' returned_value = 0;') if sig[0] != 'v' else '',
'receive':
'returned_value = ' if sig[0] != 'v' else '',
'return':
'return returned_value;' if sig[0] != 'v' else ''
})
total += '\n'.join(invokes)
# adjust sandboxing
TRAP_OOB = 'TRAP(OOB)'
assert total.count(TRAP_OOB) == 2
if settings.WASM2C_SANDBOXING == 'full':
pass # keep it
elif settings.WASM2C_SANDBOXING == 'none':
total = total.replace(TRAP_OOB, '{}')
elif settings.WASM2C_SANDBOXING == 'mask':
assert not settings.ALLOW_MEMORY_GROWTH
assert (settings.INITIAL_MEMORY &
(settings.INITIAL_MEMORY - 1)) == 0, 'poewr of 2'
total = total.replace(TRAP_OOB, '{}')
MEM_ACCESS = '[addr]'
assert total.count(MEM_ACCESS) == 3, '2 from wasm2c, 1 from runtime'
total = total.replace(MEM_ACCESS,
'[addr & %d]' % (settings.INITIAL_MEMORY - 1))
else:
exit_with_error('bad sandboxing')
# adjust prefixing: emit simple output that works with multiple libraries,
# each compiled into its own single .c file, by adding 'static' in some places
# TODO: decide on the proper pattern for this in an upstream discussion in
# wasm2c; another option would be to prefix all these things.
for rep in [
'uint32_t wasm_rt_register_func_type(',
'void wasm_rt_trap(',
'void wasm_rt_allocate_memory(',
'uint32_t wasm_rt_grow_memory(',
'void wasm_rt_allocate_table(',
'jmp_buf g_jmp_buf',
'uint32_t g_func_type_count',
'FuncType* g_func_types',
'uint32_t wasm_rt_call_stack_depth',
'uint32_t g_saved_call_stack_depth',
]:
# remove 'extern' from declaration
total = total.replace('extern ' + rep, rep)
# add 'static' to implementation
old = total
total = total.replace(rep, 'static ' + rep)
assert old != total, f'did not find "{rep}"'
# write out the final file
with open(c_file, 'w') as out:
out.write(total)
def do_wasm2c(infile):
assert Settings.STANDALONE_WASM
WASM2C = NODE_JS + [path_from_root('node_modules', 'wasm2c', 'wasm2c.js')]
WASM2C_DIR = path_from_root('node_modules', 'wasm2c')
c_file = unsuffixed(infile) + '.wasm.c'
h_file = unsuffixed(infile) + '.wasm.h'
cmd = WASM2C + [infile, '-o', c_file]
check_call(cmd)
total = '''\
/*
* This file was generated by emcc+wasm2c. To compile it, use something like
*
* $CC FILE.c -O2 -lm -DWASM_RT_MAX_CALL_STACK_DEPTH=8000
*/
'''
SEP = '\n/* ==================================== */\n'
def bundle_file(total, filename):
with open(filename) as f:
total += '// ' + filename + '\n' + f.read() + SEP
return total
# hermeticize the C file, by bundling in the wasm2c/ includes
headers = [(WASM2C_DIR, 'wasm-rt.h'), (WASM2C_DIR, 'wasm-rt-impl.h'),
('', h_file)]
for header in headers:
total = bundle_file(total, os.path.join(header[0], header[1]))
# add the wasm2c output
with open(c_file) as read_c:
c = read_c.read()
total += c + SEP
# add the wasm2c runtime
total = bundle_file(total, os.path.join(WASM2C_DIR, 'wasm-rt-impl.c'))
# add the support code
support_files = ['base']
if Settings.AUTODEBUG:
support_files.append('autodebug')
if Settings.EXPECT_MAIN:
# TODO: add an option for direct OS access. For now, do that when building
# an executable with main, as opposed to a library
support_files.append('os')
support_files.append('main')
else:
support_files.append('os_sandboxed')
support_files.append('reactor')
# for a reactor, also append wasmbox_* API definitions
with open(h_file, 'a') as f:
f.write('''
// wasmbox_* API
// TODO: optional prefixing
extern void wasmbox_init(void);
''')
for support_file in support_files:
total = bundle_file(
total, path_from_root('tools', 'wasm2c', support_file + '.c'))
# remove #includes of the headers we bundled
for header in headers:
total = total.replace('#include "%s"\n' % header[1],
'/* include of %s */\n' % header[1])
# generate the necessary invokes
invokes = []
for sig in re.findall(r"\/\* import\: 'env' 'invoke_(\w+)' \*\/", total):
def s_to_c(s):
if s == 'v':
return 'void'
elif s == 'i':
return 'u32'
elif s == 'j':
return 'u64'
elif s == 'f':
return 'f32'
elif s == 'd':
return 'f64'
else:
exit_with_error('invalid sig element:' + str(s))
def name(i):
return 'a' + str(i)
wabt_sig = sig[0] + 'i' + sig[1:]
typed_args = ['u32 fptr'] + [
s_to_c(sig[i]) + ' ' + name(i) for i in range(1, len(sig))
]
types = ['u32'] + [s_to_c(sig[i]) for i in range(1, len(sig))]
args = ['fptr'] + [name(i) for i in range(1, len(sig))]
invokes.append(
'%s_INVOKE_IMPL(%sZ_envZ_invoke_%sZ_%s, (%s), (%s), (%s), Z_dynCall_%sZ_%s);'
% ('VOID' if sig[0] == 'v' else 'RETURNING',
(s_to_c(sig[0]) + ', ') if sig[0] != 'v' else '', sig, wabt_sig,
', '.join(typed_args), ', '.join(types), ', '.join(args), sig,
wabt_sig))
total += '\n'.join(invokes)
# write out the final file
with open(c_file, 'w') as out:
out.write(total)
decrunchCallbacks[msg.data.callbackID] = null;
};
function requestDecrunch(filename, data, callback) {
decrunchWorker.postMessage({
filename: filename,
data: new Uint8Array(data),
callbackID: decrunchCallbacks.length
});
decrunchCallbacks.push(callback);
}
'''
for file_ in data_files:
if file_['dstpath'].endswith(CRUNCH_INPUT_SUFFIX):
src_dds_name = file_['srcpath']
src_crunch_name = unsuffixed(src_dds_name) + CRUNCH_OUTPUT_SUFFIX
# Preload/embed the .crn version instead of the .dds version, but use the .dds suffix for the target file in the virtual FS.
file_['srcpath'] = src_crunch_name
try:
# Do not crunch if crunched version exists and is more recent than dds source
crunch_time = os.stat(src_crunch_name).st_mtime
dds_time = os.stat(src_dds_name).st_mtime
if dds_time < crunch_time: continue
except:
pass # if one of them does not exist, continue on
# guess at format. this lets us tell crunch to not try to be clever and use odd formats like DXT5_AGBR
try:
format = run_process(['file', file_['srcpath']], stdout=PIPE).stdout
