def create_named_globals(metadata): named_globals = [] for k, v in metadata['namedGlobals'].items(): v = int(v) if shared.Settings.RELOCATABLE: v += shared.Settings.GLOBAL_BASE elif k == '__data_end': # We keep __data_end alive internally so that wasm-emscripten-finalize knows where the # static data region ends. Don't export this to JS like other user-exported global # address. continue mangled = asmjs_mangle(k) if shared.Settings.MINIMAL_RUNTIME: named_globals.append("var %s = %s;" % (mangled, v)) else: named_globals.append("var %s = Module['%s'] = %s;" % (mangled, mangled, v)) named_globals = '\n'.join(named_globals) if shared.Settings.RELOCATABLE: # wasm side modules are pure wasm, and cannot create their g$..() methods, so we help them out # TODO: this works if we are the main module, but if the supplying module is later, it won't, so # we'll need another solution for that. one option is to scan the module imports, if/when # wasm supports that, then the loader can do this. names = ["'%s'" % n for n in metadata['namedGlobals']] named_globals += ''' for (var name in [%s]) { (function(name) { Module['g$' + name] = function() { return Module[name]; }; })(name); } ''' % ','.join(names) return named_globals
def report_missing_symbols(js_library_funcs): # Report any symbol that was explicitly exported but is present neither # as a native function nor as a JS library function. defined_symbols = set( asmjs_mangle(e) for e in settings.WASM_EXPORTS).union(js_library_funcs) missing = set(settings.USER_EXPORTED_FUNCTIONS) - defined_symbols for symbol in sorted(missing): diagnostics.warning('undefined', f'undefined exported symbol: "{symbol}"') # Special hanlding for the `_main` symbol if settings.STANDALONE_WASM: # standalone mode doesn't use main, and it always reports missing entry point at link time. # In this mode we never expect _main in the export list. return # PROXY_TO_PTHREAD only makes sense with a main(), so error if one is # missing. note that when main() might arrive from another module we cannot # error here. if settings.PROXY_TO_PTHREAD and '_main' not in defined_symbols and \ not settings.RELOCATABLE: exit_with_error( 'PROXY_TO_PTHREAD proxies main() for you, but no main exists') if settings.IGNORE_MISSING_MAIN: # The default mode for emscripten is to ignore the missing main function allowing # maximum compatibility. return if settings.EXPECT_MAIN and 'main' not in settings.WASM_EXPORTS: # For compatibility with the output of wasm-ld we use the same wording here in our # error message as if wasm-ld had failed (i.e. in LLD_REPORT_UNDEFINED mode). exit_with_error( 'entry symbol not defined (pass --no-entry to suppress): main')
def make_export_wrappers(exports, delay_assignment): wrappers = [] for name in exports: mangled = asmjs_mangle(name) # The emscripten stack functions are called very early (by writeStackCookie) before # the runtime is initialized so we can't create these wrappers that check for # runtimeInitialized. if shared.Settings.ASSERTIONS and not name.startswith('emscripten_stack_'): # With assertions enabled we create a wrapper that are calls get routed through, for # the lifetime of the program. if delay_assignment: wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = createExportWrapper("%(name)s"); ''' % {'mangled': mangled, 'name': name}) else: wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = createExportWrapper("%(name)s", asm); ''' % {'mangled': mangled, 'name': name}) elif delay_assignment: # With assertions disabled the wrapper will replace the global var and Module var on # first use. wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = function() { return (%(mangled)s = Module["%(mangled)s"] = Module["asm"]["%(name)s"]).apply(null, arguments); }; ''' % {'mangled': mangled, 'name': name}) else: wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = asm["%(name)s"] ''' % {'mangled': mangled, 'name': name}) return wrappers
def compute_minimal_runtime_initializer_and_exports(post, initializers, exports, receiving): # Generate invocations for all global initializers directly off the asm export object, e.g. asm['__GLOBAL__INIT'](); post = post.replace( '/*** RUN_GLOBAL_INITIALIZERS(); ***/', '\n'.join([ "asm['" + x + "']();" for x in global_initializer_funcs(initializers) ])) if shared.Settings.WASM: # Declare all exports out to global JS scope so that JS library functions can access them in a # way that minifies well with Closure # e.g. var a,b,c,d,e,f; exports_that_are_not_initializers = [ x for x in exports if x not in initializers ] # In Wasm backend the exports are still unmangled at this point, so mangle the names here exports_that_are_not_initializers = [ asmjs_mangle(x) for x in exports_that_are_not_initializers ] post = post.replace( '/*** ASM_MODULE_EXPORTS_DECLARES ***/', 'var ' + ','.join(exports_that_are_not_initializers) + ';') # Generate assignments from all asm.js/wasm exports out to the JS variables above: e.g. a = asm['a']; b = asm['b']; post = post.replace('/*** ASM_MODULE_EXPORTS ***/', receiving) receiving = '' return post, receiving
def compute_minimal_runtime_initializer_and_exports(post, exports, receiving): # Declare all exports out to global JS scope so that JS library functions can access them in a # way that minifies well with Closure # e.g. var a,b,c,d,e,f; exports_that_are_not_initializers = [ x for x in exports if x not in WASM_INIT_FUNC ] # In Wasm backend the exports are still unmangled at this point, so mangle the names here exports_that_are_not_initializers = [ asmjs_mangle(x) for x in exports_that_are_not_initializers ] # Decide whether we should generate the global dynCalls dictionary for the dynCall() function? if shared.Settings.DYNCALLS and '$dynCall' in shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE and len( [ x for x in exports_that_are_not_initializers if x.startswith('dynCall_') ]) > 0: exports_that_are_not_initializers += ['dynCalls = {}'] post = post.replace( '/*** ASM_MODULE_EXPORTS_DECLARES ***/', 'var ' + ',\n '.join(exports_that_are_not_initializers) + ';') # Generate assignments from all asm.js/wasm exports out to the JS variables above: e.g. a = asm['a']; b = asm['b']; post = post.replace('/*** ASM_MODULE_EXPORTS ***/', receiving) return post
def make_export_wrappers(exports, delay_assignment): wrappers = [] for name in exports: mangled = asmjs_mangle(name) if shared.Settings.ASSERTIONS: # With assertions enabled we create a wrapper that are calls get routed through, for # the lifetime of the program. if delay_assignment: wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = createExportWrapper("%(name)s"); ''' % {'mangled': mangled, 'name': name}) else: wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = createExportWrapper("%(name)s", asm); ''' % {'mangled': mangled, 'name': name}) elif delay_assignment: # With assertions disabled the wrapper will replace the global var and Module var on # first use. wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = function() { return (%(mangled)s = Module["%(mangled)s"] = Module["asm"]["%(name)s"]).apply(null, arguments); }; ''' % {'mangled': mangled, 'name': name}) else: wrappers.append('''\ /** @type {function(...*):?} */ var %(mangled)s = Module["%(mangled)s"] = asm["%(name)s"] ''' % {'mangled': mangled, 'name': name}) return wrappers
def report_missing_symbols(pre): # the initial list of missing functions are that the user explicitly exported # but were not implemented in compiled code missing = set(settings.USER_EXPORTED_FUNCTIONS) - set(asmjs_mangle(e) for e in settings.WASM_EXPORTS) for requested in sorted(missing): if (f'function {requested}(') not in pre: diagnostics.warning('undefined', f'undefined exported symbol: "{requested}"') # Special hanlding for the `_main` symbol if settings.STANDALONE_WASM: # standalone mode doesn't use main, and it always reports missing entry point at link time. # In this mode we never expect _main in the export list. return if settings.IGNORE_MISSING_MAIN: # The default mode for emscripten is to ignore the missing main function allowing # maximum compatibility. return if settings.EXPECT_MAIN and 'main' not in settings.WASM_EXPORTS: # For compatibility with the output of wasm-ld we use the same wording here in our # error message as if wasm-ld had failed (i.e. in LLD_REPORT_UNDEFINED mode). exit_with_error('entry symbol not defined (pass --no-entry to suppress): main')
def update_settings_glue(metadata, DEBUG): optimize_syscalls(metadata['declares'], DEBUG) # Integrate info from backend if shared.Settings.SIDE_MODULE: # we don't need any JS library contents in side modules shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE = [] all_funcs = shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE + [ shared.JS.to_nice_ident(d) for d in metadata['declares'] ] shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE = sorted( set(all_funcs).difference(metadata['exports'])) shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += metadata['externs'] # With the wasm backend the set of implemented functions is identical to the set of exports shared.Settings.IMPLEMENTED_FUNCTIONS = [ asmjs_mangle(x) for x in metadata['exports'] ] shared.Settings.BINARYEN_FEATURES = metadata['features'] if shared.Settings.RELOCATABLE: # When building relocatable output (e.g. MAIN_MODULE) the reported table # size does not include the reserved slot at zero for the null pointer. # Instead we use __table_base to offset the elements by 1. if shared.Settings.INITIAL_TABLE == -1: shared.Settings.INITIAL_TABLE = metadata['tableSize'] + 1 shared.Settings.HAS_MAIN = shared.Settings.MAIN_MODULE or shared.Settings.STANDALONE_WASM or '_main' in shared.Settings.IMPLEMENTED_FUNCTIONS # When using dynamic linking the main function might be in a side module. # To be safe assume they do take input parametes. shared.Settings.MAIN_READS_PARAMS = metadata[ 'mainReadsParams'] or shared.Settings.MAIN_MODULE # Store exports for Closure compiler to be able to track these as globals in # -s DECLARE_ASM_MODULE_EXPORTS=0 builds. shared.Settings.MODULE_EXPORTS = [(asmjs_mangle(f), f) for f in metadata['exports']] if shared.Settings.STACK_OVERFLOW_CHECK and not shared.Settings.SIDE_MODULE: shared.Settings.EXPORTED_RUNTIME_METHODS += [ 'writeStackCookie', 'checkStackCookie' ] # writeStackCookie and checkStackCookie both rely on emscripten_stack_get_end being # exported. In theory it should always be present since its defined in compiler-rt. assert 'emscripten_stack_get_end' in metadata['exports']
def load_metadata_wasm(metadata_raw, DEBUG): try: metadata_json = json.loads(metadata_raw) except Exception: logger.error( 'emscript: failure to parse metadata output from wasm-emscripten-finalize. raw output is: \n' + metadata_raw) raise metadata = { 'declares': [], 'externs': [], 'staticBump': 0, 'tableSize': 0, 'exports': [], 'namedGlobals': {}, 'emJsFuncs': {}, 'asmConsts': {}, 'invokeFuncs': [], 'features': [], 'mainReadsParams': 1, } legacy_keys = set(['implementedFunctions', 'initializers', 'simd']) assert 'tableSize' in metadata_json.keys() for key, value in metadata_json.items(): if key in legacy_keys: continue # json.loads returns `unicode` for strings but other code in this file # generally works with utf8 encoded `str` objects, and they don't alwasy # mix well. e.g. s.replace(x, y) will blow up is `s` a uts8 str containing # non-ascii and either x or y are unicode objects. # TODO(sbc): Remove this encoding if we switch to unicode elsewhere # (specifically the glue returned from compile_settings) if type(value) == list: value = [asstr(v) for v in value] if key not in metadata: exit_with_error( 'unexpected metadata key received from wasm-emscripten-finalize: %s', key) metadata[key] = value if DEBUG: logger.debug("Metadata parsed: " + pprint.pformat(metadata)) # Calculate the subset of exports that were explicitly marked with llvm.used. # These are any exports that were not requested on the command line and are # not known auto-generated system functions. unexpected_exports = [ e for e in metadata['exports'] if treat_as_user_function(e) ] unexpected_exports = [asmjs_mangle(e) for e in unexpected_exports] unexpected_exports = [ e for e in unexpected_exports if e not in shared.Settings.EXPORTED_FUNCTIONS ] building.user_requested_exports += unexpected_exports return metadata
def load_metadata_wasm(metadata_raw, DEBUG): try: metadata_json = json.loads(metadata_raw) except Exception: logger.error( 'emscript: failure to parse metadata output from wasm-emscripten-finalize. raw output is: \n' + metadata_raw) raise metadata = { 'declares': [], 'externs': [], 'staticBump': 0, 'tableSize': 0, 'exports': [], 'namedGlobals': {}, 'emJsFuncs': {}, 'asmConsts': {}, 'invokeFuncs': [], 'features': [], 'mainReadsParams': 1, } legacy_keys = set(['implementedFunctions', 'initializers', 'simd']) assert 'tableSize' in metadata_json.keys() for key, value in metadata_json.items(): if key in legacy_keys: continue if key not in metadata: exit_with_error( 'unexpected metadata key received from wasm-emscripten-finalize: %s', key) metadata[key] = value # Support older metadata when asmConsts values were lists. We only use the first element # nowadays # TODO(sbc): remove this once binaryen has been changed to only emit the single element metadata['asmConsts'] = { k: v[0] if type(v) is list else v for k, v in metadata['asmConsts'].items() } if DEBUG: logger.debug("Metadata parsed: " + pprint.pformat(metadata)) # Calculate the subset of exports that were explicitly marked with llvm.used. # These are any exports that were not requested on the command line and are # not known auto-generated system functions. unexpected_exports = [ e for e in metadata['exports'] if treat_as_user_function(e) ] unexpected_exports = [asmjs_mangle(e) for e in unexpected_exports] unexpected_exports = [ e for e in unexpected_exports if e not in shared.Settings.EXPORTED_FUNCTIONS ] building.user_requested_exports += unexpected_exports return metadata
def create_receiving(exports): # When not declaring asm exports this section is empty and we instead programatically export # symbols on the global object by calling exportAsmFunctions after initialization if not shared.Settings.DECLARE_ASM_MODULE_EXPORTS: return '' exports_that_are_not_initializers = [x for x in exports if x != WASM_INIT_FUNC] receiving = [] # with WASM_ASYNC_COMPILATION that asm object may not exist at this point in time # so we need to support delayed assignment. delay_assignment = shared.Settings.WASM_ASYNC_COMPILATION and not shared.Settings.MINIMAL_RUNTIME if not delay_assignment: if shared.Settings.MINIMAL_RUNTIME: # In Wasm exports are assigned inside a function to variables existing in top level JS scope, i.e. # var _main; # WebAssembly.instantiate(Module["wasm"], imports).then((function(output) { # var asm = output.instance.exports; # _main = asm["_main"]; receiving += [asmjs_mangle(s) + ' = asm["' + s + '"];' for s in exports_that_are_not_initializers] else: if shared.Settings.MINIMAL_RUNTIME: # In wasm2js exports can be directly processed at top level, i.e. # var asm = Module["asm"](asmLibraryArg, buffer); # var _main = asm["_main"]; if shared.Settings.USE_PTHREADS and shared.Settings.MODULARIZE: # TODO: As a temp solution, multithreaded MODULARIZED MINIMAL_RUNTIME builds export all # symbols like regular runtime does. # Fix this by migrating worker.js code to reside inside the Module so it is in the same # scope as the rest of the JS code, or by defining an export syntax to MINIMAL_RUNTIME # that multithreaded MODULARIZEd builds can export on. receiving += [asmjs_mangle(s) + ' = Module["' + asmjs_mangle(s) + '"] = asm["' + s + '"];' for s in exports_that_are_not_initializers] else: receiving += ['var ' + asmjs_mangle(s) + ' = asm["' + asmjs_mangle(s) + '"];' for s in exports_that_are_not_initializers] else: receiving += make_export_wrappers(exports, delay_assignment) else: receiving += make_export_wrappers(exports, delay_assignment) if shared.Settings.MINIMAL_RUNTIME: return '\n '.join(receiving) + '\n' else: return '\n'.join(receiving) + '\n'
def update_settings_glue(metadata, DEBUG): optimize_syscalls(metadata['declares'], DEBUG) # Integrate info from backend if shared.Settings.SIDE_MODULE: # we don't need any JS library contents in side modules shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE = [] all_funcs = shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE + [shared.JS.to_nice_ident(d) for d in metadata['declares']] implemented_funcs = [x[1:] for x in metadata['implementedFunctions']] shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE = sorted(set(all_funcs).difference(implemented_funcs)) shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += [x[1:] for x in metadata['externs']] shared.Settings.IMPLEMENTED_FUNCTIONS = metadata['implementedFunctions'] if metadata['asmConsts']: # emit the EM_ASM signature-reading helper function only if we have any EM_ASM # functions in the module. shared.Settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE += ['$readAsmConstArgs'] # Extract the list of function signatures that MAIN_THREAD_EM_ASM blocks in # the compiled code have, each signature will need a proxy function invoker # generated for it. def read_proxied_function_signatures(asmConsts): proxied_function_signatures = set() for _, sigs, proxying_types in asmConsts.values(): for sig, proxying_type in zip(sigs, proxying_types): if proxying_type == 'sync_on_main_thread_': proxied_function_signatures.add(sig + '_sync') elif proxying_type == 'async_on_main_thread_': proxied_function_signatures.add(sig + '_async') return list(proxied_function_signatures) shared.Settings.PROXIED_FUNCTION_SIGNATURES = read_proxied_function_signatures(metadata['asmConsts']) shared.Settings.BINARYEN_FEATURES = metadata['features'] if shared.Settings.RELOCATABLE: # When building relocatable output (e.g. MAIN_MODULE) the reported table # size does not include the reserved slot at zero for the null pointer. # Instead we use __table_base to offset the elements by 1. if shared.Settings.INITIAL_TABLE == -1: shared.Settings.INITIAL_TABLE = metadata['tableSize'] + 1 shared.Settings.MAIN_READS_PARAMS = metadata['mainReadsParams'] # Store exports for Closure compiler to be able to track these as globals in # -s DECLARE_ASM_MODULE_EXPORTS=0 builds. shared.Settings.MODULE_EXPORTS = [(asmjs_mangle(f), f) for f in metadata['exports']] if shared.Settings.STACK_OVERFLOW_CHECK: if 'emscripten_stack_get_end' not in metadata['exports']: logger.warning('STACK_OVERFLOW_CHECK disabled because emscripten stack helpers not exported') shared.Settings.STACK_OVERFLOW_CHECK = 0 else: shared.Settings.EXPORTED_RUNTIME_METHODS += ['writeStackCookie', 'checkStackCookie']
def create_fp_accessors(metadata): if not shared.Settings.RELOCATABLE: return '' # Create `fp$XXX` handlers for determining function pionters (table addresses) # at runtime. # For SIDE_MODULEs these are generated by the proxyHandler at runtime. accessors = [] for fullname in metadata['declares']: if not fullname.startswith('fp$'): continue _, name, sig = fullname.split('$') mangled = asmjs_mangle(name) side = 'parent' if shared.Settings.SIDE_MODULE else '' assertion = ('\n assert(%sModule["%s"] || typeof %s !== "undefined", "external function `%s` is missing.' % (side, mangled, mangled, name) + 'perhaps a side module was not linked in? if this symbol was expected to arrive ' 'from a system library, try to build the MAIN_MODULE with ' 'EMCC_FORCE_STDLIBS=XX in the environment");') # the name of the original function is generally the normal function # name, unless it is legalized, in which case the export is the legalized # version, and the original provided by orig$X if shared.Settings.LEGALIZE_JS_FFI and not shared.JS.is_legal_sig(sig): name = 'orig$' + name accessors.append(''' Module['%(full)s'] = function() { %(assert)s // Use the original wasm function itself, for the table, from the main module. var func = Module['asm']['%(original)s']; // Try an original version from a side module. if (!func) func = Module['_%(original)s']; // Otherwise, look for a regular function or JS library function. if (!func) func = Module['%(mangled)s']; if (!func) func = %(mangled)s; var fp = addFunction(func, '%(sig)s'); Module['%(full)s'] = function() { return fp }; return fp; } ''' % {'full': asmjs_mangle(fullname), 'mangled': mangled, 'original': name, 'assert': assertion, 'sig': sig}) return '\n'.join(accessors)
def compute_minimal_runtime_initializer_and_exports(post, exports, receiving): # Declare all exports out to global JS scope so that JS library functions can access them in a # way that minifies well with Closure # e.g. var a,b,c,d,e,f; exports_that_are_not_initializers = [x for x in exports if x not in WASM_INIT_FUNC] # In Wasm backend the exports are still unmangled at this point, so mangle the names here exports_that_are_not_initializers = [asmjs_mangle(x) for x in exports_that_are_not_initializers] post = post.replace('/*** ASM_MODULE_EXPORTS_DECLARES ***/', 'var ' + ',\n '.join(exports_that_are_not_initializers) + ';') # Generate assignments from all asm.js/wasm exports out to the JS variables above: e.g. a = asm['a']; b = asm['b']; post = post.replace('/*** ASM_MODULE_EXPORTS ***/', receiving) return post
def create_named_globals(metadata): named_globals = [] for k, v in metadata['namedGlobals'].items(): v = int(v) if shared.Settings.RELOCATABLE: v += shared.Settings.GLOBAL_BASE mangled = asmjs_mangle(k) if shared.Settings.MINIMAL_RUNTIME: named_globals.append("var %s = %s;" % (mangled, v)) else: named_globals.append("var %s = Module['%s'] = %s;" % (mangled, mangled, v)) return '\n'.join(named_globals)
def create_named_globals(metadata): named_globals = [] for k, v in metadata['namedGlobals'].items(): v = int(v) if shared.Settings.RELOCATABLE: v += shared.Settings.GLOBAL_BASE elif k == '__data_end': # We keep __data_end alive internally so that wasm-emscripten-finalize knows where the # static data region ends. Don't export this to JS like other user-exported global # address. continue mangled = asmjs_mangle(k) if shared.Settings.MINIMAL_RUNTIME: named_globals.append("var %s = %s;" % (mangled, v)) else: named_globals.append("var %s = Module['%s'] = %s;" % (mangled, mangled, v)) return '\n'.join(named_globals)
def create_receiving(exports): # When not declaring asm exports this section is empty and we instead programatically export # symbols on the global object by calling exportAsmFunctions after initialization if not settings.DECLARE_ASM_MODULE_EXPORTS: return '' receiving = [] # with WASM_ASYNC_COMPILATION that asm object may not exist at this point in time # so we need to support delayed assignment. delay_assignment = settings.WASM_ASYNC_COMPILATION and not settings.MINIMAL_RUNTIME if not delay_assignment: if settings.MINIMAL_RUNTIME: # In Wasm exports are assigned inside a function to variables # existing in top level JS scope, i.e. # var _main; # WebAssembly.instantiate(Module["wasm"], imports).then((function(output) { # var asm = output.instance.exports; # _main = asm["_main"]; generate_dyncall_assignment = settings.DYNCALLS and '$dynCall' in settings.DEFAULT_LIBRARY_FUNCS_TO_INCLUDE exports_that_are_not_initializers = [ x for x in exports if x != building.WASM_CALL_CTORS ] for s in exports_that_are_not_initializers: mangled = asmjs_mangle(s) dynCallAssignment = ( 'dynCalls["' + s.replace('dynCall_', '') + '"] = ' ) if generate_dyncall_assignment and mangled.startswith( 'dynCall_') else '' receiving += [ dynCallAssignment + mangled + ' = asm["' + s + '"];' ] else: receiving += make_export_wrappers(exports, delay_assignment) else: receiving += make_export_wrappers(exports, delay_assignment) if settings.MINIMAL_RUNTIME: return '\n '.join(receiving) + '\n' else: return '\n'.join(receiving) + '\n'
def load_metadata_wasm(metadata_raw, DEBUG): try: metadata_json = json.loads(metadata_raw) except Exception: logger.error( 'emscript: failure to parse metadata output from wasm-emscripten-finalize. raw output is: \n' + metadata_raw) raise metadata = { 'aliases': {}, 'declares': [], 'implementedFunctions': [], 'externs': [], 'simd': False, # Obsolete, always False 'maxGlobalAlign': 0, 'staticBump': 0, 'tableSize': 0, 'initializers': [], 'exports': [], 'namedGlobals': {}, 'emJsFuncs': {}, 'asmConsts': {}, 'invokeFuncs': [], 'features': [], 'mainReadsParams': 1, } assert 'tableSize' in metadata_json.keys() for key, value in metadata_json.items(): # json.loads returns `unicode` for strings but other code in this file # generally works with utf8 encoded `str` objects, and they don't alwasy # mix well. e.g. s.replace(x, y) will blow up is `s` a uts8 str containing # non-ascii and either x or y are unicode objects. # TODO(sbc): Remove this encoding if we switch to unicode elsewhere # (specifically the glue returned from compile_settings) if type(value) == list: value = [asstr(v) for v in value] if key not in metadata: exit_with_error( 'unexpected metadata key received from wasm-emscripten-finalize: %s', key) metadata[key] = value if not shared.Settings.MINIMAL_RUNTIME: # In regular runtime initializers call the global var version of the export, so they get the mangled name. # In MINIMAL_RUNTIME, the initializers are called directly off the export object for minimal code size. metadata['initializers'] = [ asmjs_mangle(i) for i in metadata['initializers'] ] if DEBUG: logger.debug("Metadata parsed: " + pprint.pformat(metadata)) # Calculate the subset of exports that were explicitly marked with llvm.used. # These are any exports that were not requested on the command line and are # not known auto-generated system functions. unexpected_exports = [ e for e in metadata['exports'] if treat_as_user_function(e) ] unexpected_exports = [asmjs_mangle(e) for e in unexpected_exports] unexpected_exports = [ e for e in unexpected_exports if e not in shared.Settings.EXPORTED_FUNCTIONS ] building.user_requested_exports += unexpected_exports # With the wasm backend the set of implemented functions is identical to the set of exports # Set this key here simply so that the shared code that handle it. metadata['implementedFunctions'] = [ asmjs_mangle(x) for x in metadata['exports'] ] return metadata
def emscript(infile, outfile, memfile, temp_files, DEBUG): # Overview: # * Run wasm-emscripten-finalize to extract metadata and modify the binary # to use emscripten's wasm<->JS ABI # * Use the metadata to generate the JS glue that goes with the wasm metadata = finalize_wasm(temp_files, infile, outfile, memfile, DEBUG) update_settings_glue(metadata, DEBUG) if shared.Settings.SIDE_MODULE: return if DEBUG: logger.debug('emscript: js compiler glue') if DEBUG: t = time.time() glue, forwarded_data = compile_settings(temp_files) if DEBUG: logger.debug(' emscript: glue took %s seconds' % (time.time() - t)) t = time.time() forwarded_json = json.loads(forwarded_data) # For the wasm backend the implementedFunctions from compiler.js should # always be empty. This only gets populated for __asm function when using # the JS backend. assert not forwarded_json['Functions']['implementedFunctions'] pre, post = glue.split('// EMSCRIPTEN_END_FUNCS') # memory and global initializers global_initializers = ', '.join('{ func: function() { %s() } }' % i for i in metadata['initializers']) staticbump = shared.Settings.STATIC_BUMP if shared.Settings.MINIMAL_RUNTIME: # In minimal runtime, global initializers are run after the Wasm Module instantiation has finished. global_initializers = '' else: # In regular runtime, global initializers are recorded in an __ATINIT__ array. global_initializers = '''/* global initializers */ %s __ATINIT__.push(%s); ''' % ('if (!ENVIRONMENT_IS_PTHREAD)' if shared.Settings.USE_PTHREADS else '', global_initializers) pre = pre.replace( 'STATICTOP = STATIC_BASE + 0;', '''STATICTOP = STATIC_BASE + %d; %s ''' % (staticbump, global_initializers)) pre = apply_memory(pre) pre = apply_static_code_hooks( pre) # In regular runtime, atinits etc. exist in the preamble part post = apply_static_code_hooks( post) # In MINIMAL_RUNTIME, atinit exists in the postamble part if shared.Settings.RELOCATABLE and not shared.Settings.SIDE_MODULE: pre += 'var gb = GLOBAL_BASE, fb = 0;\n' # merge forwarded data shared.Settings.EXPORTED_FUNCTIONS = forwarded_json['EXPORTED_FUNCTIONS'] exports = metadata['exports'] # Store exports for Closure compiler to be able to track these as globals in # -s DECLARE_ASM_MODULE_EXPORTS=0 builds. shared.Settings.MODULE_EXPORTS = [(asmjs_mangle(f), f) for f in exports] if shared.Settings.ASYNCIFY: exports += [ 'asyncify_start_unwind', 'asyncify_stop_unwind', 'asyncify_start_rewind', 'asyncify_stop_rewind' ] report_missing_symbols(set([asmjs_mangle(f) for f in exports]), pre) asm_consts = create_asm_consts_wasm(forwarded_json, metadata) em_js_funcs = create_em_js(forwarded_json, metadata) asm_const_pairs = ['%s: %s' % (key, value) for key, value in asm_consts] asm_const_map = 'var ASM_CONSTS = {\n ' + ', \n '.join( asm_const_pairs) + '\n};\n' pre = pre.replace('// === Body ===', ('// === Body ===\n\n' + asm_const_map + '\n'.join(em_js_funcs) + '\n')) pre = apply_table(pre) outfile.write(pre) pre = None invoke_funcs = metadata['invokeFuncs'] try: del forwarded_json['Variables']['globals'][ '_llvm_global_ctors'] # not a true variable except KeyError: pass sending = create_sending_wasm(invoke_funcs, forwarded_json, metadata) receiving = create_receiving_wasm(exports, metadata['initializers']) if shared.Settings.MINIMAL_RUNTIME: post = compute_minimal_runtime_initializer_and_exports( post, metadata['initializers'], exports, receiving) receiving = '' module = create_module_wasm(sending, receiving, invoke_funcs, metadata) write_output_file(outfile, post, module) module = None outfile.close()
def emscript(in_wasm, out_wasm, outfile_js, memfile, temp_files, DEBUG): # Overview: # * Run wasm-emscripten-finalize to extract metadata and modify the binary # to use emscripten's wasm<->JS ABI # * Use the metadata to generate the JS glue that goes with the wasm if shared.Settings.SINGLE_FILE: # placeholder strings for JS glue, to be replaced with subresource locations in do_binaryen shared.Settings.WASM_BINARY_FILE = '<<< WASM_BINARY_FILE >>>' else: # set file locations, so that JS glue can find what it needs shared.Settings.WASM_BINARY_FILE = shared.JS.escape_for_js_string( os.path.basename(out_wasm)) metadata = finalize_wasm(in_wasm, out_wasm, memfile, DEBUG) update_settings_glue(metadata, DEBUG) if shared.Settings.SIDE_MODULE: logger.debug('emscript: skipping remaining js glue generation') return if DEBUG: logger.debug('emscript: js compiler glue') t = time.time() glue, forwarded_data = compile_settings(temp_files) if DEBUG: logger.debug(' emscript: glue took %s seconds' % (time.time() - t)) t = time.time() forwarded_json = json.loads(forwarded_data) # For the wasm backend the implementedFunctions from compiler.js should # always be empty. This only gets populated for __asm function when using # the JS backend. assert not forwarded_json['Functions']['implementedFunctions'] pre, post = glue.split('// EMSCRIPTEN_END_FUNCS') exports = metadata['exports'] if shared.Settings.ASYNCIFY: exports += [ 'asyncify_start_unwind', 'asyncify_stop_unwind', 'asyncify_start_rewind', 'asyncify_stop_rewind' ] all_exports = exports + list(metadata['namedGlobals'].keys()) all_exports = set([asmjs_mangle(e) for e in all_exports]) report_missing_symbols(all_exports, pre) if not outfile_js: logger.debug('emscript: skipping remaining js glue generation') return # memory and global initializers if shared.Settings.RELOCATABLE: static_bump = align_memory( webassembly.parse_dylink_section(in_wasm)[0]) memory = Memory(static_bump) logger.debug('stack_base: %d, stack_max: %d, dynamic_base: %d', memory.stack_base, memory.stack_max, memory.dynamic_base) pre = apply_memory(pre, memory) post = apply_memory(post, memory) pre = apply_static_code_hooks( pre) # In regular runtime, atinits etc. exist in the preamble part post = apply_static_code_hooks( post) # In MINIMAL_RUNTIME, atinit exists in the postamble part # merge forwarded data shared.Settings.EXPORTED_FUNCTIONS = forwarded_json['EXPORTED_FUNCTIONS'] asm_consts = create_asm_consts(metadata) em_js_funcs = create_em_js(forwarded_json, metadata) asm_const_pairs = ['%s: %s' % (key, value) for key, value in asm_consts] asm_const_map = 'var ASM_CONSTS = {\n ' + ', \n '.join( asm_const_pairs) + '\n};\n' pre = pre.replace('// === Body ===', ('// === Body ===\n\n' + asm_const_map + '\n'.join(em_js_funcs) + '\n')) with open(outfile_js, 'w') as out: out.write(pre) pre = None invoke_funcs = metadata['invokeFuncs'] sending = create_sending(invoke_funcs, metadata) receiving = create_receiving(exports) if shared.Settings.MINIMAL_RUNTIME: post = compute_minimal_runtime_initializer_and_exports( post, exports, receiving) receiving = '' module = create_module(sending, receiving, invoke_funcs, metadata) write_output_file(out, post, module) module = None
def emscript(infile, outfile_js, memfile, temp_files, DEBUG): # Overview: # * Run wasm-emscripten-finalize to extract metadata and modify the binary # to use emscripten's wasm<->JS ABI # * Use the metadata to generate the JS glue that goes with the wasm metadata = finalize_wasm(infile, memfile, DEBUG) update_settings_glue(metadata, DEBUG) if not outfile_js: logger.debug('emscript: skipping js compiler glue') return if DEBUG: logger.debug('emscript: js compiler glue') if DEBUG: t = time.time() glue, forwarded_data = compile_settings(temp_files) if DEBUG: logger.debug(' emscript: glue took %s seconds' % (time.time() - t)) t = time.time() forwarded_json = json.loads(forwarded_data) # For the wasm backend the implementedFunctions from compiler.js should # always be empty. This only gets populated for __asm function when using # the JS backend. assert not forwarded_json['Functions']['implementedFunctions'] pre, post = glue.split('// EMSCRIPTEN_END_FUNCS') # memory and global initializers # In minimal runtime, global initializers are run after the Wasm Module instantiation has finished. if not shared.Settings.MINIMAL_RUNTIME: global_initializers = ', '.join('{ func: function() { %s() } }' % i for i in metadata['initializers']) # In regular runtime, global initializers are recorded in an __ATINIT__ array. global_initializers = '__ATINIT__.push(%s);' % global_initializers if shared.Settings.USE_PTHREADS: global_initializers = 'if (!ENVIRONMENT_IS_PTHREAD) ' + global_initializers pre += '\n' + global_initializers + '\n' if shared.Settings.RELOCATABLE: static_bump = align_memory(webassembly.parse_dylink_section(infile)[0]) memory = Memory(static_bump) logger.debug('stack_base: %d, stack_max: %d, dynamic_base: %d', memory.stack_base, memory.stack_max, memory.dynamic_base) pre = apply_memory(pre, memory) post = apply_memory(post, memory) pre = apply_static_code_hooks( pre) # In regular runtime, atinits etc. exist in the preamble part post = apply_static_code_hooks( post) # In MINIMAL_RUNTIME, atinit exists in the postamble part # merge forwarded data shared.Settings.EXPORTED_FUNCTIONS = forwarded_json['EXPORTED_FUNCTIONS'] exports = metadata['exports'] # Store exports for Closure compiler to be able to track these as globals in # -s DECLARE_ASM_MODULE_EXPORTS=0 builds. shared.Settings.MODULE_EXPORTS = [(asmjs_mangle(f), f) for f in exports] if shared.Settings.ASYNCIFY: exports += [ 'asyncify_start_unwind', 'asyncify_stop_unwind', 'asyncify_start_rewind', 'asyncify_stop_rewind' ] all_exports = exports + list(metadata['namedGlobals'].keys()) all_exports = set([asmjs_mangle(e) for e in all_exports]) report_missing_symbols(all_exports, pre) asm_consts = create_asm_consts(metadata) em_js_funcs = create_em_js(forwarded_json, metadata) asm_const_pairs = ['%s: %s' % (key, value) for key, value in asm_consts] asm_const_map = 'var ASM_CONSTS = {\n ' + ', \n '.join( asm_const_pairs) + '\n};\n' pre = pre.replace('// === Body ===', ('// === Body ===\n\n' + asm_const_map + '\n'.join(em_js_funcs) + '\n')) with open(outfile_js, 'w') as out: out.write(pre) pre = None invoke_funcs = metadata['invokeFuncs'] sending = create_sending(invoke_funcs, metadata) receiving = create_receiving(exports, metadata['initializers']) if shared.Settings.MINIMAL_RUNTIME: post = compute_minimal_runtime_initializer_and_exports( post, metadata['initializers'], exports, receiving) receiving = '' module = create_module(sending, receiving, invoke_funcs, metadata) write_output_file(out, post, module) module = None
def finalize_wasm(infile, outfile, memfile): building.save_intermediate(infile, 'base.wasm') args = [] # if we don't need to modify the wasm, don't tell finalize to emit a wasm file modify_wasm = False if settings.WASM2JS: # wasm2js requires full legalization (and will do extra wasm binary # later processing later anyhow) modify_wasm = True if settings.USE_PTHREADS and settings.RELOCATABLE: # HACK: When settings.USE_PTHREADS and settings.RELOCATABLE are set finalize needs to scan # more than just the start function for memory.init instructions. This means it can't run # with setSkipFunctionBodies() enabled. Currently the only way to force this is to set an # output file. # TODO(sbc): Find a better way to do this. modify_wasm = True if settings.GENERATE_SOURCE_MAP: building.emit_wasm_source_map(infile, infile + '.map', outfile) building.save_intermediate(infile + '.map', 'base_wasm.map') args += [ '--output-source-map-url=' + settings.SOURCE_MAP_BASE + os.path.basename(outfile) + '.map' ] modify_wasm = True if settings.DEBUG_LEVEL >= 2 or settings.ASYNCIFY_ADD or settings.ASYNCIFY_ADVISE or settings.ASYNCIFY_ONLY or settings.ASYNCIFY_REMOVE or settings.EMIT_SYMBOL_MAP or settings.EMIT_NAME_SECTION: args.append('-g') if settings.WASM_BIGINT: args.append('--bigint') if settings.DYNCALLS: # we need to add all dyncalls to the wasm modify_wasm = True else: if settings.WASM_BIGINT: args.append('--no-dyncalls') else: args.append('--dyncalls-i64') # we need to add some dyncalls to the wasm modify_wasm = True if settings.LEGALIZE_JS_FFI: # When we dynamically link our JS loader adds functions from wasm modules to # the table. It must add the original versions of them, not legalized ones, # so that indirect calls have the right type, so export those. if settings.RELOCATABLE: args.append('--pass-arg=legalize-js-interface-export-originals') modify_wasm = True else: args.append('--no-legalize-javascript-ffi') if memfile: args.append(f'--separate-data-segments={memfile}') args.append(f'--global-base={settings.GLOBAL_BASE}') modify_wasm = True if settings.SIDE_MODULE: args.append('--side-module') if settings.STACK_OVERFLOW_CHECK >= 2: args.append('--check-stack-overflow') modify_wasm = True if settings.STANDALONE_WASM: args.append('--standalone-wasm') if settings.DEBUG_LEVEL >= 3: args.append('--dwarf') # Currently we have two different ways to extract the metadata from the # wasm binary: # 1. via wasm-emscripten-finalize (binaryen) # 2. via local python code # We also have a 'compare' mode that runs both extraction methods and # checks that they produce identical results. read_metadata = os.environ.get('EMCC_READ_METADATA', 'python') if read_metadata == 'binaryen': metadata = get_metadata_binaryen(infile, outfile, modify_wasm, args) elif read_metadata == 'python': metadata = get_metadata_python(infile, outfile, modify_wasm, args) elif read_metadata == 'compare': shutil.copy2(infile, infile + '.bak') if settings.GENERATE_SOURCE_MAP: shutil.copy2(infile + '.map', infile + '.map.bak') pymetadata = get_metadata_python(infile, outfile, modify_wasm, args) shutil.move(infile + '.bak', infile) if settings.GENERATE_SOURCE_MAP: shutil.move(infile + '.map.bak', infile + '.map') metadata = get_metadata_binaryen(infile, outfile, modify_wasm, args) compare_metadata(metadata, pymetadata) else: assert False if modify_wasm: building.save_intermediate(infile, 'post_finalize.wasm') elif infile != outfile: shutil.copy(infile, outfile) if settings.GENERATE_SOURCE_MAP: building.save_intermediate(infile + '.map', 'post_finalize.map') if memfile: # we have a separate .mem file. binaryen did not strip any trailing zeros, # because it's an ABI question as to whether it is valid to do so or not. # we can do so here, since we make sure to zero out that memory (even in # the dynamic linking case, our loader zeros it out) remove_trailing_zeros(memfile) expected_exports = set(settings.EXPORTED_FUNCTIONS) expected_exports.update(asmjs_mangle(s) for s in settings.REQUIRED_EXPORTS) # Calculate the subset of exports that were explicitly marked with llvm.used. # These are any exports that were not requested on the command line and are # not known auto-generated system functions. unexpected_exports = [ e for e in metadata['exports'] if treat_as_user_function(e) ] unexpected_exports = [asmjs_mangle(e) for e in unexpected_exports] unexpected_exports = [ e for e in unexpected_exports if e not in expected_exports ] building.user_requested_exports.update(unexpected_exports) settings.EXPORTED_FUNCTIONS.extend(unexpected_exports) return metadata