def run(self, args): return jsrun.run_js(self.filename, engine=self.engine, args=args, stderr=PIPE, full_output=True, assert_returncode=None)
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files)): ll = ''.join(funcs) + '\n' + meta funcs_file = temp_files.get('.func_%d.ll' % i).name open(funcs_file, 'w').write(ll) out = jsrun.run_js( compiler, engine=compiler_engine, args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE) tempfiles.try_delete(funcs_file) return out
def execute_js(engine): print('(run in %s)' % engine) try: js = jsrun.run_js(filename + '.js', engine=shutil.NODE_JS, check_timeout=True, assert_returncode=None) except CalledProcessError: print('failed to run in primary') return False js = js.split('\n')[ 0] + '\n' # remove any extra printed stuff (node workarounds) return correct1 == js or correct2 == js
def test_d8_path(self): """ Test that running JS commands works for node, d8, and jsc and is not path dependent """ # Fake some JS engines restore_and_set_up() sample_script = path_from_root('tests', 'print_args.js') # Note that the path contains 'd8'. test_path = path_from_root('tests', 'fake', 'abcd8765') if not os.path.exists(test_path): os.makedirs(test_path) with env_modify({'EM_IGNORE_SANITY': '1'}): jsengines = [('d8', V8_ENGINE), ('d8_g', V8_ENGINE), ('js', SPIDERMONKEY_ENGINE), ('node', NODE_JS), ('nodejs', NODE_JS)] for filename, engine in jsengines: if type(engine) is list: engine = engine[0] if engine == '': print('WARNING: Not testing engine %s, not configured.' % (filename)) continue print(filename, engine) test_engine_path = os.path.join(test_path, filename) f = open(test_engine_path, 'w') f.write('#!/bin/sh\n') f.write('%s $@\n' % (engine)) f.close() os.chmod(test_engine_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) try: out = jsrun.run_js(sample_script, engine=test_engine_path, args=['--foo'], full_output=True, assert_returncode=0, skip_check=True) except Exception as e: if 'd8' in filename: assert False, 'Your d8 version does not correctly parse command-line arguments, please upgrade or delete from ~/.emscripten config file: %s' % ( e) else: assert False, 'Error running script command: %s' % (e) self.assertEqual('0: --foo', out.strip())
def process_funcs((i, funcs_file, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, DEBUG)): try: #print >> sys.stderr, 'running', str([settings_file, funcs_file, 'funcs', forwarded_file] + libraries).replace("'/", "'") # can use this in src/compiler_funcs.html arguments, # # just copy temp dir to under this one out = jsrun.run_js( compiler, engine=compiler_engine, args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) except KeyboardInterrupt: # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt raise Exception() if DEBUG: logging.debug('.') return out
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)): funcs_file = temp_files.get('.func_%d.ll' % i).name f = open(funcs_file, 'w') f.write(funcs) funcs = None f.write('\n') f.write(meta) f.close() out = jsrun.run_js( compiler, engine=compiler_engine, args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) tempfiles.try_delete(funcs_file) if DEBUG: print >> sys.stderr, '.' return out
def test_d8_path(self): """ Test that running JS commands works for node, d8, and jsc and is not path dependent """ # Fake some JS engines restore_and_set_up() sample_script = path_from_root('tests', 'print_args.js') # Note that the path contains 'd8'. test_path = path_from_root('tests', 'fake', 'abcd8765') if not os.path.exists(test_path): os.makedirs(test_path) with env_modify({'EM_IGNORE_SANITY': '1'}): jsengines = [('d8', V8_ENGINE), ('d8_g', V8_ENGINE), ('js', SPIDERMONKEY_ENGINE), ('node', NODE_JS), ('nodejs', NODE_JS)] for filename, engine in jsengines: if type(engine) is list: engine = engine[0] if engine == '': print('WARNING: Not testing engine %s, not configured.' % (filename)) continue print(filename, engine) test_engine_path = os.path.join(test_path, filename) f = open(test_engine_path, 'w') f.write('#!/bin/sh\n') f.write('%s $@\n' % (engine)) f.close() os.chmod(test_engine_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC) try: out = jsrun.run_js(sample_script, engine=test_engine_path, args=['--foo'], full_output=True, assert_returncode=0, skip_check=True) except Exception as e: if 'd8' in filename: assert False, 'Your d8 version does not correctly parse command-line arguments, please upgrade or delete from ~/.emscripten config file: %s' % (e) else: assert False, 'Error running script command: %s' % (e) self.assertEqual('0: --foo', out.strip())
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)): try: funcs_file = temp_files.get('.func_%d.ll' % i).name f = open(funcs_file, 'w') f.write(funcs) funcs = None f.write('\n') f.write(meta) f.close() out = jsrun.run_js( compiler, engine=compiler_engine, args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) except KeyboardInterrupt: # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt raise Exception() finally: tempfiles.try_delete(funcs_file) if DEBUG: print >> sys.stderr, '.' return out
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() curr_func = None for line in ll_lines: if in_func: curr_func.append(line) if line.startswith('}'): in_func = False funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier # pre needs to know about all implemented functions, even for non-pre func pre.append(curr_func[0]) pre.append(line) curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input); dfp.write("\n\n========================== settings_text\n\n"); dfp.write(settings_text); dfp.write("\n\n========================== libraries\n\n"); dfp.write("\n".join(libraries)) dfp.close() print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() forwarded_json = json.loads(forwarded_data) indexed_functions = set() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if len(chunks) > 0: if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) commands = [ (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks) ] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys(): if key in all_exported_functions or (export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 for indexed in indexed_functions: #print >> sys.stderr, 'indaxx', indexed, i forwarded_json['Functions']['indexedFunctions'][indexed] = i # make sure not to modify this python object later - we use it in indexize i += 2 forwarded_json['Functions']['nextIndex'] = i def split_32(x): x = int(x) return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): # In the global initial allocation, we need to split up into Uint8 format ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) #if DEBUG: outfile.write('// pre\n') outfile.write(blockaddrsize(indexize(pre))) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] # Find function table calls without function tables generated for them for funcs_js_item in funcs_js: for use in set(re.findall(r'{{{ FTM_[\w\d_$]+ }}}', funcs_js_item)): sig = use[8:len(use)-4] if sig not in last_forwarded_json['Functions']['tables']: if DEBUG: print >> sys.stderr, 'add empty function table', sig last_forwarded_json['Functions']['tables'][sig] = 'var FUNCTION_TABLE_' + sig + ' = [0,0];\n' def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig)-1)]) coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) return ('function %s(%s) { %s abort(%d); %s }' % (bad, params, coercions, i, ret), raw.replace('[0,', '[' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0]', ',' + bad + ']').replace(',0]', ',' + bad + ']').replace(',0\n', ',' + bad + '\n')) infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) asm_setup = '' maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat'] + [m.replace('.', '_') for m in math_envs] if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed'] or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) args = ','.join(['a' + str(i) for i in range(1, len(sig))]) args = 'index' + (',' if args else '') + args asm_setup += ''' function invoke_%s(%s) { try { %sModule.dynCall_%s(%s); } catch(e) { asm.setThrew(1); } } ''' % (sig, args, 'return ' if sig[0] != 'v' else '', sig, args) basic_funcs.append('invoke_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1]; asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # sent data the_global = '{ ' + ', '.join([math_fix(s) + ': ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join([math_fix(s) + ': ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' # received if not simple: receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm.' + s for s in exported_implemented_functions + function_tables]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) funcs_js = [''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { 'use asm'; var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup,) + '\n' + asm_global_vars + ''' var __THREW__ = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; STACKTOP = ((STACKTOP + 3)>>2)<<2; return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw) { threw = threw|0; __THREW__ = threw; } ''' + ''.join([''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10)])] + funcs_js + [''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) }; Runtime.stackSave = function() { return asm.stackSave() }; Runtime.stackRestore = function(top) { asm.stackRestore(top) }; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] # Set function table masks def function_table_maskize(js): masks = {} default = None for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): masks[sig] = str(table.count(',')) default = sig def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % ( time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later func_idents = [] meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() for line in ll_lines: if in_func: funcs[-1][1].append(line) if line.startswith('}'): in_func = False funcs[-1] = (funcs[-1][0], ''.join(funcs[-1][1])) pre.append( line ) # pre needs it to, so we know about all implemented functions else: if line.startswith(';'): continue if line.startswith('define '): in_func = True funcs.append( (line, [line])) # use the entire line as the identifier pre.append( line ) # pre needs it to, so we know about all implemented functions elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append( line ) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024 * 1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % ( time.time() - t) #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >> sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input) dfp.write("\n\n========================== settings_text\n\n") dfp.write(settings_text) dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() print >> sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % ( time.time() - t) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3 * len(meta) + len( forwarded_data ) / 3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() forwarded_json = json.loads(forwarded_data) indexed_functions = set() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey( keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len( cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len( chunks) == 1, 'no point in splitting up without multiple cores' if len(chunks) > 0: if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % ( len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size / (1024 * 1024.), len(meta) / (1024 * 1024.), len(forwarded_data) / (1024 * 1024.), total_ll_size / (1024 * 1024.)) commands = [(i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks)] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len( chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len( output ) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[ 0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set( settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings[ 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json[ 'Types']['preciseI64MathUsed'] or curr_forwarded_json['Types'][ 'preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions'][ 'blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions'][ 'indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] for key in curr_forwarded_json['Functions'][ 'implementedFunctions'].iterkeys(): if key in all_exported_functions or ( export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions'][ 'unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 for indexed in indexed_functions: #print >> sys.stderr, 'indaxx', indexed, i forwarded_json['Functions']['indexedFunctions'][ indexed] = i # make sure not to modify this python object later - we use it in indexize i += 2 forwarded_json['Functions']['nextIndex'] = i indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), js) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): return re.sub( r'{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) #if DEBUG: outfile.write('// pre\n') outfile.write(blockaddrsize(indexize(pre))) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % ( time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) post, last_forwarded_data = out.split( '//FORWARDED_DATA:' ) # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] # Find function table calls without function tables generated for them for funcs_js_item in funcs_js: for use in set(re.findall(r'{{{ FTM_[\w\d_$]+ }}}', funcs_js_item)): sig = use[8:len(use) - 4] if sig not in last_forwarded_json['Functions']['tables']: if DEBUG: print >> sys.stderr, 'add empty function table', sig last_forwarded_json['Functions']['tables'][ sig] = 'var FUNCTION_TABLE_' + sig + ' = [0,0];\n' def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig) - 1)]) coercions = ';'.join([ 'p%d = %sp%d%s' % (p, '+' if sig[p + 1] != 'i' else '', p, '' if sig[p + 1] != 'i' else '|0') for p in range(len(sig) - 1) ]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) return ('function %s(%s) { %s abort(%d); %s }' % (bad, params, coercions, i, ret), raw.replace('[0,', '[' + bad + ',').replace( ',0,', ',' + bad + ',').replace(',0,', ',' + bad + ',').replace( ',0]', ',' + bad + ']').replace(',0]', ',' + bad + ']').replace( ',0\n', ',' + bad + '\n')) infos = [ make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems() ] function_tables_defs = '\n'.join( [info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join( [info[1] for info in infos]) asm_setup = '' maths = [ 'Math.' + func for func in [ 'floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul' ] ] fundamentals = [ 'Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array' ] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join( ['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) basic_funcs = [ 'abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat' ] + [m.replace('.', '_') for m in math_envs] if settings['SAFE_HEAP']: basic_funcs += [ 'SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR' ] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed']: basic_funcs += [ 'i64Math_' + op for op in ['add', 'subtract', 'multiply', 'divide', 'modulo'] ] asm_setup += ''' var i64Math_add = function(a, b, c, d) { i64Math.add(a, b, c, d) }; var i64Math_subtract = function(a, b, c, d) { i64Math.subtract(a, b, c, d) }; var i64Math_multiply = function(a, b, c, d) { i64Math.multiply(a, b, c, d) }; var i64Math_divide = function(a, b, c, d, e) { i64Math.divide(a, b, c, d, e) }; var i64Math_modulo = function(a, b, c, d, e) { i64Math.modulo(a, b, c, d, e) }; ''' asm_runtime_funcs = [ 'stackAlloc', 'stackSave', 'stackRestore', 'setThrew' ] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = [ 'dynCall_' + table for table in last_forwarded_json['Functions']['tables'] ] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join([ 'a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig)) ]) coerced_args = ','.join( [asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce( 'FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals'][ '_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map( lambda g: g['name'], filter( lambda g: settings['NAMED_GLOBALS'] or g.get( 'external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = [ '_' + x for x in forwarded_json['Functions']['libraryFunctions'].keys() ] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # sent data the_global = '{ ' + ', '.join( [math_fix(s) + ': ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join([ math_fix(s) + ': ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars ]) + ' }' # received if not simple: receiving = ';\n'.join([ 'var ' + s + ' = Module["' + s + '"] = asm.' + s for s in exported_implemented_functions + function_tables ]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map( len, funcs_js), len(asm_setup), len(asm_global_vars), len( asm_global_funcs), len(pre_tables), len( '\n'.join(function_tables_impls)), len( function_tables_defs.replace( '\n', '\n ')), len(exports), len( the_global), len(sending), len(receiving) funcs_js = [ ''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { 'use asm'; var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup, ) + '\n' + asm_global_vars + ''' var __THREW__ = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; STACKTOP = ((STACKTOP + 3)>>2)<<2; return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw) { threw = threw|0; __THREW__ = threw; } ''' + ''.join([ ''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10) ]) ] + funcs_js + [ ''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) }; Runtime.stackSave = function() { return asm.stackSave() }; Runtime.stackRestore = function(top) { asm.stackRestore(top) }; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving) ] # Set function table masks def function_table_maskize(js): masks = {} default = None for sig, table in last_forwarded_json['Functions'][ 'tables'].iteritems(): masks[sig] = str(table.count(',')) default = sig def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([ table for table in last_forwarded_json['Functions'] ['tables'].itervalues() ]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % ( time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() curr_func = None for line in ll_lines: if in_func: curr_func.append(line) if line.startswith('}'): in_func = False funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier # pre needs to know about all implemented functions, even for non-pre func pre.append(curr_func[0]) pre.append(line) curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) if len(funcs) == 0: print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)' #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input) dfp.write("\n\n========================== settings_text\n\n") dfp.write(settings_text) dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) indexed_functions = set() forwarded_json = json.loads(forwarded_data) for key in forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if len(chunks) > 0: if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) commands = [ (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks) ] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['hasInlineJS'] = forwarded_json['Types']['hasInlineJS'] or curr_forwarded_json['Types']['hasInlineJS'] forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] export_all = settings['EXPORT_ALL'] for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys(): if key in all_exported_functions or export_all or (export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value for key, value in curr_forwarded_json['Functions']['neededTables'].iteritems(): forwarded_json['Functions']['neededTables'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 # universal counter if settings['ASM_JS']: i += 2*settings['RESERVED_FUNCTION_POINTERS'] table_counters = {} # table-specific counters alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] sig = None for indexed in indexed_functions: if alias: sig = forwarded_json['Functions']['implementedFunctions'].get(indexed) or forwarded_json['Functions']['unimplementedFunctions'].get(indexed) assert sig, indexed if sig not in table_counters: table_counters[sig] = 2 + 2*settings['RESERVED_FUNCTION_POINTERS'] curr = table_counters[sig] table_counters[sig] += 2 else: curr = i i += 2 #print >> sys.stderr, 'function indexing', indexed, curr, sig forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): x = int(x) return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): # In the global initial allocation, we need to split up into Uint8 format ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) pre = blockaddrsize(indexize(pre)) if settings.get('ASM_JS'): # move postsets into the asm module class PostSets: js = '' def handle_post_sets(m): PostSets.js = m.group(0) return '\n' pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre) #if DEBUG: outfile.write('// pre\n') outfile.write(pre) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig)-1)]) coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) start = raw.index('[') end = raw.rindex(']') body = raw[start+1:end].split(',') for j in range(settings['RESERVED_FUNCTION_POINTERS']): body[2 + 2*j] = 'jsCall_%s_%s' % (sig, j) def fix_item(item): newline = '\n' in item return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') body = ','.join(map(fix_item, body)) return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start+1] + body + raw[end:]) infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) asm_setup = '' maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) if settings['TO_FLOAT32']: maths += ['Math.toFloat32'] basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat'] + [m.replace('.', '_') for m in math_envs] if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] if settings['ASSERTIONS']: basic_funcs += ['nullFunc'] asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n' basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed'] or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] if settings.get('DLOPEN_SUPPORT'): for sig in last_forwarded_json['Functions']['tables'].iterkeys(): basic_vars.append('F_BASE_%s' % sig) asm_setup += ' var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n' asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) for i in range(settings['RESERVED_FUNCTION_POINTERS']): jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce('jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0]) function_tables_impls.append(''' function jsCall_%s_%s(%s) { %s %s; } ''' % (sig, i, args, arg_coercions, jsret)) from tools import shared shared.Settings.copy(settings) asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' basic_funcs.append('invoke_%s' % sig) if settings.get('DLOPEN_SUPPORT'): asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n' basic_funcs.append('extCall_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exported_implemented_functions.append('runPostSets') exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): assert settings.get('TARGET_LE32'), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' for key, value in forwarded_json['Variables']['globals'].iteritems(): if value.get('linkable'): init = forwarded_json['Variables']['indexedGlobals'][key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' asm_global_vars += ' var %s=%s;\n' % (key, str(init)) # sent data the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' # received if not simple: receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) funcs_js = [''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { %s var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup, "'use asm';" if not forwarded_json['Types']['hasInlineJS'] and not settings['SIDE_MODULE'] else "'almost asm';") + '\n' + asm_global_vars + ''' var __THREW__ = 0; var threwValue = 0; var setjmpId = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; ''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw, value) { threw = threw|0; value = value|0; if ((__THREW__|0) == 0) { __THREW__ = threw; threwValue = value; } } function copyTempFloat(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; } function copyTempDouble(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0]; HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0]; HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0]; HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0]; } ''' + ''.join([''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10)])] + [PostSets.js + '\n'] + funcs_js + [''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] if not settings.get('SIDE_MODULE'): funcs_js.append(''' Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) }; Runtime.stackSave = function() { return asm['stackSave']() }; Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; ''') # Set function table masks masks = {} max_mask = 0 for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): mask = table.count(',') masks[sig] = str(mask) max_mask = max(mask, max_mask) def function_table_maskize(js, masks): def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) if settings.get('DLOPEN_SUPPORT'): funcs_js.append(''' asm.maxFunctionIndex = %(max_mask)d; DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module); Module.SYMBOL_TABLE = SYMBOL_TABLE; ''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) else: function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] # Create symbol table for self-dlopen if settings.get('DLOPEN_SUPPORT'): symbol_table = {} for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems(): if forwarded_json['Variables']['globals'][k]['named']: symbol_table[k] = str(v + forwarded_json['Runtime']['GLOBAL_BASE']) for raw in last_forwarded_json['Functions']['tables'].itervalues(): if raw == '': continue table = map(string.strip, raw[raw.find('[')+1:raw.find(']')].split(",")) for i in range(len(table)): value = table[i] if value != '0': if settings.get('SIDE_MODULE'): symbol_table[value] = 'FUNCTION_TABLE_OFFSET+' + str(i) else: symbol_table[value] = str(i) outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table).replace('"', '')) for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % ( time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() curr_func = None for line in ll_lines: if in_func: curr_func.append(line) if line.startswith('}'): in_func = False funcs.append((curr_func[0], ''.join(curr_func) )) # use the entire line as the identifier # pre needs to know about all implemented functions, even for non-pre func pre.append(curr_func[0]) pre.append(line) curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append( line ) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024 * 1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % ( time.time() - t) if len(funcs) == 0: print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)' #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >> sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input) dfp.write("\n\n========================== settings_text\n\n") dfp.write(settings_text) dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() print >> sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % ( time.time() - t) indexed_functions = set() forwarded_json = json.loads(forwarded_data) for key in forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3 * len(meta) + len( forwarded_data ) / 3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey( keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len( cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if len(chunks) > 0: if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len( chunks) == 1, 'no point in splitting up without multiple cores' if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % ( len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size / (1024 * 1024.), len(meta) / (1024 * 1024.), len(forwarded_data) / (1024 * 1024.), total_ll_size / (1024 * 1024.)) commands = [(i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks)] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len( chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len( output ) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[ 0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set( settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings[ 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json[ 'Types']['preciseI64MathUsed'] or curr_forwarded_json['Types'][ 'preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions'][ 'blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions'][ 'indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] export_all = settings['EXPORT_ALL'] for key in curr_forwarded_json['Functions'][ 'implementedFunctions'].iterkeys(): if key in all_exported_functions or export_all or ( export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions'][ 'unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value for key, value in curr_forwarded_json['Functions'][ 'neededTables'].iteritems(): forwarded_json['Functions']['neededTables'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 # universal counter if settings['ASM_JS']: i += 2 * settings['RESERVED_FUNCTION_POINTERS'] table_counters = {} # table-specific counters alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] sig = None for indexed in indexed_functions: if alias: sig = forwarded_json['Functions']['implementedFunctions'].get( indexed) or forwarded_json['Functions'][ 'unimplementedFunctions'].get(indexed) assert sig, indexed if sig not in table_counters: table_counters[ sig] = 2 + 2 * settings['RESERVED_FUNCTION_POINTERS'] curr = table_counters[sig] table_counters[sig] += 2 else: curr = i i += 2 #print >> sys.stderr, 'function indexing', indexed, curr, sig forwarded_json['Functions']['indexedFunctions'][ indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): x = int(x) return '%d,%d,%d,%d' % (x & 255, (x >> 8) & 255, (x >> 16) & 255, (x >> 24) & 255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): # In the global initial allocation, we need to split up into Uint8 format ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): ret = re.sub( r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) return re.sub( r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) pre = blockaddrsize(indexize(pre)) if settings.get('ASM_JS'): # move postsets into the asm module class PostSets: js = '' def handle_post_sets(m): PostSets.js = m.group(0) return '\n' pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre) #if DEBUG: outfile.write('// pre\n') outfile.write(pre) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % ( time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) post, last_forwarded_data = out.split( '//FORWARDED_DATA:' ) # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig) - 1)]) coercions = ';'.join([ 'p%d = %sp%d%s' % (p, '+' if sig[p + 1] != 'i' else '', p, '' if sig[p + 1] != 'i' else '|0') for p in range(len(sig) - 1) ]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) start = raw.index('[') end = raw.rindex(']') body = raw[start + 1:end].split(',') for j in range(settings['RESERVED_FUNCTION_POINTERS']): body[2 + 2 * j] = 'jsCall_%s_%s' % (sig, j) def fix_item(item): newline = '\n' in item return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') body = ','.join(map(fix_item, body)) return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start + 1] + body + raw[end:]) infos = [ make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems() ] function_tables_defs = '\n'.join( [info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join( [info[1] for info in infos]) asm_setup = '' maths = [ 'Math.' + func for func in [ 'floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul' ] ] fundamentals = [ 'Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array' ] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join( ['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) if settings['TO_FLOAT32']: maths += ['Math.toFloat32'] basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat' ] + [m.replace('.', '_') for m in math_envs] if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') if settings['SAFE_HEAP']: basic_funcs += [ 'SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR' ] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] if settings['ASSERTIONS']: basic_funcs += ['nullFunc'] asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n' basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed'] or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] asm_runtime_funcs = [ 'stackAlloc', 'stackSave', 'stackRestore', 'setThrew' ] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = [ 'dynCall_' + table for table in last_forwarded_json['Functions']['tables'] ] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join([ 'a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig)) ]) coerced_args = ','.join( [asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce( 'FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) for i in range(settings['RESERVED_FUNCTION_POINTERS']): jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce( 'jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0]) function_tables_impls.append(''' function jsCall_%s_%s(%s) { %s %s; } ''' % (sig, i, args, arg_coercions, jsret)) from tools import shared asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' basic_funcs.append('invoke_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exported_implemented_functions.append('runPostSets') exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals'][ '_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map( lambda g: g['name'], filter( lambda g: settings['NAMED_GLOBALS'] or g.get( 'external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = [ '_' + key for key, value in forwarded_json['Functions'] ['libraryFunctions'].iteritems() if value != 2 ] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): assert settings.get( 'TARGET_LE32' ), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' for key, value in forwarded_json['Variables']['globals'].iteritems( ): if value.get('linkable'): init = forwarded_json['Variables']['indexedGlobals'][ key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' asm_global_vars += ' var %s=%s;\n' % (key, str(init)) # sent data the_global = '{ ' + ', '.join( ['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join([ '"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars ]) + ' }' # received if not simple: receiving = ';\n'.join([ 'var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables ]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map( len, funcs_js), len(asm_setup), len(asm_global_vars), len( asm_global_funcs), len(pre_tables), len( '\n'.join(function_tables_impls)), len( function_tables_defs.replace( '\n', '\n ')), len(exports), len( the_global), len(sending), len(receiving) funcs_js = [ ''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { 'use asm'; var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup, ) + '\n' + asm_global_vars + ''' var __THREW__ = 0; var threwValue = 0; var setjmpId = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; ''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw, value) { threw = threw|0; value = value|0; if ((__THREW__|0) == 0) { __THREW__ = threw; threwValue = value; } } function copyTempFloat(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; } function copyTempDouble(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0]; HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0]; HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0]; HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0]; } ''' + ''.join([ ''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10) ]) ] + [PostSets.js + '\n'] + funcs_js + [ ''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) }; Runtime.stackSave = function() { return asm['stackSave']() }; Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving) ] # Set function table masks def function_table_maskize(js): masks = {} default = None for sig, table in last_forwarded_json['Functions'][ 'tables'].iteritems(): masks[sig] = str(table.count(',')) default = sig def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([ table for table in last_forwarded_json['Functions'] ['tables'].itervalues() ]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] # Create symbol table for self-dlopen if settings.get('DLOPEN_SUPPORT'): symbol_table = {} for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems(): if forwarded_json['Variables']['globals'][k]['named']: symbol_table[k] = v + forwarded_json['Runtime']['GLOBAL_BASE'] for raw in last_forwarded_json['Functions']['tables'].itervalues(): if raw == '': continue table = map(string.strip, raw[raw.find('[') + 1:raw.find(']')].split(",")) symbol_table.update( map(lambda x: (x[1], x[0]), filter(lambda x: x[1] != '0', enumerate(table)))) outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table)) for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % ( time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ assert settings['ASM_JS'], 'fastcomp is asm.js-only (mode 1 or 2)' success = False try: # Overview: # * Run LLVM backend to emit JS. JS includes function bodies, memory initializer, # and various metadata # * Run compiler.js on the metadata to emit the shell js code, pre/post-ambles, # JS library dependencies, etc. temp_js = temp_files.get('.4.js').name backend_compiler = os.path.join(shared.LLVM_ROOT, 'llc') backend_args = [backend_compiler, infile, '-march=js', '-filetype=asm', '-o', temp_js] if settings['PRECISE_F32']: backend_args += ['-emscripten-precise-f32'] if settings['USE_PTHREADS']: backend_args += ['-emscripten-enable-pthreads'] if settings['WARN_UNALIGNED']: backend_args += ['-emscripten-warn-unaligned'] if settings['RESERVED_FUNCTION_POINTERS'] > 0: backend_args += ['-emscripten-reserved-function-pointers=%d' % settings['RESERVED_FUNCTION_POINTERS']] if settings['ASSERTIONS'] > 0: backend_args += ['-emscripten-assertions=%d' % settings['ASSERTIONS']] if settings['ALIASING_FUNCTION_POINTERS'] == 0: backend_args += ['-emscripten-no-aliasing-function-pointers'] if settings['EMULATED_FUNCTION_POINTERS']: backend_args += ['-emscripten-emulated-function-pointers'] if settings['RELOCATABLE']: backend_args += ['-emscripten-relocatable'] backend_args += ['-emscripten-global-base=0'] elif settings['GLOBAL_BASE'] >= 0: backend_args += ['-emscripten-global-base=%d' % settings['GLOBAL_BASE']] backend_args += ['-O' + str(settings['OPT_LEVEL'])] if DEBUG: logging.debug('emscript: llvm backend: ' + ' '.join(backend_args)) t = time.time() shared.jsrun.timeout_run(subprocess.Popen(backend_args, stdout=subprocess.PIPE)) if DEBUG: logging.debug(' emscript: llvm backend took %s seconds' % (time.time() - t)) t = time.time() # Split up output backend_output = open(temp_js).read() #if DEBUG: print >> sys.stderr, backend_output start_funcs_marker = '// EMSCRIPTEN_START_FUNCTIONS' end_funcs_marker = '// EMSCRIPTEN_END_FUNCTIONS' metadata_split_marker = '// EMSCRIPTEN_METADATA' start_funcs = backend_output.index(start_funcs_marker) end_funcs = backend_output.rindex(end_funcs_marker) metadata_split = backend_output.rindex(metadata_split_marker) funcs = backend_output[start_funcs+len(start_funcs_marker):end_funcs] metadata_raw = backend_output[metadata_split+len(metadata_split_marker):] #if DEBUG: print >> sys.stderr, "METAraw", metadata_raw try: metadata = cjson.decode(metadata_raw) except Exception, e: logging.error('emscript: failure to parse metadata output from compiler backend. raw output is: \n' + metadata_raw[:10000]) raise e mem_init = backend_output[end_funcs+len(end_funcs_marker):metadata_split] #if DEBUG: print >> sys.stderr, "FUNCS", funcs #if DEBUG: print >> sys.stderr, "META", metadata #if DEBUG: print >> sys.stderr, "meminit", mem_init # if emulating pointer casts, force all tables to the size of the largest if settings['EMULATE_FUNCTION_POINTER_CASTS']: max_size = 0 for k, v in metadata['tables'].iteritems(): max_size = max(max_size, v.count(',')+1) for k, v in metadata['tables'].iteritems(): curr = v.count(',')+1 if curr < max_size: metadata['tables'][k] = v.replace(']', (',0'*(max_size - curr)) + ']') if settings['SIDE_MODULE']: for k in metadata['tables'].keys(): metadata['tables'][k] = metadata['tables'][k].replace('var FUNCTION_TABLE_', 'var SIDE_FUNCTION_TABLE_') # function table masks table_sizes = {} for k, v in metadata['tables'].iteritems(): table_sizes[k] = str(v.count(',')) # undercounts by one, but that is what we want #if settings['ASSERTIONS'] >= 2 and table_sizes[k] == 0: # print >> sys.stderr, 'warning: no function pointers with signature ' + k + ', but there is a call, which will abort if it occurs (this can result from undefined behavior, check for compiler warnings on your source files and consider -Werror)' funcs = re.sub(r"#FM_(\w+)#", lambda m: table_sizes[m.groups(0)[0]], funcs) # fix +float into float.0, if not running js opts if not settings['RUNNING_JS_OPTS']: def fix_dot_zero(m): num = m.group(3) # TODO: handle 0x floats? if num.find('.') < 0: e = num.find('e'); if e < 0: num += '.0' else: num = num[:e] + '.0' + num[e:] return m.group(1) + m.group(2) + num funcs = re.sub(r'([(=,+\-*/%<>:?] *)\+(-?)((0x)?[0-9a-f]*\.?[0-9]+([eE][-+]?[0-9]+)?)', lambda m: fix_dot_zero(m), funcs) # js compiler if DEBUG: logging.debug('emscript: js compiler glue') # Settings changes i64_funcs = ['i64Add', 'i64Subtract', '__muldi3', '__divdi3', '__udivdi3', '__remdi3', '__uremdi3'] for i64_func in i64_funcs: if i64_func in metadata['declares']: settings['PRECISE_I64_MATH'] = 2 break metadata['declares'] = filter(lambda i64_func: i64_func not in ['getHigh32', 'setHigh32', '__muldi3', '__divdi3', '__remdi3', '__udivdi3', '__uremdi3'], metadata['declares']) # FIXME: do these one by one as normal js lib funcs # Integrate info from backend if settings['SIDE_MODULE']: settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] = [] # we don't need any JS library contents in side modules settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] = list( set(settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] + map(shared.JS.to_nice_ident, metadata['declares'])).difference( map(lambda x: x[1:], metadata['implementedFunctions']) ) ) + map(lambda x: x[1:], metadata['externs']) if metadata['simd']: settings['SIMD'] = 1 if metadata['cantValidate'] and settings['ASM_JS'] != 2: logging.warning('disabling asm.js validation due to use of non-supported features: ' + metadata['cantValidate']) settings['ASM_JS'] = 2 # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Call js compiler if DEBUG: t = time.time() out = jsrun.run_js(path_from_root('src', 'compiler.js'), compiler_engine, [settings_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src'), error_limit=300) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' glue, forwarded_data = out.split('//FORWARDED_DATA:') if DEBUG: logging.debug(' emscript: glue took %s seconds' % (time.time() - t)) t = time.time() last_forwarded_json = forwarded_json = json.loads(forwarded_data) # merge in information from llvm backend last_forwarded_json['Functions']['tables'] = metadata['tables'] pre, post = glue.split('// EMSCRIPTEN_END_FUNCS') #print >> sys.stderr, 'glue:', pre, '\n\n||||||||||||||||\n\n', post, '...............' # memory and global initializers global_initializers = str(', '.join(map(lambda i: '{ func: function() { %s() } }' % i, metadata['initializers']))) if settings['SIMD'] == 1: pre = open(path_from_root(os.path.join('src', 'ecmascript_simd.js'))).read() + '\n\n' + pre staticbump = mem_init.count(',')+1 while staticbump % 16 != 0: staticbump += 1 pre = pre.replace('STATICTOP = STATIC_BASE + 0;', '''STATICTOP = STATIC_BASE + %d; /* global initializers */ %s __ATINIT__.push(%s); %s''' % (staticbump, 'if (!ENVIRONMENT_IS_PTHREAD)' if settings['USE_PTHREADS'] else '', global_initializers, mem_init)) # XXX wrong size calculation! if settings['SIDE_MODULE']: pre = pre.replace('Runtime.GLOBAL_BASE', 'gb').replace('{{{ STATIC_BUMP }}}', str(staticbump)) funcs_js = [funcs] parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] funcs_js.append(parts[1]) # merge forwarded data settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) if settings['EXPORT_FUNCTION_TABLES']: for table in last_forwarded_json['Functions']['tables'].values(): for func in table.split('[')[1].split(']')[0].split(','): if func[0] == '_': all_exported_functions.add(func) exported_implemented_functions = set(metadata['exports']) export_bindings = settings['EXPORT_BINDINGS'] export_all = settings['EXPORT_ALL'] all_implemented = metadata['implementedFunctions'] + forwarded_json['Functions']['implementedFunctions'].keys() # XXX perf? for key in all_implemented: if key in all_exported_functions or export_all or (export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) implemented_functions = set(metadata['implementedFunctions']) if settings['ASSERTIONS'] and settings.get('ORIGINAL_EXPORTED_FUNCTIONS'): original_exports = settings['ORIGINAL_EXPORTED_FUNCTIONS'] if original_exports[0] == '@': original_exports = json.loads(open(original_exports[1:]).read()) for requested in original_exports: if requested not in all_implemented and \ requested != '_malloc': # special-case malloc, EXPORTED by default for internal use, but we bake in a trivial allocator and warn at runtime if used in ASSERTIONS logging.warning('function requested to be exported, but not implemented: "%s"', requested) asm_consts = [0]*len(metadata['asmConsts']) for k, v in metadata['asmConsts'].iteritems(): const = v.encode('utf-8') if const[0] == '"' and const[-1] == '"': const = const[1:-1] const = '{ ' + const + ' }' i = 0 args = [] while ('$' + str(i)) in const: args.append('$' + str(i)) i += 1 const = 'function(' + ', '.join(args ) + ') ' + const asm_consts[int(k)] = const asm_const_funcs = [] for arity in metadata['asmConstArities']: forwarded_json['Functions']['libraryFunctions']['_emscripten_asm_const_%d' % arity] = 1 args = ['a%d' % i for i in range(arity)] all_args = ['code'] + args asm_const_funcs.append(r''' function _emscripten_asm_const_%d(%s) { return ASM_CONSTS[code](%s); }''' % (arity, ', '.join(all_args), ', '.join(args))) pre = pre.replace('// === Body ===', '// === Body ===\n' + '\nvar ASM_CONSTS = [' + ', '.join(asm_consts) + '];\n' + '\n'.join(asm_const_funcs) + '\n') #if DEBUG: outfile.write('// pre\n') outfile.write(pre) pre = None #if DEBUG: outfile.write('// funcs\n') # when emulating function pointer casts, we need to know what is the target of each pointer if settings['EMULATE_FUNCTION_POINTER_CASTS']: function_pointer_targets = {} for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): start = table.index('[') end = table.rindex(']') body = table[start+1:end].split(',') parsed = map(lambda x: x.strip(), body) for i in range(len(parsed)): if parsed[i] != '0': assert i not in function_pointer_targets function_pointer_targets[i] = [sig, str(parsed[i])] # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' if not settings['BOOTSTRAPPING_STRUCT_INFO'] and len(funcs_js) > 1: funcs_js[1] = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), funcs_js[1]) class Counter: i = 0 j = 0 if 'pre' in last_forwarded_json['Functions']['tables']: pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] else: pre_tables = '' def unfloat(s): return 'd' if s == 'f' else s # lower float to double for ffis if settings['ASSERTIONS'] >= 2: debug_tables = {} def make_params(sig): return ','.join(['p%d' % p for p in range(len(sig)-1)]) def make_coerced_params(sig): return ','.join([shared.JS.make_coercion('p%d', unfloat(sig[p+1]), settings) % p for p in range(len(sig)-1)]) def make_coercions(sig): return ';'.join(['p%d = %s' % (p, shared.JS.make_coercion('p%d' % p, sig[p+1], settings)) for p in range(len(sig)-1)]) + ';' def make_func(name, code, params, coercions): return 'function %s(%s) { %s %s }' % (name, params, coercions, code) in_table = set() def make_table(sig, raw): params = make_params(sig) coerced_params = make_coerced_params(sig) coercions = make_coercions(sig) def make_bad(target=None): i = Counter.i Counter.i += 1 if target is None: target = i name = 'b' + str(i) if not settings['ASSERTIONS']: code = 'abort(%s);' % target else: code = 'nullFunc_' + sig + '(%d);' % target if sig[0] != 'v': code += 'return %s' % shared.JS.make_initializer(sig[0], settings) + ';' return name, make_func(name, code, params, coercions) bad, bad_func = make_bad() # the default bad func if settings['ASSERTIONS'] <= 1: Counter.pre = [bad_func] else: Counter.pre = [] start = raw.index('[') end = raw.rindex(']') body = raw[start+1:end].split(',') if settings['EMULATED_FUNCTION_POINTERS']: def receive(item): if item == '0': return item else: if item in all_implemented: in_table.add(item) return "asm['" + item + "']" else: return item # this is not implemented; it would normally be wrapped, but with emulation, we just use it directly outside body = map(receive, body) for j in range(settings['RESERVED_FUNCTION_POINTERS']): curr = 'jsCall_%s_%s' % (sig, j) body[settings['FUNCTION_POINTER_ALIGNMENT'] * (1 + j)] = curr implemented_functions.add(curr) Counter.j = 0 def fix_item(item): j = Counter.j Counter.j += 1 newline = Counter.j % 30 == 29 if item == '0': if j > 0 and settings['EMULATE_FUNCTION_POINTER_CASTS'] and j in function_pointer_targets: # emulate all non-null pointer calls, if asked to proper_sig, proper_target = function_pointer_targets[j] if settings['EMULATED_FUNCTION_POINTERS']: if proper_target in all_implemented: proper_target = "asm['" + proper_target + "']" def make_emulated_param(i): if i >= len(sig): return shared.JS.make_initializer(proper_sig[i], settings) # extra param, just send a zero return shared.JS.make_coercion('p%d' % (i-1), proper_sig[i], settings, convert_from=sig[i]) proper_code = proper_target + '(' + ','.join(map(lambda i: make_emulated_param(i+1), range(len(proper_sig)-1))) + ')' if proper_sig[0] != 'v': # proper sig has a return, which the wrapper may or may not use proper_code = shared.JS.make_coercion(proper_code, proper_sig[0], settings) if proper_sig[0] != sig[0]: # first coercion ensured we call the target ok; this one ensures we return the right type in the wrapper proper_code = shared.JS.make_coercion(proper_code, sig[0], settings, convert_from=proper_sig[0]) if sig[0] != 'v': proper_code = 'return ' + proper_code else: # proper sig has no return, we may need a fake return if sig[0] != 'v': proper_code = 'return ' + shared.JS.make_initializer(sig[0], settings) name = 'fpemu_%s_%d' % (sig, j) wrapper = make_func(name, proper_code, params, coercions) Counter.pre.append(wrapper) return name if not newline else (name + '\n') if settings['ASSERTIONS'] <= 1: return bad if not newline else (bad + '\n') else: specific_bad, specific_bad_func = make_bad(j) Counter.pre.append(specific_bad_func) return specific_bad if not newline else (specific_bad + '\n') if item not in implemented_functions and not settings['EMULATED_FUNCTION_POINTERS']: # when emulating function pointers, we don't need wrappers # this is imported into asm, we must wrap it call_ident = item if call_ident in metadata['redirects']: call_ident = metadata['redirects'][call_ident] if not call_ident.startswith('_') and not call_ident.startswith('Math_'): call_ident = '_' + call_ident code = call_ident + '(' + coerced_params + ')' if sig[0] != 'v': # ffis cannot return float if sig[0] == 'f': code = '+' + code code = 'return ' + shared.JS.make_coercion(code, sig[0], settings) code += ';' Counter.pre.append(make_func(item + '__wrapper', code, params, coercions)) return item + '__wrapper' return item if not newline else (item + '\n') if settings['ASSERTIONS'] >= 2: debug_tables[sig] = body body = ','.join(map(fix_item, body)) return ('\n'.join(Counter.pre), ''.join([raw[:start+1], body, raw[end:]])) infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] Counter.pre = [] function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n' if not settings['EMULATED_FUNCTION_POINTERS']: function_tables_defs += '\n// EMSCRIPTEN_END_FUNCS\n' function_tables_defs += '\n'.join([info[1] for info in infos]) asm_setup = '' if settings['ASSERTIONS'] >= 2: for sig in last_forwarded_json['Functions']['tables']: asm_setup += '\nvar debug_table_' + sig + ' = ' + json.dumps(debug_tables[sig]) + ';' maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul', 'min', 'clz32']] simdfloattypes = ['float32x4'] simdinttypes = ['int32x4'] simdtypes = simdfloattypes + simdinttypes simdfuncs = ['check', 'add', 'sub', 'neg', 'mul', 'equal', 'lessThan', 'greaterThan', 'notEqual', 'lessThanOrEqual', 'greaterThanOrEqual', 'select', 'and', 'or', 'xor', 'not', 'splat', 'swizzle', 'shuffle', 'withX', 'withY', 'withZ', 'withW', 'load', 'store', 'loadX', 'storeX', 'loadXY', 'storeXY', 'loadXYZ', 'storeXYZ'] simdfloatfuncs = simdfuncs + ['div', 'min', 'max', 'minNum', 'maxNum', 'sqrt', 'abs', 'fromInt32x4', 'fromInt32x4Bits', 'reciprocalApproximation', 'reciprocalSqrtApproximation']; simdintfuncs = simdfuncs + ['fromFloat32x4', 'fromFloat32x4Bits', 'shiftRightArithmeticByScalar', 'shiftRightLogicalByScalar', 'shiftLeftByScalar']; fundamentals = ['Math'] if settings['USE_PTHREADS']: fundamentals += ['SharedInt8Array', 'SharedInt16Array', 'SharedInt32Array', 'SharedUint8Array', 'SharedUint16Array', 'SharedUint32Array', 'SharedFloat32Array', 'SharedFloat64Array', 'Atomics'] else: fundamentals += ['Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] fundamentals += ['NaN', 'Infinity'] if metadata['simd']: fundamentals += ['SIMD'] if settings['ALLOW_MEMORY_GROWTH']: fundamentals.append('byteLength') math_envs = [] provide_fround = settings['PRECISE_F32'] or settings['SIMD'] if provide_fround: maths += ['Math.fround'] def get_function_pointer_error(sig): if settings['ASSERTIONS'] <= 1: extra = ' Module["printErr"]("Build with ASSERTIONS=2 for more info.");' pointer = ' ' else: pointer = ' \'" + x + "\' ' extra = ' Module["printErr"]("This pointer might make sense in another type signature: ' # sort signatures, attempting to show most likely related ones first sigs = last_forwarded_json['Functions']['tables'].keys() def keyfunc(other): ret = 0 minlen = min(len(other), len(sig)) maxlen = min(len(other), len(sig)) if other.startswith(sig) or sig.startswith(other): ret -= 1000 # prioritize prefixes, could be dropped params ret -= 133*difflib.SequenceMatcher(a=other, b=sig).ratio() # prioritize on diff similarity ret += 15*abs(len(other) - len(sig))/float(maxlen) # deprioritize the bigger the length difference is for i in range(minlen): if other[i] == sig[i]: ret -= 5/float(maxlen) # prioritize on identically-placed params ret += 20*len(other) # deprioritize on length return ret sigs.sort(key=keyfunc) for other in sigs: if other != sig: extra += other + ': " + debug_table_' + other + '[x] + " ' extra += '"); ' return 'Module["printErr"]("Invalid function pointer' + pointer + 'called with signature \'' + sig + '\'. ' + \ 'Perhaps this is an invalid value (e.g. caused by calling a virtual method on a NULL pointer)? ' + \ 'Or calling a function with an incorrect type, which will fail? ' + \ '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \ '"); ' + extra basic_funcs = ['abort', 'assert'] + [m.replace('.', '_') for m in math_envs] if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_FT_MASK'] if settings['ASSERTIONS']: if settings['ASSERTIONS'] >= 2: import difflib for sig in last_forwarded_json['Functions']['tables'].iterkeys(): basic_funcs += ['nullFunc_' + sig] asm_setup += '\nfunction nullFunc_' + sig + '(x) { ' + get_function_pointer_error(sig) + 'abort(x) }\n' basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = [] if metadata.get('preciseI64MathUsed'): basic_vars += ['cttz_i8'] else: if forwarded_json['Functions']['libraryFunctions'].get('_llvm_cttz_i32'): basic_vars += ['cttz_i8'] if settings['RELOCATABLE']: basic_vars += ['gb', 'fb'] if not settings['SIDE_MODULE']: asm_setup += 'var gb = Runtime.GLOBAL_BASE, fb = 0;\n' asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace', 'setThrew'] if not settings['RELOCATABLE']: asm_runtime_funcs += ['setTempRet0', 'getTempRet0'] else: basic_funcs += ['setTempRet0', 'getTempRet0'] asm_setup += 'var setTempRet0 = Runtime.setTempRet0, getTempRet0 = Runtime.getTempRet0;\n' # See if we need ASYNCIFY functions # We might not need them even if ASYNCIFY is enabled need_asyncify = '_emscripten_alloc_async_context' in exported_implemented_functions if need_asyncify: basic_vars += ['___async', '___async_unwind', '___async_retval', '___async_cur_frame'] asm_runtime_funcs += ['setAsync'] if settings.get('EMTERPRETIFY'): asm_runtime_funcs += ['emterpret'] if settings.get('EMTERPRETIFY_ASYNC'): asm_runtime_funcs += ['setAsyncState', 'emtStackSave'] # function tables if not settings['EMULATED_FUNCTION_POINTERS']: function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] else: function_tables = [] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join(['a' + str(i) + '=' + shared.JS.make_coercion('a' + str(i), sig[i], settings) + ';' for i in range(1, len(sig))]) coerced_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0], settings) if not settings['EMULATED_FUNCTION_POINTERS']: function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) else: function_tables_impls.append(''' var dynCall_%s = ftCall_%s; ''' % (sig, sig)) ffi_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings, ffi_arg=True) for i in range(1, len(sig))]) for i in range(settings['RESERVED_FUNCTION_POINTERS']): jsret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('jsCall_%s(%d%s%s)' % (sig, i, ',' if ffi_args else '', ffi_args), sig[0], settings, ffi_result=True) function_tables_impls.append(''' function jsCall_%s_%s(%s) { %s %s; } ''' % (sig, i, args, arg_coercions, jsret)) shared.Settings.copy(settings) asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' basic_funcs.append('invoke_%s' % sig) if settings.get('RESERVED_FUNCTION_POINTERS'): asm_setup += '\n' + shared.JS.make_jscall(sig) + '\n' basic_funcs.append('jsCall_%s' % sig) if settings.get('EMULATED_FUNCTION_POINTERS'): args = ['a%d' % i for i in range(len(sig)-1)] full_args = ['x'] + args table_access = 'FUNCTION_TABLE_' + sig if settings['SIDE_MODULE']: table_access = 'parentModule["' + table_access + '"]' # side module tables were merged into the parent, we need to access the global one prelude = ''' if (x < 0 || x >= %s.length) { Module.printErr("Function table mask error (out of range)"); %s ; abort(x) }''' % (table_access, get_function_pointer_error(sig)) asm_setup += ''' function ftCall_%s(%s) {%s return %s[x](%s); } ''' % (sig, ', '.join(full_args), prelude, table_access, ', '.join(args)) basic_funcs.append('ftCall_%s' % sig) def quote(prop): if settings['CLOSURE_COMPILER'] == 2: return "'" + prop + "'" else: return prop def access_quote(prop): if settings['CLOSURE_COMPILER'] == 2: return "['" + prop + "']" else: return '.' + prop # calculate exports exported_implemented_functions = list(exported_implemented_functions) + metadata['initializers'] exported_implemented_functions.append('runPostSets') if settings['ALLOW_MEMORY_GROWTH']: exported_implemented_functions.append('_emscripten_replace_memory') all_exported = exported_implemented_functions + asm_runtime_funcs + function_tables exported_implemented_functions = list(set(exported_implemented_functions)) if settings['EMULATED_FUNCTION_POINTERS']: all_exported = list(set(all_exported).union(in_table)) exports = [] for export in all_exported: exports.append(quote(export) + ": " + export) exports = '{ ' + ', '.join(exports) + ' }' # calculate globals try: del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable except: pass if not settings['RELOCATABLE']: global_vars = metadata['externs'] else: global_vars = [] # linkable code accesses globals through function calls global_funcs = list(set([key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2]).difference(set(global_vars)).difference(implemented_functions)) if settings['RELOCATABLE']: global_funcs += ['g$' + extern for extern in metadata['externs']] side = 'parent' if settings['SIDE_MODULE'] else '' def check(extern): if settings['ASSERTIONS']: return 'assert(' + side + 'Module["' + extern + '"]);' return '' for extern in metadata['externs']: asm_setup += 'var g$' + extern + ' = function() { ' + check(extern) + ' return ' + side + 'Module["' + extern + '"] };\n' def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global' + access_quote(g) + ';\n' for g in maths]); asm_global_funcs += ''.join([' var ' + g + '=env' + access_quote(math_fix(g)) + ';\n' for g in basic_funcs + global_funcs]) if metadata['simd']: asm_global_funcs += ''.join([' var SIMD_' + ty + '=global' + access_quote('SIMD') + access_quote(ty) + ';\n' for ty in simdtypes]) asm_global_funcs += ''.join([' var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdinttypes for g in simdintfuncs]) asm_global_funcs += ''.join([' var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdfloattypes for g in simdfloatfuncs]) if settings['USE_PTHREADS']: # asm_global_funcs += ''.join([' var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'exchange', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor', 'fence']]) # TODO: Once bug https://bugzilla.mozilla.org/show_bug.cgi?id=1141986 is implemented, replace the following line with the above one! asm_global_funcs += ''.join([' var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor', 'fence']]) asm_global_vars = ''.join([' var ' + g + '=env' + access_quote(g) + '|0;\n' for g in basic_vars + global_vars]) # sent data the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' # received receiving = '' if settings['ASSERTIONS']: # assert on the runtime being in a valid state when calling into compiled code. The only exceptions are # some support code receiving = '\n'.join(['var real_' + s + ' = asm["' + s + '"]; asm["' + s + '''"] = function() { assert(runtimeInitialized, 'you need to wait for the runtime to be ready (e.g. wait for main() to be called)'); assert(!runtimeExited, 'the runtime was exited (use NO_EXIT_RUNTIME to keep it alive after main() exits)'); return real_''' + s + '''.apply(null, arguments); }; ''' for s in exported_implemented_functions if s not in ['_memcpy', '_memset', 'runPostSets', '_emscripten_replace_memory']]) if not settings['SWAPPABLE_ASM_MODULE']: receiving += ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables]) else: receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + s + ' = Module["' + s + '"] = function() { return Module["asm"]["' + s + '"].apply(null, arguments) }' for s in exported_implemented_functions + function_tables]) receiving += ';\n' if settings['EXPORT_FUNCTION_TABLES']: for table in last_forwarded_json['Functions']['tables'].values(): tableName = table.split()[1] table = table.replace('var ' + tableName, 'var ' + tableName + ' = Module["' + tableName + '"]') receiving += table + '\n' # finalize if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving)])) if not settings.get('EMULATED_FUNCTION_POINTERS'): final_function_tables = '\n'.join(function_tables_impls) + '\n' + function_tables_defs else: asm_setup += '\n' + '\n'.join(function_tables_impls) + '\n' receiving += '\n' + function_tables_defs + '\n' + ''.join(['Module["dynCall_%s"] = dynCall_%s\n' % (sig, sig) for sig in last_forwarded_json['Functions']['tables']]) for sig in last_forwarded_json['Functions']['tables'].keys(): name = 'FUNCTION_TABLE_' + sig fullname = name if not settings['SIDE_MODULE'] else ('SIDE_' + name) receiving += 'Module["' + name + '"] = ' + fullname + ';\n' final_function_tables = '\n// EMSCRIPTEN_END_FUNCS\n' if settings['RELOCATABLE']: receiving += ''' var NAMED_GLOBALS = { %s }; for (var named in NAMED_GLOBALS) { Module['_' + named] = gb + NAMED_GLOBALS[named]; } Module['NAMED_GLOBALS'] = NAMED_GLOBALS; ''' % ', '.join('"' + k + '": ' + str(v) for k, v in metadata['namedGlobals'].iteritems()) receiving += ''.join(["Module['%s'] = Module['%s']\n" % (k, v) for k, v in metadata['aliases'].iteritems()]) funcs_js = [''' %s Module%s = %s; Module%s = %s; // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { %s %s ''' % (asm_setup, access_quote('asmGlobalArg'), the_global, access_quote('asmLibraryArg'), sending, "'use asm';" if not metadata.get('hasInlineJS') and settings['ASM_JS'] == 1 else "'almost asm';", ''' var HEAP8 = new global%s(buffer); var HEAP16 = new global%s(buffer); var HEAP32 = new global%s(buffer); var HEAPU8 = new global%s(buffer); var HEAPU16 = new global%s(buffer); var HEAPU32 = new global%s(buffer); var HEAPF32 = new global%s(buffer); var HEAPF64 = new global%s(buffer); ''' % (access_quote('SharedInt8Array' if settings['USE_PTHREADS'] else 'Int8Array'), access_quote('SharedInt16Array' if settings['USE_PTHREADS'] else 'Int16Array'), access_quote('SharedInt32Array' if settings['USE_PTHREADS'] else 'Int32Array'), access_quote('SharedUint8Array' if settings['USE_PTHREADS'] else 'Uint8Array'), access_quote('SharedUint16Array' if settings['USE_PTHREADS'] else 'Uint16Array'), access_quote('SharedUint32Array' if settings['USE_PTHREADS'] else 'Uint32Array'), access_quote('SharedFloat32Array' if settings['USE_PTHREADS'] else 'Float32Array'), access_quote('SharedFloat64Array' if settings['USE_PTHREADS'] else 'Float64Array')) if not settings['ALLOW_MEMORY_GROWTH'] else ''' var Int8View = global%s; var Int16View = global%s; var Int32View = global%s; var Uint8View = global%s; var Uint16View = global%s; var Uint32View = global%s; var Float32View = global%s; var Float64View = global%s; var HEAP8 = new Int8View(buffer); var HEAP16 = new Int16View(buffer); var HEAP32 = new Int32View(buffer); var HEAPU8 = new Uint8View(buffer); var HEAPU16 = new Uint16View(buffer); var HEAPU32 = new Uint32View(buffer); var HEAPF32 = new Float32View(buffer); var HEAPF64 = new Float64View(buffer); var byteLength = global.byteLength; ''' % (access_quote('Int8Array'), access_quote('Int16Array'), access_quote('Int32Array'), access_quote('Uint8Array'), access_quote('Uint16Array'), access_quote('Uint32Array'), access_quote('Float32Array'), access_quote('Float64Array'))) + '\n' + asm_global_vars + (''' var __THREW__ = 0; var threwValue = 0; var setjmpId = 0; var undef = 0; var nan = global%s, inf = global%s; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' % (access_quote('NaN'), access_quote('Infinity'))) + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs] + \ [' var tempFloat = %s;\n' % ('Math_fround(0)' if provide_fround else '0.0')] + \ [' var asyncState = 0;\n' if settings.get('EMTERPRETIFY_ASYNC') else ''] + \ ([' const f0 = Math_fround(0);\n'] if provide_fround else []) + \ ['' if not settings['ALLOW_MEMORY_GROWTH'] else ''' function _emscripten_replace_memory(newBuffer) { if ((byteLength(newBuffer) & 0xffffff || byteLength(newBuffer) <= 0xffffff) || byteLength(newBuffer) > 0x80000000) return false; HEAP8 = new Int8View(newBuffer); HEAP16 = new Int16View(newBuffer); HEAP32 = new Int32View(newBuffer); HEAPU8 = new Uint8View(newBuffer); HEAPU16 = new Uint16View(newBuffer); HEAPU32 = new Uint32View(newBuffer); HEAPF32 = new Float32View(newBuffer); HEAPF64 = new Float64View(newBuffer); buffer = newBuffer; return true; } '''] + [''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; STACKTOP = (STACKTOP + 15)&-16; ''' + ('if ((STACKTOP|0) >= (STACK_MAX|0)) abort();\n' if settings['ASSERTIONS'] else '') + ''' return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function establishStackSpace(stackBase, stackMax) { stackBase = stackBase|0; stackMax = stackMax|0; STACKTOP = stackBase; STACK_MAX = stackMax; } ''' + (''' function setAsync() { ___async = 1; }''' if need_asyncify else '') + (''' function emterpret(pc) { // this will be replaced when the emterpreter code is generated; adding it here allows validation until then pc = pc | 0; assert(0); } ''' if settings['EMTERPRETIFY'] else '') + (''' function setAsyncState(x) { x = x | 0; asyncState = x; } function emtStackSave() { return EMTSTACKTOP|0; } ''' if settings['EMTERPRETIFY_ASYNC'] else '') + ''' function setThrew(threw, value) { threw = threw|0; value = value|0; if ((__THREW__|0) == 0) { __THREW__ = threw; threwValue = value; } } function copyTempFloat(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; } function copyTempDouble(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0]; HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0]; HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0]; HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0]; HEAP8[tempDoublePtr+4>>0] = HEAP8[ptr+4>>0]; HEAP8[tempDoublePtr+5>>0] = HEAP8[ptr+5>>0]; HEAP8[tempDoublePtr+6>>0] = HEAP8[ptr+6>>0]; HEAP8[tempDoublePtr+7>>0] = HEAP8[ptr+7>>0]; } '''] + [''' function setTempRet0(value) { value = value|0; tempRet0 = value; } function getTempRet0() { return tempRet0|0; } ''' if not settings['RELOCATABLE'] else ''] + funcs_js + [''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; ''' % (pre_tables + final_function_tables, exports, 'Module' + access_quote('asmGlobalArg'), 'Module' + access_quote('asmLibraryArg'), receiving)] if not settings.get('SIDE_MODULE'): funcs_js.append(''' Runtime.stackAlloc = asm['stackAlloc']; Runtime.stackSave = asm['stackSave']; Runtime.stackRestore = asm['stackRestore']; Runtime.establishStackSpace = asm['establishStackSpace']; ''') if not settings['RELOCATABLE']: funcs_js.append(''' Runtime.setTempRet0 = asm['setTempRet0']; Runtime.getTempRet0 = asm['getTempRet0']; ''') # Set function table masks masks = {} max_mask = 0 for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): mask = table.count(',') masks[sig] = str(mask) max_mask = max(mask, max_mask) def function_table_maskize(js, masks): def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) if settings['SIDE_MODULE']: funcs_js.append(''' Runtime.registerFunctions(%(sigs)s, Module); ''' % { 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) for i in range(len(funcs_js)): # do this loop carefully to save memory if WINDOWS: funcs_js[i] = funcs_js[i].replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n! outfile.write(funcs_js[i]) funcs_js = None if WINDOWS: post = post.replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n! outfile.write(post) outfile.close() if DEBUG: logging.debug(' emscript: final python processing took %s seconds' % (time.time() - t)) success = True
def main(): data_files = [] export_name = 'Module' leading = '' has_preloaded = False plugins = [] jsoutput = None from_emcc = False force = True # If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally # cache VFS XHR so that subsequent page loads can read the data from the # offline cache instead. use_preload_cache = False indexeddb_name = 'EM_PRELOAD_CACHE' # If set to True, the blob received from XHR is moved to the Emscripten HEAP, # optimizing for mmap() performance (if ALLOW_MEMORY_GROWTH=0). # If set to False, the XHR blob is kept intact, and fread()s etc. are performed # directly to that data. This optimizes for minimal memory usage and fread() # performance. heap_copy = True # If set to True, the package metadata is stored separately from js-output # file which makes js-output file immutable to the package content changes. # If set to False, the package metadata is stored inside the js-output file # which makes js-output file to mutate on each invocation of this packager tool. separate_metadata = False lz4 = False use_preload_plugins = False for arg in sys.argv[2:]: if arg == '--preload': has_preloaded = True leading = 'preload' elif arg == '--embed': leading = 'embed' elif arg == '--exclude': leading = 'exclude' elif arg == '--no-force': force = False leading = '' elif arg == '--use-preload-cache': use_preload_cache = True leading = '' elif arg.startswith('--indexedDB-name'): indexeddb_name = arg.split('=', 1)[1] if '=' in arg else None leading = '' elif arg == '--no-heap-copy': heap_copy = False leading = '' elif arg == '--separate-metadata': separate_metadata = True leading = '' elif arg == '--lz4': lz4 = True leading = '' elif arg == '--use-preload-plugins': use_preload_plugins = True leading = '' elif arg.startswith('--js-output'): jsoutput = arg.split('=', 1)[1] if '=' in arg else None leading = '' elif arg.startswith('--export-name'): if '=' in arg: export_name = arg.split('=', 1)[1] leading = '' elif arg.startswith('--from-emcc'): from_emcc = True leading = '' elif arg.startswith('--plugin'): plugin = open(arg.split('=', 1)[1], 'r').read() eval(plugin) # should append itself to plugins leading = '' elif leading == 'preload' or leading == 'embed': mode = leading # position of @ if we're doing 'src@dst'. '__' is used to keep the index # same with the original if they escaped with '@@'. at_position = arg.replace('@@', '__').find('@') # '@@' in input string means there is an actual @ character, a single '@' # means the 'src@dst' notation. uses_at_notation = (at_position != -1) if uses_at_notation: srcpath = arg[0:at_position].replace('@@', '@') # split around the @ dstpath = arg[at_position + 1:].replace('@@', '@') else: # Use source path as destination path. srcpath = dstpath = arg.replace('@@', '@') if os.path.isfile(srcpath) or os.path.isdir(srcpath): data_files.append({ 'srcpath': srcpath, 'dstpath': dstpath, 'mode': mode, 'explicit_dst_path': uses_at_notation }) else: print('Warning: ' + arg + ' does not exist, ignoring.', file=sys.stderr) elif leading == 'exclude': excluded_patterns.append(arg) else: print('Unknown parameter:', arg, file=sys.stderr) sys.exit(1) if (not force) and not data_files: has_preloaded = False if not has_preloaded or jsoutput is None: assert not separate_metadata, ( 'cannot separate-metadata without both --preloaded files ' 'and a specified --js-output') if not from_emcc: print( 'Remember to build the main file with -s FORCE_FILESYSTEM=1 ' 'so that it includes support for loading this file package', file=sys.stderr) ret = '' # emcc.py will add this to the output itself, so it is only needed for # standalone calls if not from_emcc: ret = ''' var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {}; ''' % { "EXPORT_NAME": export_name } ret += ''' if (!Module.expectedDataFileDownloads) { Module.expectedDataFileDownloads = 0; Module.finishedDataFileDownloads = 0; } Module.expectedDataFileDownloads++; (function() { var loadPackage = function(metadata) { ''' code = ''' function assert(check, msg) { if (!check) throw msg + new Error().stack; } ''' for file_ in data_files: if not should_ignore(file_['srcpath']): if os.path.isdir(file_['srcpath']): add(file_['mode'], file_['srcpath'], file_['dstpath']) else: new_data_files.append(file_) data_files = [ file_ for file_ in new_data_files if not os.path.isdir(file_['srcpath']) ] if len(data_files) == 0: print('Nothing to do!', file=sys.stderr) sys.exit(1) # Absolutize paths, and check that they make sense # os.getcwd() always returns the hard path with any symbolic links resolved, # even if we cd'd into a symbolic link. curr_abspath = os.path.abspath(os.getcwd()) for file_ in data_files: if not file_['explicit_dst_path']: # This file was not defined with src@dst, so we inferred the destination # from the source. In that case, we require that the destination not be # under the current location path = file_['dstpath'] # Use os.path.realpath to resolve any symbolic links to hard paths, # to match the structure in curr_abspath. abspath = os.path.realpath(os.path.abspath(path)) if DEBUG: print(path, abspath, curr_abspath, file=sys.stderr) if not abspath.startswith(curr_abspath): print( 'Error: Embedding "%s" which is below the current directory ' '"%s". This is invalid since the current directory becomes the ' 'root that the generated code will see' % (path, curr_abspath), file=sys.stderr) sys.exit(1) file_['dstpath'] = abspath[len(curr_abspath) + 1:] if os.path.isabs(path): print( 'Warning: Embedding an absolute file/directory name "%s" to the ' 'virtual filesystem. The file will be made available in the ' 'relative path "%s". You can use the explicit syntax ' '--preload-file srcpath@dstpath to explicitly specify the target ' 'location the absolute source path should be directed to.' % (path, file_['dstpath']), file=sys.stderr) for file_ in data_files: # name in the filesystem, native and emulated file_['dstpath'] = file_['dstpath'].replace(os.path.sep, '/') # If user has submitted a directory name as the destination but omitted # the destination filename, use the filename from source file if file_['dstpath'].endswith('/'): file_['dstpath'] = file_['dstpath'] + os.path.basename( file_['srcpath']) # make destination path always relative to the root file_['dstpath'] = posixpath.normpath( os.path.join('/', file_['dstpath'])) if DEBUG: print('Packaging file "%s" to VFS in path "%s".' % (file_['srcpath'], file_['dstpath']), file=sys.stderr) # Remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/) seen = {} def was_seen(name): if seen.get(name): return True seen[name] = 1 return False data_files = [ file_ for file_ in data_files if not was_seen(file_['dstpath']) ] if AV_WORKAROUND: random.shuffle(data_files) # Apply plugins for file_ in data_files: for plugin in plugins: plugin(file_) metadata = {'files': []} # Set up folders partial_dirs = [] for file_ in data_files: dirname = os.path.dirname(file_['dstpath']) dirname = dirname.lstrip( '/') # absolute paths start with '/', remove that if dirname != '': parts = dirname.split('/') for i in range(len(parts)): partial = '/'.join(parts[:i + 1]) if partial not in partial_dirs: code += ( '''Module['FS_createPath']('/%s', '%s', true, true);\n''' % ('/'.join(parts[:i]), parts[i])) partial_dirs.append(partial) if has_preloaded: # Bundle all datafiles into one archive. Avoids doing lots of simultaneous # XHRs which has overhead. data = open(data_target, 'wb') start = 0 for file_ in data_files: file_['data_start'] = start curr = open(file_['srcpath'], 'rb').read() file_['data_end'] = start + len(curr) if AV_WORKAROUND: curr += '\x00' start += len(curr) data.write(curr) data.close() # TODO: sha256sum on data_target if start > 256 * 1024 * 1024: print( 'warning: file packager is creating an asset bundle of %d MB. ' 'this is very large, and browsers might have trouble loading it. ' 'see https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/' % (start / (1024 * 1024)), file=sys.stderr) create_preloaded = ''' Module['FS_createPreloadedFile'](this.name, null, byteArray, true, true, function() { Module['removeRunDependency']('fp ' + that.name); }, function() { if (that.audio) { Module['removeRunDependency']('fp ' + that.name); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang) } else { err('Preloading file ' + that.name + ' failed'); } }, false, true); // canOwn this data in the filesystem, it is a slide into the heap that will never change ''' create_data = ''' Module['FS_createDataFile'](this.name, null, byteArray, true, true, true); // canOwn this data in the filesystem, it is a slide into the heap that will never change Module['removeRunDependency']('fp ' + that.name); ''' # Data requests - for getting a block of data out of the big archive - have # a similar API to XHRs code += ''' function DataRequest(start, end, audio) { this.start = start; this.end = end; this.audio = audio; } DataRequest.prototype = { requests: {}, open: function(mode, name) { this.name = name; this.requests[name] = this; Module['addRunDependency']('fp ' + this.name); }, send: function() {}, onload: function() { var byteArray = this.byteArray.subarray(this.start, this.end); this.finish(byteArray); }, finish: function(byteArray) { var that = this; %s this.requests[this.name] = null; } }; %s ''' % (create_preloaded if use_preload_plugins else create_data, ''' var files = metadata['files']; for (var i = 0; i < files.length; ++i) { new DataRequest(files[i]['start'], files[i]['end'], files[i]['audio']).open('GET', files[i]['filename']); } ''' if not lz4 else '') counter = 0 for file_ in data_files: filename = file_['dstpath'] dirname = os.path.dirname(filename) basename = os.path.basename(filename) if file_['mode'] == 'embed': # Embed data = list(bytearray(open(file_['srcpath'], 'rb').read())) code += '''var fileData%d = [];\n''' % counter if data: parts = [] chunk_size = 10240 start = 0 while start < len(data): parts.append( '''fileData%d.push.apply(fileData%d, %s);\n''' % (counter, counter, str( data[start:start + chunk_size]))) start += chunk_size code += ''.join(parts) code += ( '''Module['FS_createDataFile']('%s', '%s', fileData%d, true, true, false);\n''' % (dirname, basename, counter)) counter += 1 elif file_['mode'] == 'preload': # Preload counter += 1 metadata['files'].append({ 'filename': file_['dstpath'], 'start': file_['data_start'], 'end': file_['data_end'], 'audio': 1 if filename[-4:] in AUDIO_SUFFIXES else 0, }) else: assert 0 if has_preloaded: if not lz4: # Get the big archive and split it up if heap_copy: use_data = ''' // copy the entire loaded file into a spot in the heap. Files will refer to slices in that. They cannot be freed though // (we may be allocating before malloc is ready, during startup). var ptr = Module['getMemory'](byteArray.length); Module['HEAPU8'].set(byteArray, ptr); DataRequest.prototype.byteArray = Module['HEAPU8'].subarray(ptr, ptr+byteArray.length); ''' else: use_data = ''' // Reuse the bytearray from the XHR as the source for file reads. DataRequest.prototype.byteArray = byteArray; ''' use_data += ''' var files = metadata['files']; for (var i = 0; i < files.length; ++i) { DataRequest.prototype.requests[files[i].filename].onload(); } ''' use_data += ( " Module['removeRunDependency']('datafile_%s');\n" % shared.JS.escape_for_js_string(data_target)) else: # LZ4FS usage temp = data_target + '.orig' shutil.move(data_target, temp) meta = run_js(shared.path_from_root('tools', 'lz4-compress.js'), shared.NODE_JS, [ shared.path_from_root('src', 'mini-lz4.js'), temp, data_target ], stdout=PIPE) os.unlink(temp) use_data = ''' var compressedData = %s; compressedData['data'] = byteArray; assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with -s LZ4=1 ?'); LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData }); Module['removeRunDependency']('datafile_%s'); ''' % (meta, shared.JS.escape_for_js_string(data_target)) package_uuid = uuid.uuid4() package_name = data_target remote_package_size = os.path.getsize(package_name) remote_package_name = os.path.basename(package_name) ret += r''' var PACKAGE_PATH; if (typeof window === 'object') { PACKAGE_PATH = window['encodeURIComponent'](window.location.pathname.toString().substring(0, window.location.pathname.toString().lastIndexOf('/')) + '/'); } else if (typeof location !== 'undefined') { // worker PACKAGE_PATH = encodeURIComponent(location.pathname.toString().substring(0, location.pathname.toString().lastIndexOf('/')) + '/'); } else { throw 'using preloaded data can only be done on a web page or in a web worker'; } var PACKAGE_NAME = '%s'; var REMOTE_PACKAGE_BASE = '%s'; if (typeof Module['locateFilePackage'] === 'function' && !Module['locateFile']) { Module['locateFile'] = Module['locateFilePackage']; err('warning: you defined Module.locateFilePackage, that has been renamed to Module.locateFile (using your locateFilePackage for now)'); } var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE; ''' % (shared.JS.escape_for_js_string(data_target), shared.JS.escape_for_js_string(remote_package_name)) metadata['remote_package_size'] = remote_package_size metadata['package_uuid'] = str(package_uuid) ret += ''' var REMOTE_PACKAGE_SIZE = metadata['remote_package_size']; var PACKAGE_UUID = metadata['package_uuid']; ''' if use_preload_cache: code += r''' var indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB; var IDB_RO = "readonly"; var IDB_RW = "readwrite"; var DB_NAME = "''' + indexeddb_name + '''"; var DB_VERSION = 1; var METADATA_STORE_NAME = 'METADATA'; var PACKAGE_STORE_NAME = 'PACKAGES'; function openDatabase(callback, errback) { try { var openRequest = indexedDB.open(DB_NAME, DB_VERSION); } catch (e) { return errback(e); } openRequest.onupgradeneeded = function(event) { var db = event.target.result; if(db.objectStoreNames.contains(PACKAGE_STORE_NAME)) { db.deleteObjectStore(PACKAGE_STORE_NAME); } var packages = db.createObjectStore(PACKAGE_STORE_NAME); if(db.objectStoreNames.contains(METADATA_STORE_NAME)) { db.deleteObjectStore(METADATA_STORE_NAME); } var metadata = db.createObjectStore(METADATA_STORE_NAME); }; openRequest.onsuccess = function(event) { var db = event.target.result; callback(db); }; openRequest.onerror = function(error) { errback(error); }; }; // This is needed as chromium has a limit on per-entry files in IndexedDB // https://cs.chromium.org/chromium/src/content/renderer/indexed_db/webidbdatabase_impl.cc?type=cs&sq=package:chromium&g=0&l=177 // https://cs.chromium.org/chromium/src/out/Debug/gen/third_party/blink/public/mojom/indexeddb/indexeddb.mojom.h?type=cs&sq=package:chromium&g=0&l=60 // We set the chunk size to 64MB to stay well-below the limit var CHUNK_SIZE = 64 * 1024 * 1024; function cacheRemotePackage( db, packageName, packageData, packageMeta, callback, errback ) { var transactionPackages = db.transaction([PACKAGE_STORE_NAME], IDB_RW); var packages = transactionPackages.objectStore(PACKAGE_STORE_NAME); var chunkSliceStart = 0; var nextChunkSliceStart = 0; var chunkCount = Math.ceil(packageData.byteLength / CHUNK_SIZE); var finishedChunks = 0; for (var chunkId = 0; chunkId < chunkCount; chunkId++) { nextChunkSliceStart += CHUNK_SIZE; var putPackageRequest = packages.put( packageData.slice(chunkSliceStart, nextChunkSliceStart), 'package/' + packageName + '/' + chunkId ); chunkSliceStart = nextChunkSliceStart; putPackageRequest.onsuccess = function(event) { finishedChunks++; if (finishedChunks == chunkCount) { var transaction_metadata = db.transaction( [METADATA_STORE_NAME], IDB_RW ); var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME); var putMetadataRequest = metadata.put( { 'uuid': packageMeta.uuid, 'chunkCount': chunkCount }, 'metadata/' + packageName ); putMetadataRequest.onsuccess = function(event) { callback(packageData); }; putMetadataRequest.onerror = function(error) { errback(error); }; } }; putPackageRequest.onerror = function(error) { errback(error); }; } } /* Check if there's a cached package, and if so whether it's the latest available */ function checkCachedPackage(db, packageName, callback, errback) { var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO); var metadata = transaction.objectStore(METADATA_STORE_NAME); var getRequest = metadata.get('metadata/' + packageName); getRequest.onsuccess = function(event) { var result = event.target.result; if (!result) { return callback(false, null); } else { return callback(PACKAGE_UUID === result['uuid'], result); } }; getRequest.onerror = function(error) { errback(error); }; } function fetchCachedPackage(db, packageName, metadata, callback, errback) { var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO); var packages = transaction.objectStore(PACKAGE_STORE_NAME); var chunksDone = 0; var totalSize = 0; var chunkCount = metadata['chunkCount']; var chunks = new Array(chunkCount); for (var chunkId = 0; chunkId < chunkCount; chunkId++) { var getRequest = packages.get('package/' + packageName + '/' + chunkId); getRequest.onsuccess = function(event) { // If there's only 1 chunk, there's nothing to concatenate it with so we can just return it now if (chunkCount == 1) { callback(event.target.result); } else { chunksDone++; totalSize += event.target.result.byteLength; chunks.push(event.target.result); if (chunksDone == chunkCount) { if (chunksDone == 1) { callback(event.target.result); } else { var tempTyped = new Uint8Array(totalSize); var byteOffset = 0; for (var chunkId in chunks) { var buffer = chunks[chunkId]; tempTyped.set(new Uint8Array(buffer), byteOffset); byteOffset += buffer.byteLength; buffer = undefined; } chunks = undefined; callback(tempTyped.buffer); tempTyped = undefined; } } } }; getRequest.onerror = function(error) { errback(error); }; } } ''' ret += r''' function fetchRemotePackage(packageName, packageSize, callback, errback) { var xhr = new XMLHttpRequest(); xhr.open('GET', packageName, true); xhr.responseType = 'arraybuffer'; xhr.onprogress = function(event) { var url = packageName; var size = packageSize; if (event.total) size = event.total; if (event.loaded) { if (!xhr.addedTotal) { xhr.addedTotal = true; if (!Module.dataFileDownloads) Module.dataFileDownloads = {}; Module.dataFileDownloads[url] = { loaded: event.loaded, total: size }; } else { Module.dataFileDownloads[url].loaded = event.loaded; } var total = 0; var loaded = 0; var num = 0; for (var download in Module.dataFileDownloads) { var data = Module.dataFileDownloads[download]; total += data.total; loaded += data.loaded; num++; } total = Math.ceil(total * Module.expectedDataFileDownloads/num); if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')'); } else if (!Module.dataFileDownloads) { if (Module['setStatus']) Module['setStatus']('Downloading data...'); } }; xhr.onerror = function(event) { throw new Error("NetworkError for: " + packageName); } xhr.onload = function(event) { if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0 var packageData = xhr.response; callback(packageData); } else { throw new Error(xhr.statusText + " : " + xhr.responseURL); } }; xhr.send(null); }; function handleError(error) { console.error('package error:', error); }; ''' code += r''' function processPackageData(arrayBuffer) { Module.finishedDataFileDownloads++; assert(arrayBuffer, 'Loading data file failed.'); assert(arrayBuffer instanceof ArrayBuffer, 'bad input to processPackageData'); var byteArray = new Uint8Array(arrayBuffer); var curr; %s }; Module['addRunDependency']('datafile_%s'); ''' % (use_data, shared.JS.escape_for_js_string(data_target)) # use basename because from the browser's point of view, # we need to find the datafile in the same dir as the html file code += r''' if (!Module.preloadResults) Module.preloadResults = {}; ''' if use_preload_cache: code += r''' function preloadFallback(error) { console.error(error); console.error('falling back to default preload behavior'); fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, processPackageData, handleError); }; openDatabase( function(db) { checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, function(useCached, metadata) { Module.preloadResults[PACKAGE_NAME] = {fromCache: useCached}; if (useCached) { fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, metadata, processPackageData, preloadFallback); } else { fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, function(packageData) { cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}, processPackageData, function(error) { console.error(error); processPackageData(packageData); }); } , preloadFallback); } } , preloadFallback); } , preloadFallback); if (Module['setStatus']) Module['setStatus']('Downloading...'); ''' else: # Not using preload cache, so we might as well start the xhr ASAP, # potentially before JS parsing of the main codebase if it's after us. # Only tricky bit is the fetch is async, but also when runWithFS is called # is async, so we handle both orderings. ret += r''' var fetchedCallback = null; var fetched = Module['getPreloadedPackage'] ? Module['getPreloadedPackage'](REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE) : null; if (!fetched) fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, function(data) { if (fetchedCallback) { fetchedCallback(data); fetchedCallback = null; } else { fetched = data; } }, handleError); ''' code += r''' Module.preloadResults[PACKAGE_NAME] = {fromCache: false}; if (fetched) { processPackageData(fetched); fetched = null; } else { fetchedCallback = processPackageData; } ''' ret += ''' function runWithFS() { ''' ret += code ret += ''' } if (Module['calledRun']) { runWithFS(); } else { if (!Module['preRun']) Module['preRun'] = []; Module["preRun"].push(runWithFS); // FS is not initialized yet, wait for it } ''' if separate_metadata: _metadata_template = ''' Module['removeRunDependency']('%(metadata_file)s'); } function runMetaWithFS() { Module['addRunDependency']('%(metadata_file)s'); var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s'; var xhr = new XMLHttpRequest(); xhr.onreadystatechange = function() { if (xhr.readyState === 4 && xhr.status === 200) { loadPackage(JSON.parse(xhr.responseText)); } } xhr.open('GET', REMOTE_METADATA_NAME, true); xhr.overrideMimeType('application/json'); xhr.send(null); } if (Module['calledRun']) { runMetaWithFS(); } else { if (!Module['preRun']) Module['preRun'] = []; Module["preRun"].push(runMetaWithFS); } ''' % { 'metadata_file': os.path.basename(jsoutput + '.metadata') } else: _metadata_template = ''' } loadPackage(%s); ''' % json.dumps(metadata) ret += '''%s })(); ''' % _metadata_template if force or len(data_files): if jsoutput is None: print(ret) else: # Overwrite the old jsoutput file (if exists) only when its content # differs from the current generated one, otherwise leave the file # untouched preserving its old timestamp if os.path.isfile(jsoutput): f = open(jsoutput, 'r+') old = f.read() if old != ret: f.seek(0) f.write(ret) f.truncate() else: f = open(jsoutput, 'w') f.write(ret) f.close() if separate_metadata: f = open(jsoutput + '.metadata', 'w') json.dump(metadata, f, separators=(',', ':')) f.close() return 0
''' use_data += ''' var files = metadata.files; for (var i = 0; i < files.length; ++i) { DataRequest.prototype.requests[files[i].filename].onload(); } ''' use_data += (" Module['removeRunDependency']('datafile_%s');\n" % shared.JS.escape_for_js_string(data_target)) else: # LZ4FS usage temp = data_target + '.orig' shutil.move(data_target, temp) meta = run_js(shared.path_from_root('tools', 'lz4-compress.js'), shared.NODE_JS, [shared.path_from_root('src', 'mini-lz4.js'), temp, data_target], stdout=PIPE) os.unlink(temp) use_data = ''' var compressedData = %s; compressedData.data = byteArray; assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with -s LZ4=1 ?'); LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData }); Module['removeRunDependency']('datafile_%s'); ''' % (meta, shared.JS.escape_for_js_string(data_target)) package_uuid = uuid.uuid4() package_name = data_target statinfo = os.stat(package_name) remote_package_size = statinfo.st_size remote_package_name = os.path.basename(package_name)
var files = metadata.files; for (var i = 0; i < files.length; ++i) { DataRequest.prototype.requests[files[i].filename].onload(); } ''' use_data += ( " Module['removeRunDependency']('datafile_%s');\n" % shared.JS.escape_for_js_string(data_target)) else: # LZ4FS usage temp = data_target + '.orig' shutil.move(data_target, temp) meta = run_js( shared.path_from_root('tools', 'lz4-compress.js'), shared.NODE_JS, [shared.path_from_root('src', 'mini-lz4.js'), temp, data_target], stdout=PIPE) os.unlink(temp) use_data = ''' var compressedData = %s; compressedData.data = byteArray; assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with -s LZ4=1 ?'); LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData }); Module['removeRunDependency']('datafile_%s'); ''' % (meta, shared.JS.escape_for_js_string(data_target)) package_uuid = uuid.uuid4() package_name = data_target remote_package_size = os.path.getsize(package_name) remote_package_name = os.path.basename(package_name)