def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() curr_func = None for line in ll_lines: if in_func: curr_func.append(line) if line.startswith('}'): in_func = False funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier # pre needs to know about all implemented functions, even for non-pre func pre.append(curr_func[0]) pre.append(line) curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) if len(funcs) == 0: print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)' #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input) dfp.write("\n\n========================== settings_text\n\n") dfp.write(settings_text) dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) indexed_functions = set() forwarded_json = json.loads(forwarded_data) for key in forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if len(chunks) > 0: if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) commands = [ (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks) ] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['hasInlineJS'] = forwarded_json['Types']['hasInlineJS'] or curr_forwarded_json['Types']['hasInlineJS'] forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] export_all = settings['EXPORT_ALL'] for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys(): if key in all_exported_functions or export_all or (export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value for key, value in curr_forwarded_json['Functions']['neededTables'].iteritems(): forwarded_json['Functions']['neededTables'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 # universal counter if settings['ASM_JS']: i += 2*settings['RESERVED_FUNCTION_POINTERS'] table_counters = {} # table-specific counters alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] sig = None for indexed in indexed_functions: if alias: sig = forwarded_json['Functions']['implementedFunctions'].get(indexed) or forwarded_json['Functions']['unimplementedFunctions'].get(indexed) assert sig, indexed if sig not in table_counters: table_counters[sig] = 2 + 2*settings['RESERVED_FUNCTION_POINTERS'] curr = table_counters[sig] table_counters[sig] += 2 else: curr = i i += 2 #print >> sys.stderr, 'function indexing', indexed, curr, sig forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): x = int(x) return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): # In the global initial allocation, we need to split up into Uint8 format ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) pre = blockaddrsize(indexize(pre)) if settings.get('ASM_JS'): # move postsets into the asm module class PostSets: js = '' def handle_post_sets(m): PostSets.js = m.group(0) return '\n' pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre) #if DEBUG: outfile.write('// pre\n') outfile.write(pre) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE, cwd=path_from_root('src')) post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig)-1)]) coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) start = raw.index('[') end = raw.rindex(']') body = raw[start+1:end].split(',') for j in range(settings['RESERVED_FUNCTION_POINTERS']): body[2 + 2*j] = 'jsCall_%s_%s' % (sig, j) def fix_item(item): newline = '\n' in item return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') body = ','.join(map(fix_item, body)) return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start+1] + body + raw[end:]) infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) asm_setup = '' maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) if settings['TO_FLOAT32']: maths += ['Math.toFloat32'] basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat'] + [m.replace('.', '_') for m in math_envs] if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] if settings['ASSERTIONS']: basic_funcs += ['nullFunc'] asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n' basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed'] or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] if settings.get('DLOPEN_SUPPORT'): for sig in last_forwarded_json['Functions']['tables'].iterkeys(): basic_vars.append('F_BASE_%s' % sig) asm_setup += ' var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n' asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) for i in range(settings['RESERVED_FUNCTION_POINTERS']): jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce('jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0]) function_tables_impls.append(''' function jsCall_%s_%s(%s) { %s %s; } ''' % (sig, i, args, arg_coercions, jsret)) from tools import shared shared.Settings.copy(settings) asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' basic_funcs.append('invoke_%s' % sig) if settings.get('DLOPEN_SUPPORT'): asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n' basic_funcs.append('extCall_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exported_implemented_functions.append('runPostSets') exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): assert settings.get('TARGET_LE32'), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' for key, value in forwarded_json['Variables']['globals'].iteritems(): if value.get('linkable'): init = forwarded_json['Variables']['indexedGlobals'][key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' asm_global_vars += ' var %s=%s;\n' % (key, str(init)) # sent data the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' # received if not simple: receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) funcs_js = [''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { %s var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup, "'use asm';" if not forwarded_json['Types']['hasInlineJS'] and not settings['SIDE_MODULE'] else "'almost asm';") + '\n' + asm_global_vars + ''' var __THREW__ = 0; var threwValue = 0; var setjmpId = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; ''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw, value) { threw = threw|0; value = value|0; if ((__THREW__|0) == 0) { __THREW__ = threw; threwValue = value; } } function copyTempFloat(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; } function copyTempDouble(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0]; HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0]; HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0]; HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0]; } ''' + ''.join([''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10)])] + [PostSets.js + '\n'] + funcs_js + [''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] if not settings.get('SIDE_MODULE'): funcs_js.append(''' Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) }; Runtime.stackSave = function() { return asm['stackSave']() }; Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; ''') # Set function table masks masks = {} max_mask = 0 for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): mask = table.count(',') masks[sig] = str(mask) max_mask = max(mask, max_mask) def function_table_maskize(js, masks): def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js) if settings.get('DLOPEN_SUPPORT'): funcs_js.append(''' asm.maxFunctionIndex = %(max_mask)d; DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module); Module.SYMBOL_TABLE = SYMBOL_TABLE; ''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) else: function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] # Create symbol table for self-dlopen if settings.get('DLOPEN_SUPPORT'): symbol_table = {} for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems(): if forwarded_json['Variables']['globals'][k]['named']: symbol_table[k] = str(v + forwarded_json['Runtime']['GLOBAL_BASE']) for raw in last_forwarded_json['Functions']['tables'].itervalues(): if raw == '': continue table = map(string.strip, raw[raw.find('[')+1:raw.find(']')].split(",")) for i in range(len(table)): value = table[i] if value != '0': if settings.get('SIDE_MODULE'): symbol_table[value] = 'FUNCTION_TABLE_OFFSET+' + str(i) else: symbol_table[value] = str(i) outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table).replace('"', '')) for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % ( time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() curr_func = None for line in ll_lines: if in_func: curr_func.append(line) if line.startswith('}'): in_func = False funcs.append((curr_func[0], ''.join(curr_func) )) # use the entire line as the identifier # pre needs to know about all implemented functions, even for non-pre func pre.append(curr_func[0]) pre.append(line) curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append( line ) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024 * 1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % ( time.time() - t) if len(funcs) == 0: print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)' #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >> sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input) dfp.write("\n\n========================== settings_text\n\n") dfp.write(settings_text) dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() print >> sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % ( time.time() - t) indexed_functions = set() forwarded_json = json.loads(forwarded_data) for key in forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3 * len(meta) + len( forwarded_data ) / 3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey( keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len( cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if len(chunks) > 0: if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len( chunks) == 1, 'no point in splitting up without multiple cores' if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % ( len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size / (1024 * 1024.), len(meta) / (1024 * 1024.), len(forwarded_data) / (1024 * 1024.), total_ll_size / (1024 * 1024.)) commands = [(i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks)] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len( chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len( output ) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[ 0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set( settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings[ 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json[ 'Types']['preciseI64MathUsed'] or curr_forwarded_json['Types'][ 'preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions'][ 'blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions'][ 'indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] export_all = settings['EXPORT_ALL'] for key in curr_forwarded_json['Functions'][ 'implementedFunctions'].iterkeys(): if key in all_exported_functions or export_all or ( export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions'][ 'unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value for key, value in curr_forwarded_json['Functions'][ 'neededTables'].iteritems(): forwarded_json['Functions']['neededTables'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 # universal counter if settings['ASM_JS']: i += 2 * settings['RESERVED_FUNCTION_POINTERS'] table_counters = {} # table-specific counters alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS'] sig = None for indexed in indexed_functions: if alias: sig = forwarded_json['Functions']['implementedFunctions'].get( indexed) or forwarded_json['Functions'][ 'unimplementedFunctions'].get(indexed) assert sig, indexed if sig not in table_counters: table_counters[ sig] = 2 + 2 * settings['RESERVED_FUNCTION_POINTERS'] curr = table_counters[sig] table_counters[sig] += 2 else: curr = i i += 2 #print >> sys.stderr, 'function indexing', indexed, curr, sig forwarded_json['Functions']['indexedFunctions'][ indexed] = curr # make sure not to modify this python object later - we use it in indexize def split_32(x): x = int(x) return '%d,%d,%d,%d' % (x & 255, (x >> 8) & 255, (x >> 16) & 255, (x >> 24) & 255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): # In the global initial allocation, we need to split up into Uint8 format ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): ret = re.sub( r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) return re.sub( r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) pre = blockaddrsize(indexize(pre)) if settings.get('ASM_JS'): # move postsets into the asm module class PostSets: js = '' def handle_post_sets(m): PostSets.js = m.group(0) return '\n' pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre) #if DEBUG: outfile.write('// pre\n') outfile.write(pre) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % ( time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) post, last_forwarded_data = out.split( '//FORWARDED_DATA:' ) # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig) - 1)]) coercions = ';'.join([ 'p%d = %sp%d%s' % (p, '+' if sig[p + 1] != 'i' else '', p, '' if sig[p + 1] != 'i' else '|0') for p in range(len(sig) - 1) ]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) start = raw.index('[') end = raw.rindex(']') body = raw[start + 1:end].split(',') for j in range(settings['RESERVED_FUNCTION_POINTERS']): body[2 + 2 * j] = 'jsCall_%s_%s' % (sig, j) def fix_item(item): newline = '\n' in item return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '') body = ','.join(map(fix_item, body)) return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start + 1] + body + raw[end:]) infos = [ make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems() ] function_tables_defs = '\n'.join( [info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join( [info[1] for info in infos]) asm_setup = '' maths = [ 'Math.' + func for func in [ 'floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul' ] ] fundamentals = [ 'Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array' ] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join( ['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) if settings['TO_FLOAT32']: maths += ['Math.toFloat32'] basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat' ] + [m.replace('.', '_') for m in math_envs] if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall') if settings['SAFE_HEAP']: basic_funcs += [ 'SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR' ] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] if settings['ASSERTIONS']: basic_funcs += ['nullFunc'] asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n' basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed'] or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] asm_runtime_funcs = [ 'stackAlloc', 'stackSave', 'stackRestore', 'setThrew' ] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = [ 'dynCall_' + table for table in last_forwarded_json['Functions']['tables'] ] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join([ 'a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig)) ]) coerced_args = ','.join( [asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce( 'FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) for i in range(settings['RESERVED_FUNCTION_POINTERS']): jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce( 'jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0]) function_tables_impls.append(''' function jsCall_%s_%s(%s) { %s %s; } ''' % (sig, i, args, arg_coercions, jsret)) from tools import shared asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n' basic_funcs.append('invoke_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exported_implemented_functions.append('runPostSets') exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals'][ '_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map( lambda g: g['name'], filter( lambda g: settings['NAMED_GLOBALS'] or g.get( 'external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = [ '_' + key for key, value in forwarded_json['Functions'] ['libraryFunctions'].iteritems() if value != 2 ] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'): assert settings.get( 'TARGET_LE32' ), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)' for key, value in forwarded_json['Variables']['globals'].iteritems( ): if value.get('linkable'): init = forwarded_json['Variables']['indexedGlobals'][ key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0' asm_global_vars += ' var %s=%s;\n' % (key, str(init)) # sent data the_global = '{ ' + ', '.join( ['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join([ '"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars ]) + ' }' # received if not simple: receiving = ';\n'.join([ 'var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables ]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map( len, funcs_js), len(asm_setup), len(asm_global_vars), len( asm_global_funcs), len(pre_tables), len( '\n'.join(function_tables_impls)), len( function_tables_defs.replace( '\n', '\n ')), len(exports), len( the_global), len(sending), len(receiving) funcs_js = [ ''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { 'use asm'; var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup, ) + '\n' + asm_global_vars + ''' var __THREW__ = 0; var threwValue = 0; var setjmpId = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; ''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + ''' return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw, value) { threw = threw|0; value = value|0; if ((__THREW__|0) == 0) { __THREW__ = threw; threwValue = value; } } function copyTempFloat(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; } function copyTempDouble(ptr) { ptr = ptr|0; HEAP8[tempDoublePtr] = HEAP8[ptr]; HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0]; HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0]; HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0]; HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0]; HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0]; HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0]; HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0]; } ''' + ''.join([ ''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10) ]) ] + [PostSets.js + '\n'] + funcs_js + [ ''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) }; Runtime.stackSave = function() { return asm['stackSave']() }; Runtime.stackRestore = function(top) { asm['stackRestore'](top) }; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving) ] # Set function table masks def function_table_maskize(js): masks = {} default = None for sig, table in last_forwarded_json['Functions'][ 'tables'].iteritems(): masks[sig] = str(table.count(',')) default = sig def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([ table for table in last_forwarded_json['Functions'] ['tables'].itervalues() ]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] # Create symbol table for self-dlopen if settings.get('DLOPEN_SUPPORT'): symbol_table = {} for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems(): if forwarded_json['Variables']['globals'][k]['named']: symbol_table[k] = v + forwarded_json['Runtime']['GLOBAL_BASE'] for raw in last_forwarded_json['Functions']['tables'].itervalues(): if raw == '': continue table = map(string.strip, raw[raw.find('[') + 1:raw.find(']')].split(",")) symbol_table.update( map(lambda x: (x[1], x[0]), filter(lambda x: x[1] != '0', enumerate(table)))) outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table)) for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % ( time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % (time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() curr_func = None for line in ll_lines: if in_func: curr_func.append(line) if line.startswith('}'): in_func = False funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier # pre needs to know about all implemented functions, even for non-pre func pre.append(curr_func[0]) pre.append(line) curr_func = None else: if line.startswith(';'): continue if line.startswith('define '): in_func = True curr_func = [line] elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % (time.time() - t) #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input); dfp.write("\n\n========================== settings_text\n\n"); dfp.write(settings_text); dfp.write("\n\n========================== libraries\n\n"); dfp.write("\n".join(libraries)) dfp.close() print >>sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % (time.time() - t) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() forwarded_json = json.loads(forwarded_data) indexed_functions = set() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len(cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if len(chunks) > 0: if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len(chunks) == 1, 'no point in splitting up without multiple cores' if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.)) commands = [ (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks) ] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len(chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys(): if key in all_exported_functions or (export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % (time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 for indexed in indexed_functions: #print >> sys.stderr, 'indaxx', indexed, i forwarded_json['Functions']['indexedFunctions'][indexed] = i # make sure not to modify this python object later - we use it in indexize i += 2 forwarded_json['Functions']['nextIndex'] = i def split_32(x): x = int(x) return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255) indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): # In the global initial allocation, we need to split up into Uint8 format ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js) return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret) #if DEBUG: outfile.write('// pre\n') outfile.write(blockaddrsize(indexize(pre))) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % (time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest # Move preAsms to their right place def move_preasm(m): contents = m.groups(0)[0] outfile.write(contents + '\n') return '' post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs) funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] # Find function table calls without function tables generated for them for funcs_js_item in funcs_js: for use in set(re.findall(r'{{{ FTM_[\w\d_$]+ }}}', funcs_js_item)): sig = use[8:len(use)-4] if sig not in last_forwarded_json['Functions']['tables']: if DEBUG: print >> sys.stderr, 'add empty function table', sig last_forwarded_json['Functions']['tables'][sig] = 'var FUNCTION_TABLE_' + sig + ' = [0,0];\n' def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig)-1)]) coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) return ('function %s(%s) { %s abort(%d); %s }' % (bad, params, coercions, i, ret), raw.replace('[0,', '[' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0]', ',' + bad + ']').replace(',0]', ',' + bad + ']').replace(',0\n', ',' + bad + '\n')) infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()] function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos]) asm_setup = '' maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']] fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat'] + [m.replace('.', '_') for m in math_envs] if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed'] or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \ forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'): basic_vars += ['cttz_i8', 'ctlz_i8'] asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))]) coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) args = ','.join(['a' + str(i) for i in range(1, len(sig))]) args = 'index' + (',' if args else '') + args asm_setup += ''' function invoke_%s(%s) { try { %sModule.dynCall_%s(%s); } catch(e) { asm.setThrew(1); } } ''' % (sig, args, 'return ' if sig[0] != 'v' else '', sig, args) basic_funcs.append('invoke_%s' % sig) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1]; asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # sent data the_global = '{ ' + ', '.join([math_fix(s) + ': ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join([math_fix(s) + ': ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }' # received if not simple: receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm.' + s for s in exported_implemented_functions + function_tables]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n ')), len(exports), len(the_global), len(sending), len(receiving) funcs_js = [''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { 'use asm'; var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup,) + '\n' + asm_global_vars + ''' var __THREW__ = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; STACKTOP = ((STACKTOP + 3)>>2)<<2; return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw) { threw = threw|0; __THREW__ = threw; } ''' + ''.join([''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10)])] + funcs_js + [''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) }; Runtime.stackSave = function() { return asm.stackSave() }; Runtime.stackRestore = function(top) { asm.stackRestore(top) }; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving)] # Set function table masks def function_table_maskize(js): masks = {} default = None for sig, table in last_forwarded_json['Functions']['tables'].iteritems(): masks[sig] = str(table.count(',')) default = sig def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % (time.time() - t) outfile.close()
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None, jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None): """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible Args: infile: The path to the input LLVM assembly file. settings: JSON-formatted settings that override the values defined in src/settings.js. outfile: The file where the output is written. """ compiler = path_from_root('src', 'compiler.js') # Parallelization: We run 3 phases: # 1 aka 'pre' : Process types and metadata and so forth, and generate the preamble. # 2 aka 'funcs': Process functions. We can parallelize this, working on each function independently. # 3 aka 'post' : Process globals, generate postamble and finishing touches. if DEBUG: print >> sys.stderr, 'emscript: ll=>js' if jcache: jcache.ensure() # Pre-scan ll and alter settings as necessary if DEBUG: t = time.time() ll = open(infile).read() scan(ll, settings) total_ll_size = len(ll) ll = None # allow collection if DEBUG: print >> sys.stderr, ' emscript: scan took %s seconds' % ( time.time() - t) # Split input into the relevant parts for each phase pre = [] funcs = [] # split up functions here, for parallelism later func_idents = [] meta = [] # needed by each function XXX if DEBUG: t = time.time() in_func = False ll_lines = open(infile).readlines() for line in ll_lines: if in_func: funcs[-1][1].append(line) if line.startswith('}'): in_func = False funcs[-1] = (funcs[-1][0], ''.join(funcs[-1][1])) pre.append( line ) # pre needs it to, so we know about all implemented functions else: if line.startswith(';'): continue if line.startswith('define '): in_func = True funcs.append( (line, [line])) # use the entire line as the identifier pre.append( line ) # pre needs it to, so we know about all implemented functions elif line.find(' = type { ') > 0: pre.append(line) # type elif line.startswith('!'): if line.startswith('!llvm.module'): continue # we can ignore that meta.append(line) # metadata else: pre.append( line ) # pre needs it so we know about globals in pre and funcs. So emit globals there ll_lines = None meta = ''.join(meta) if DEBUG and len(meta) > 1024 * 1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down' if DEBUG: print >> sys.stderr, ' emscript: split took %s seconds' % ( time.time() - t) #if DEBUG: # print >> sys.stderr, '========= pre ================\n' # print >> sys.stderr, ''.join(pre) # print >> sys.stderr, '========== funcs ===============\n' # for func in funcs: # print >> sys.stderr, '\n// ===\n\n', ''.join(func) # print >> sys.stderr, '=========================\n' # Save settings to a file to work around v8 issue 1579 settings_file = temp_files.get('.txt').name def save_settings(): global settings_text settings_text = json.dumps(settings, sort_keys=True) s = open(settings_file, 'w') s.write(settings_text) s.close() save_settings() # Phase 1 - pre if DEBUG: t = time.time() pre_file = temp_files.get('.pre.ll').name pre_input = ''.join(pre) + '\n' + meta out = None if jcache: keys = [pre_input, settings_text, ','.join(libraries)] shortkey = jcache.get_shortkey(keys) if DEBUG_CACHE: print >> sys.stderr, 'shortkey', shortkey out = jcache.get(shortkey, keys) if DEBUG_CACHE and not out: dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey) dfp = open(dfpath, 'w') dfp.write(pre_input) dfp.write("\n\n========================== settings_text\n\n") dfp.write(settings_text) dfp.write("\n\n========================== libraries\n\n") dfp.write("\n".join(libraries)) dfp.close() print >> sys.stderr, ' cache miss, key data dumped to %s' % dfpath if out and DEBUG: print >> sys.stderr, ' loading pre from jcache' if not out: open(pre_file, 'w').write(pre_input) out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?' if jcache: if DEBUG: print >> sys.stderr, ' saving pre to jcache' jcache.set(shortkey, keys, out) pre, forwarded_data = out.split('//FORWARDED_DATA:') forwarded_file = temp_files.get('.json').name open(forwarded_file, 'w').write(forwarded_data) if DEBUG: print >> sys.stderr, ' emscript: phase 1 took %s seconds' % ( time.time() - t) # Phase 2 - func cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count()) assert cores >= 1 if cores > 1: intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE)) chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks) chunk_size += 3 * len(meta) + len( forwarded_data ) / 3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant) chunk_size = min(MAX_CHUNK_SIZE, chunk_size) else: chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size if DEBUG: t = time.time() forwarded_json = json.loads(forwarded_data) indexed_functions = set() if settings.get('ASM_JS'): settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS'] save_settings() chunks = cache_module.chunkify( funcs, chunk_size, jcache.get_cachename('emscript_files') if jcache else None) funcs = None if jcache: # load chunks from cache where we can # TODO: ignore small chunks cached_outputs = [] def load_from_cache(chunk): keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey( keys) # TODO: share shortkeys with later code out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?) if out: cached_outputs.append(out) return False return True chunks = filter(load_from_cache, chunks) if len(cached_outputs) > 0: if out and DEBUG: print >> sys.stderr, ' loading %d funcchunks from jcache' % len( cached_outputs) else: cached_outputs = [] # TODO: minimize size of forwarded data from funcs to what we actually need if cores == 1 and total_ll_size < MAX_CHUNK_SIZE: assert len( chunks) == 1, 'no point in splitting up without multiple cores' if len(chunks) > 0: if DEBUG: print >> sys.stderr, ' emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % ( len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size / (1024 * 1024.), len(meta) / (1024 * 1024.), len(forwarded_data) / (1024 * 1024.), total_ll_size / (1024 * 1024.)) commands = [(i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG) for i, chunk in enumerate(chunks)] if len(chunks) > 1: pool = multiprocessing.Pool(processes=cores) outputs = pool.map(process_funcs, commands, chunksize=1) elif len(chunks) == 1: outputs = [process_funcs(commands[0])] commands = None else: outputs = [] if jcache: # save chunks to cache for i in range(len(chunks)): chunk = chunks[i] keys = [settings_text, forwarded_data, chunk] shortkey = jcache.get_shortkey(keys) jcache.set(shortkey, keys, outputs[i]) if out and DEBUG and len(chunks) > 0: print >> sys.stderr, ' saving %d funcchunks to jcache' % len( chunks) chunks = None if jcache: outputs += cached_outputs # TODO: preserve order outputs = [output.split('//FORWARDED_DATA:') for output in outputs] for output in outputs: assert len( output ) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[ 0][-3000:] if DEBUG: print >> sys.stderr, ' emscript: phase 2 took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # merge forwarded data if settings.get('ASM_JS'): all_exported_functions = set( settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise for additional_export in settings[ 'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented all_exported_functions.add('_' + additional_export) exported_implemented_functions = set() for func_js, curr_forwarded_data in outputs: curr_forwarded_json = json.loads(curr_forwarded_data) forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json[ 'Types']['preciseI64MathUsed'] or curr_forwarded_json['Types'][ 'preciseI64MathUsed'] for key, value in curr_forwarded_json['Functions'][ 'blockAddresses'].iteritems(): forwarded_json['Functions']['blockAddresses'][key] = value for key in curr_forwarded_json['Functions'][ 'indexedFunctions'].iterkeys(): indexed_functions.add(key) if settings.get('ASM_JS'): export_bindings = settings['EXPORT_BINDINGS'] for key in curr_forwarded_json['Functions'][ 'implementedFunctions'].iterkeys(): if key in all_exported_functions or ( export_bindings and key.startswith('_emscripten_bind')): exported_implemented_functions.add(key) for key, value in curr_forwarded_json['Functions'][ 'unimplementedFunctions'].iteritems(): forwarded_json['Functions']['unimplementedFunctions'][key] = value if settings.get('ASM_JS'): parts = pre.split('// ASM_LIBRARY FUNCTIONS\n') if len(parts) > 1: pre = parts[0] outputs.append([parts[1]]) funcs_js = [output[0] for output in outputs] outputs = None if DEBUG: print >> sys.stderr, ' emscript: phase 2b took %s seconds' % ( time.time() - t) if DEBUG: t = time.time() # calculations on merged forwarded data forwarded_json['Functions']['indexedFunctions'] = {} i = 2 for indexed in indexed_functions: #print >> sys.stderr, 'indaxx', indexed, i forwarded_json['Functions']['indexedFunctions'][ indexed] = i # make sure not to modify this python object later - we use it in indexize i += 2 forwarded_json['Functions']['nextIndex'] = i indexing = forwarded_json['Functions']['indexedFunctions'] def indexize(js): return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), js) blockaddrs = forwarded_json['Functions']['blockAddresses'] def blockaddrsize(js): return re.sub( r'{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js) #if DEBUG: outfile.write('// pre\n') outfile.write(blockaddrsize(indexize(pre))) pre = None #if DEBUG: outfile.write('// funcs\n') # forward forwarded_data = json.dumps(forwarded_json) forwarded_file = temp_files.get('.2.json').name open(forwarded_file, 'w').write(indexize(forwarded_data)) if DEBUG: print >> sys.stderr, ' emscript: phase 2c took %s seconds' % ( time.time() - t) # Phase 3 - post if DEBUG: t = time.time() post_file = temp_files.get('.post.ll').name open(post_file, 'w').write('\n') # no input, just processing of forwarded data out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src')) post, last_forwarded_data = out.split( '//FORWARDED_DATA:' ) # if this fails, perhaps the process failed prior to printing forwarded data? last_forwarded_json = json.loads(last_forwarded_data) if settings.get('ASM_JS'): post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n') post = post_rest funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n'] simple = os.environ.get('EMCC_SIMPLE_ASM') class Counter: i = 0 pre_tables = last_forwarded_json['Functions']['tables']['pre'] del last_forwarded_json['Functions']['tables']['pre'] # Find function table calls without function tables generated for them for funcs_js_item in funcs_js: for use in set(re.findall(r'{{{ FTM_[\w\d_$]+ }}}', funcs_js_item)): sig = use[8:len(use) - 4] if sig not in last_forwarded_json['Functions']['tables']: if DEBUG: print >> sys.stderr, 'add empty function table', sig last_forwarded_json['Functions']['tables'][ sig] = 'var FUNCTION_TABLE_' + sig + ' = [0,0];\n' def make_table(sig, raw): i = Counter.i Counter.i += 1 bad = 'b' + str(i) params = ','.join(['p%d' % p for p in range(len(sig) - 1)]) coercions = ';'.join([ 'p%d = %sp%d%s' % (p, '+' if sig[p + 1] != 'i' else '', p, '' if sig[p + 1] != 'i' else '|0') for p in range(len(sig) - 1) ]) + ';' ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else '')) return ('function %s(%s) { %s abort(%d); %s }' % (bad, params, coercions, i, ret), raw.replace('[0,', '[' + bad + ',').replace( ',0,', ',' + bad + ',').replace(',0,', ',' + bad + ',').replace( ',0]', ',' + bad + ']').replace(',0]', ',' + bad + ']').replace( ',0\n', ',' + bad + '\n')) infos = [ make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems() ] function_tables_defs = '\n'.join( [info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join( [info[1] for info in infos]) asm_setup = '' maths = [ 'Math.' + func for func in [ 'floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul' ] ] fundamentals = [ 'Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array' ] math_envs = ['Math.min'] # TODO: move min to maths asm_setup += '\n'.join( ['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs]) basic_funcs = [ 'abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat' ] + [m.replace('.', '_') for m in math_envs] if settings['SAFE_HEAP']: basic_funcs += [ 'SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR' ] if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8'] basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT'] basic_float_vars = ['NaN', 'Infinity'] if forwarded_json['Types']['preciseI64MathUsed']: basic_funcs += [ 'i64Math_' + op for op in ['add', 'subtract', 'multiply', 'divide', 'modulo'] ] asm_setup += ''' var i64Math_add = function(a, b, c, d) { i64Math.add(a, b, c, d) }; var i64Math_subtract = function(a, b, c, d) { i64Math.subtract(a, b, c, d) }; var i64Math_multiply = function(a, b, c, d) { i64Math.multiply(a, b, c, d) }; var i64Math_divide = function(a, b, c, d, e) { i64Math.divide(a, b, c, d, e) }; var i64Math_modulo = function(a, b, c, d, e) { i64Math.modulo(a, b, c, d, e) }; ''' asm_runtime_funcs = [ 'stackAlloc', 'stackSave', 'stackRestore', 'setThrew' ] + ['setTempRet%d' % i for i in range(10)] # function tables def asm_coerce(value, sig): if sig == 'v': return value return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '') function_tables = [ 'dynCall_' + table for table in last_forwarded_json['Functions']['tables'] ] function_tables_impls = [] for sig in last_forwarded_json['Functions']['tables'].iterkeys(): args = ','.join(['a' + str(i) for i in range(1, len(sig))]) arg_coercions = ' '.join([ 'a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig)) ]) coerced_args = ','.join( [asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))]) ret = ('return ' if sig[0] != 'v' else '') + asm_coerce( 'FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0]) function_tables_impls.append(''' function dynCall_%s(index%s%s) { index = index|0; %s %s; } ''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret)) # calculate exports exported_implemented_functions = list(exported_implemented_functions) exports = [] if not simple: for export in exported_implemented_functions + asm_runtime_funcs + function_tables: exports.append("%s: %s" % (export, export)) exports = '{ ' + ', '.join(exports) + ' }' else: exports = '_main' # calculate globals try: del forwarded_json['Variables']['globals'][ '_llvm_global_ctors'] # not a true variable except: pass # If no named globals, only need externals global_vars = map( lambda g: g['name'], filter( lambda g: settings['NAMED_GLOBALS'] or g.get( 'external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values())) global_funcs = [ '_' + x for x in forwarded_json['Functions']['libraryFunctions'].keys() ] def math_fix(g): return g if not g.startswith('Math_') else g.split('_')[1] asm_global_funcs = ''.join([' var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \ ''.join([' var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs]) asm_global_vars = ''.join([' var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \ ''.join([' var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars]) # sent data the_global = '{ ' + ', '.join( [math_fix(s) + ': ' + s for s in fundamentals]) + ' }' sending = '{ ' + ', '.join([ math_fix(s) + ': ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars ]) + ' }' # received if not simple: receiving = ';\n'.join([ 'var ' + s + ' = Module["' + s + '"] = asm.' + s for s in exported_implemented_functions + function_tables ]) else: receiving = 'var _main = Module["_main"] = asm;' # finalize if DEBUG: print >> sys.stderr, 'asm text sizes', map( len, funcs_js), len(asm_setup), len(asm_global_vars), len( asm_global_funcs), len(pre_tables), len( '\n'.join(function_tables_impls)), len( function_tables_defs.replace( '\n', '\n ')), len(exports), len( the_global), len(sending), len(receiving) funcs_js = [ ''' %s function asmPrintInt(x, y) { Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack); } function asmPrintFloat(x, y) { Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack); } // EMSCRIPTEN_START_ASM var asm = (function(global, env, buffer) { 'use asm'; var HEAP8 = new global.Int8Array(buffer); var HEAP16 = new global.Int16Array(buffer); var HEAP32 = new global.Int32Array(buffer); var HEAPU8 = new global.Uint8Array(buffer); var HEAPU16 = new global.Uint16Array(buffer); var HEAPU32 = new global.Uint32Array(buffer); var HEAPF32 = new global.Float32Array(buffer); var HEAPF64 = new global.Float64Array(buffer); ''' % (asm_setup, ) + '\n' + asm_global_vars + ''' var __THREW__ = 0; var undef = 0; var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0; ''' + ''.join([''' var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + ''' // EMSCRIPTEN_START_FUNCS function stackAlloc(size) { size = size|0; var ret = 0; ret = STACKTOP; STACKTOP = (STACKTOP + size)|0; STACKTOP = ((STACKTOP + 3)>>2)<<2; return ret|0; } function stackSave() { return STACKTOP|0; } function stackRestore(top) { top = top|0; STACKTOP = top; } function setThrew(threw) { threw = threw|0; __THREW__ = threw; } ''' + ''.join([ ''' function setTempRet%d(value) { value = value|0; tempRet%d = value; } ''' % (i, i) for i in range(10) ]) ] + funcs_js + [ ''' %s return %s; }) // EMSCRIPTEN_END_ASM (%s, %s, buffer); %s; Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) }; Runtime.stackSave = function() { return asm.stackSave() }; Runtime.stackRestore = function(top) { asm.stackRestore(top) }; ''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n '), exports, the_global, sending, receiving) ] # Set function table masks def function_table_maskize(js): masks = {} default = None for sig, table in last_forwarded_json['Functions'][ 'tables'].iteritems(): masks[sig] = str(table.count(',')) default = sig def fix(m): sig = m.groups(0)[0] return masks[sig] return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]] funcs_js = map(function_table_maskize, funcs_js) else: function_tables_defs = '\n'.join([ table for table in last_forwarded_json['Functions'] ['tables'].itervalues() ]) outfile.write(function_tables_defs) funcs_js = [''' // EMSCRIPTEN_START_FUNCS '''] + funcs_js + [''' // EMSCRIPTEN_END_FUNCS '''] for funcs_js_item in funcs_js: # do this loop carefully to save memory funcs_js_item = indexize(funcs_js_item) funcs_js_item = blockaddrsize(funcs_js_item) outfile.write(funcs_js_item) funcs_js = None outfile.write(indexize(post)) if DEBUG: print >> sys.stderr, ' emscript: phase 3 took %s seconds' % ( time.time() - t) outfile.close()