Example #1
0
 def run(self, args):
     return jsrun.run_js(self.filename,
                         engine=self.engine,
                         args=args,
                         stderr=PIPE,
                         full_output=True,
                         assert_returncode=None)
Example #2
0
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files)):
  ll = ''.join(funcs) + '\n' + meta
  funcs_file = temp_files.get('.func_%d.ll' % i).name
  open(funcs_file, 'w').write(ll)
  out = jsrun.run_js(
    compiler,
    engine=compiler_engine,
    args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
    stdout=subprocess.PIPE)
  tempfiles.try_delete(funcs_file)
  return out
Example #3
0
 def execute_js(engine):
     print('(run in %s)' % engine)
     try:
         js = jsrun.run_js(filename + '.js',
                           engine=shutil.NODE_JS,
                           check_timeout=True,
                           assert_returncode=None)
     except CalledProcessError:
         print('failed to run in primary')
         return False
     js = js.split('\n')[
         0] + '\n'  # remove any extra printed stuff (node workarounds)
     return correct1 == js or correct2 == js
    def test_d8_path(self):
        """ Test that running JS commands works for node, d8, and jsc and is not path dependent """
        # Fake some JS engines
        restore_and_set_up()

        sample_script = path_from_root('tests', 'print_args.js')

        # Note that the path contains 'd8'.
        test_path = path_from_root('tests', 'fake', 'abcd8765')
        if not os.path.exists(test_path):
            os.makedirs(test_path)

        with env_modify({'EM_IGNORE_SANITY': '1'}):
            jsengines = [('d8', V8_ENGINE), ('d8_g', V8_ENGINE),
                         ('js', SPIDERMONKEY_ENGINE), ('node', NODE_JS),
                         ('nodejs', NODE_JS)]
            for filename, engine in jsengines:
                if type(engine) is list:
                    engine = engine[0]
                if engine == '':
                    print('WARNING: Not testing engine %s, not configured.' %
                          (filename))
                    continue

                print(filename, engine)

                test_engine_path = os.path.join(test_path, filename)
                f = open(test_engine_path, 'w')
                f.write('#!/bin/sh\n')
                f.write('%s $@\n' % (engine))
                f.close()
                os.chmod(test_engine_path,
                         stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)

                try:
                    out = jsrun.run_js(sample_script,
                                       engine=test_engine_path,
                                       args=['--foo'],
                                       full_output=True,
                                       assert_returncode=0,
                                       skip_check=True)
                except Exception as e:
                    if 'd8' in filename:
                        assert False, 'Your d8 version does not correctly parse command-line arguments, please upgrade or delete from ~/.emscripten config file: %s' % (
                            e)
                    else:
                        assert False, 'Error running script command: %s' % (e)

                self.assertEqual('0: --foo', out.strip())
Example #5
0
def process_funcs((i, funcs_file, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, DEBUG)):
  try:
    #print >> sys.stderr, 'running', str([settings_file, funcs_file, 'funcs', forwarded_file] + libraries).replace("'/", "'") # can use this in src/compiler_funcs.html arguments,
    #                                                                                                                         # just copy temp dir to under this one
    out = jsrun.run_js(
      compiler,
      engine=compiler_engine,
      args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
      stdout=subprocess.PIPE,
      stderr=STDERR_FILE,
      cwd=path_from_root('src'))
  except KeyboardInterrupt:
    # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt
    raise Exception()
  if DEBUG: logging.debug('.')
  return out
Example #6
0
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)):
  funcs_file = temp_files.get('.func_%d.ll' % i).name
  f = open(funcs_file, 'w')
  f.write(funcs)
  funcs = None
  f.write('\n')
  f.write(meta)
  f.close()
  out = jsrun.run_js(
    compiler,
    engine=compiler_engine,
    args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
    stdout=subprocess.PIPE,
    cwd=path_from_root('src'))
  tempfiles.try_delete(funcs_file)
  if DEBUG: print >> sys.stderr, '.'
  return out
Example #7
0
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)):
  funcs_file = temp_files.get('.func_%d.ll' % i).name
  f = open(funcs_file, 'w')
  f.write(funcs)
  funcs = None
  f.write('\n')
  f.write(meta)
  f.close()
  out = jsrun.run_js(
    compiler,
    engine=compiler_engine,
    args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
    stdout=subprocess.PIPE,
    cwd=path_from_root('src'))
  tempfiles.try_delete(funcs_file)
  if DEBUG: print >> sys.stderr, '.'
  return out
Example #8
0
def process_funcs((i, funcs_file, meta, settings_file, compiler,
                   forwarded_file, libraries, compiler_engine, DEBUG)):
    try:
        #print >> sys.stderr, 'running', str([settings_file, funcs_file, 'funcs', forwarded_file] + libraries).replace("'/", "'") # can use this in src/compiler_funcs.html arguments,
        #                                                                                                                         # just copy temp dir to under this one
        out = jsrun.run_js(
            compiler,
            engine=compiler_engine,
            args=[settings_file, funcs_file, 'funcs', forwarded_file] +
            libraries,
            stdout=subprocess.PIPE,
            stderr=STDERR_FILE,
            cwd=path_from_root('src'))
    except KeyboardInterrupt:
        # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt
        raise Exception()
    if DEBUG: logging.debug('.')
    return out
Example #9
0
  def test_d8_path(self):
    """ Test that running JS commands works for node, d8, and jsc and is not path dependent """
    # Fake some JS engines
    restore_and_set_up()

    sample_script = path_from_root('tests', 'print_args.js')

    # Note that the path contains 'd8'.
    test_path = path_from_root('tests', 'fake', 'abcd8765')
    if not os.path.exists(test_path):
      os.makedirs(test_path)

    with env_modify({'EM_IGNORE_SANITY': '1'}):
      jsengines = [('d8',     V8_ENGINE),
                   ('d8_g',   V8_ENGINE),
                   ('js',     SPIDERMONKEY_ENGINE),
                   ('node',   NODE_JS),
                   ('nodejs', NODE_JS)]
      for filename, engine in jsengines:
        if type(engine) is list:
          engine = engine[0]
        if engine == '':
            print('WARNING: Not testing engine %s, not configured.' % (filename))
            continue

        print(filename, engine)

        test_engine_path = os.path.join(test_path, filename)
        f = open(test_engine_path, 'w')
        f.write('#!/bin/sh\n')
        f.write('%s $@\n' % (engine))
        f.close()
        os.chmod(test_engine_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)

        try:
          out = jsrun.run_js(sample_script, engine=test_engine_path, args=['--foo'], full_output=True, assert_returncode=0, skip_check=True)
        except Exception as e:
          if 'd8' in filename:
            assert False, 'Your d8 version does not correctly parse command-line arguments, please upgrade or delete from ~/.emscripten config file: %s' % (e)
          else:
            assert False, 'Error running script command: %s' % (e)

        self.assertEqual('0: --foo', out.strip())
Example #10
0
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)):
  try:
    funcs_file = temp_files.get('.func_%d.ll' % i).name
    f = open(funcs_file, 'w')
    f.write(funcs)
    funcs = None
    f.write('\n')
    f.write(meta)
    f.close()
    out = jsrun.run_js(
      compiler,
      engine=compiler_engine,
      args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
      stdout=subprocess.PIPE,
      cwd=path_from_root('src'))
  except KeyboardInterrupt:
    # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt
    raise Exception()
  finally:
    tempfiles.try_delete(funcs_file)
  if DEBUG: print >> sys.stderr, '.'
  return out
Example #11
0
def process_funcs((i, funcs, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)):
  try:
    funcs_file = temp_files.get('.func_%d.ll' % i).name
    f = open(funcs_file, 'w')
    f.write(funcs)
    funcs = None
    f.write('\n')
    f.write(meta)
    f.close()
    out = jsrun.run_js(
      compiler,
      engine=compiler_engine,
      args=[settings_file, funcs_file, 'funcs', forwarded_file] + libraries,
      stdout=subprocess.PIPE,
      cwd=path_from_root('src'))
  except KeyboardInterrupt:
    # Python 2.7 seems to lock up when a child process throws KeyboardInterrupt
    raise Exception()
  finally:
    tempfiles.try_delete(funcs_file)
  if DEBUG: print >> sys.stderr, '.'
  return out
Example #12
0
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
             jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None):
  """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible

  Args:
    infile: The path to the input LLVM assembly file.
    settings: JSON-formatted settings that override the values
      defined in src/settings.js.
    outfile: The file where the output is written.
  """

  compiler = path_from_root('src', 'compiler.js')

  # Parallelization: We run 3 phases:
  #   1 aka 'pre'  : Process types and metadata and so forth, and generate the preamble.
  #   2 aka 'funcs': Process functions. We can parallelize this, working on each function independently.
  #   3 aka 'post' : Process globals, generate postamble and finishing touches.

  if DEBUG: print >> sys.stderr, 'emscript: ll=>js'

  if jcache: jcache.ensure()

  # Pre-scan ll and alter settings as necessary
  if DEBUG: t = time.time()
  ll = open(infile).read()
  scan(ll, settings)
  total_ll_size = len(ll)
  ll = None # allow collection
  if DEBUG: print >> sys.stderr, '  emscript: scan took %s seconds' % (time.time() - t)

  # Split input into the relevant parts for each phase
  pre = []
  funcs = [] # split up functions here, for parallelism later
  meta = [] # needed by each function XXX

  if DEBUG: t = time.time()
  in_func = False
  ll_lines = open(infile).readlines()
  curr_func = None
  for line in ll_lines:
    if in_func:
      curr_func.append(line)
      if line.startswith('}'):
        in_func = False
        funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier
        # pre needs to know about all implemented functions, even for non-pre func
        pre.append(curr_func[0])
        pre.append(line)
        curr_func = None
    else:
      if line.startswith(';'): continue
      if line.startswith('define '):
        in_func = True
        curr_func = [line]
      elif line.find(' = type { ') > 0:
        pre.append(line) # type
      elif line.startswith('!'):
        if line.startswith('!llvm.module'): continue # we can ignore that
        meta.append(line) # metadata
      else:
        pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there
  ll_lines = None
  meta = ''.join(meta)
  if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down'
  if DEBUG: print >> sys.stderr, '  emscript: split took %s seconds' % (time.time() - t)

  #if DEBUG:
  #  print >> sys.stderr, '========= pre ================\n'
  #  print >> sys.stderr, ''.join(pre)
  #  print >> sys.stderr, '========== funcs ===============\n'
  #  for func in funcs:
  #    print >> sys.stderr, '\n// ===\n\n', ''.join(func)
  #  print >> sys.stderr, '=========================\n'

  # Save settings to a file to work around v8 issue 1579
  settings_file = temp_files.get('.txt').name
  def save_settings():
    global settings_text
    settings_text = json.dumps(settings, sort_keys=True)
    s = open(settings_file, 'w')
    s.write(settings_text)
    s.close()
  save_settings()

  # Phase 1 - pre
  if DEBUG: t = time.time()
  pre_file = temp_files.get('.pre.ll').name
  pre_input = ''.join(pre) + '\n' + meta
  out = None
  if jcache:
    keys = [pre_input, settings_text, ','.join(libraries)]
    shortkey = jcache.get_shortkey(keys)
    if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey

    out = jcache.get(shortkey, keys)

    if DEBUG_CACHE and not out:
      dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey)
      dfp = open(dfpath, 'w')
      dfp.write(pre_input);
      dfp.write("\n\n========================== settings_text\n\n");
      dfp.write(settings_text);
      dfp.write("\n\n========================== libraries\n\n");
      dfp.write("\n".join(libraries))
      dfp.close()
      print >>sys.stderr, '  cache miss, key data dumped to %s' % dfpath

    if out and DEBUG: print >> sys.stderr, '  loading pre from jcache'
  if not out:
    open(pre_file, 'w').write(pre_input)
    out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE,
                       cwd=path_from_root('src'))
    assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?'
    if jcache:
      if DEBUG: print >> sys.stderr, '  saving pre to jcache'
      jcache.set(shortkey, keys, out)
  pre, forwarded_data = out.split('//FORWARDED_DATA:')
  forwarded_file = temp_files.get('.json').name
  open(forwarded_file, 'w').write(forwarded_data)
  if DEBUG: print >> sys.stderr, '  emscript: phase 1 took %s seconds' % (time.time() - t)

  # Phase 2 - func

  cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
  assert cores >= 1
  if cores > 1:
    intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
    chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks)
    chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant)
    chunk_size = min(MAX_CHUNK_SIZE, chunk_size)
  else:
    chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size

  if DEBUG: t = time.time()
  forwarded_json = json.loads(forwarded_data)
  indexed_functions = set()
  if settings.get('ASM_JS'):
    settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS']
    save_settings()

  chunks = cache_module.chunkify(
    funcs, chunk_size,
    jcache.get_cachename('emscript_files') if jcache else None)

  funcs = None

  if jcache:
    # load chunks from cache where we can # TODO: ignore small chunks
    cached_outputs = []
    def load_from_cache(chunk):
      keys = [settings_text, forwarded_data, chunk]
      shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code
      out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?)
      if out:
        cached_outputs.append(out)
        return False
      return True
    chunks = filter(load_from_cache, chunks)
    if len(cached_outputs) > 0:
      if out and DEBUG: print >> sys.stderr, '  loading %d funcchunks from jcache' % len(cached_outputs)
    else:
      cached_outputs = []

  # TODO: minimize size of forwarded data from funcs to what we actually need

  if len(chunks) > 0:
    if cores == 1 and total_ll_size < MAX_CHUNK_SIZE:
      assert len(chunks) == 1, 'no point in splitting up without multiple cores'

    if DEBUG: print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))

    commands = [
      (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)
      for i, chunk in enumerate(chunks)
    ]

    if len(chunks) > 1:
      pool = multiprocessing.Pool(processes=cores)
      outputs = pool.map(process_funcs, commands, chunksize=1)
    elif len(chunks) == 1:
      outputs = [process_funcs(commands[0])]

    commands = None

  else:
    outputs = []

  if jcache:
    # save chunks to cache
    for i in range(len(chunks)):
      chunk = chunks[i]
      keys = [settings_text, forwarded_data, chunk]
      shortkey = jcache.get_shortkey(keys)
      jcache.set(shortkey, keys, outputs[i])
    if out and DEBUG and len(chunks) > 0: print >> sys.stderr, '  saving %d funcchunks to jcache' % len(chunks)

  chunks = None

  if jcache: outputs += cached_outputs # TODO: preserve order

  outputs = [output.split('//FORWARDED_DATA:') for output in outputs]
  for output in outputs:
    assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:]

  if DEBUG: print >> sys.stderr, '  emscript: phase 2 took %s seconds' % (time.time() - t)
  if DEBUG: t = time.time()

  # merge forwarded data
  if settings.get('ASM_JS'):
    all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise
    for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented
      all_exported_functions.add('_' + additional_export)
    exported_implemented_functions = set()
  for func_js, curr_forwarded_data in outputs:
    curr_forwarded_json = json.loads(curr_forwarded_data)
    forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed']
    for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems():
      forwarded_json['Functions']['blockAddresses'][key] = value
    for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys():
      indexed_functions.add(key)
    if settings.get('ASM_JS'):
      export_bindings = settings['EXPORT_BINDINGS']
      for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys():
        if key in all_exported_functions or (export_bindings and key.startswith('_emscripten_bind')):
          exported_implemented_functions.add(key)
    for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems():
      forwarded_json['Functions']['unimplementedFunctions'][key] = value

  if settings.get('ASM_JS'):
    parts = pre.split('// ASM_LIBRARY FUNCTIONS\n')
    if len(parts) > 1:
      pre = parts[0]
      outputs.append([parts[1]])
  funcs_js = [output[0] for output in outputs]

  outputs = None
  if DEBUG: print >> sys.stderr, '  emscript: phase 2b took %s seconds' % (time.time() - t)
  if DEBUG: t = time.time()

  # calculations on merged forwarded data
  forwarded_json['Functions']['indexedFunctions'] = {}
  i = 2
  for indexed in indexed_functions:
    #print >> sys.stderr, 'indaxx', indexed, i
    forwarded_json['Functions']['indexedFunctions'][indexed] = i # make sure not to modify this python object later - we use it in indexize
    i += 2
  forwarded_json['Functions']['nextIndex'] = i

  def split_32(x):
    x = int(x)
    return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255)

  indexing = forwarded_json['Functions']['indexedFunctions']
  def indexize(js):
    # In the global initial allocation, we need to split up into Uint8 format
    ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js)
    return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret)

  blockaddrs = forwarded_json['Functions']['blockAddresses']
  def blockaddrsize(js):
    ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js)
    return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret)

  #if DEBUG: outfile.write('// pre\n')
  outfile.write(blockaddrsize(indexize(pre)))
  pre = None

  #if DEBUG: outfile.write('// funcs\n')

  # forward
  forwarded_data = json.dumps(forwarded_json)
  forwarded_file = temp_files.get('.2.json').name
  open(forwarded_file, 'w').write(indexize(forwarded_data))
  if DEBUG: print >> sys.stderr, '  emscript: phase 2c took %s seconds' % (time.time() - t)

  # Phase 3 - post
  if DEBUG: t = time.time()
  post_file = temp_files.get('.post.ll').name
  open(post_file, 'w').write('\n') # no input, just processing of forwarded data
  out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE,
                     cwd=path_from_root('src'))
  post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data?
  last_forwarded_json = json.loads(last_forwarded_data)

  if settings.get('ASM_JS'):
    post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n')
    post = post_rest

    # Move preAsms to their right place
    def move_preasm(m):
      contents = m.groups(0)[0]
      outfile.write(contents + '\n')
      return ''
    post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs)

    funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n']

    simple = os.environ.get('EMCC_SIMPLE_ASM')
    class Counter:
      i = 0
    pre_tables = last_forwarded_json['Functions']['tables']['pre']
    del last_forwarded_json['Functions']['tables']['pre']

    # Find function table calls without function tables generated for them
    for funcs_js_item in funcs_js:
      for use in set(re.findall(r'{{{ FTM_[\w\d_$]+ }}}', funcs_js_item)):
        sig = use[8:len(use)-4]
        if sig not in last_forwarded_json['Functions']['tables']:
          if DEBUG: print >> sys.stderr, 'add empty function table', sig
          last_forwarded_json['Functions']['tables'][sig] = 'var FUNCTION_TABLE_' + sig + ' = [0,0];\n'

    def make_table(sig, raw):
      i = Counter.i
      Counter.i += 1
      bad = 'b' + str(i)
      params = ','.join(['p%d' % p for p in range(len(sig)-1)])
      coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';'
      ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else ''))
      return ('function %s(%s) { %s abort(%d); %s }' % (bad, params, coercions, i, ret), raw.replace('[0,', '[' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0,', ',' + bad + ',').replace(',0]', ',' + bad + ']').replace(',0]', ',' + bad + ']').replace(',0\n', ',' + bad + '\n'))
    infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()]
    function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos])

    asm_setup = ''
    maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']]
    fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array']
    math_envs = ['Math.min'] # TODO: move min to maths
    asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs])
    basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat', 'copyTempDouble', 'copyTempFloat'] + [m.replace('.', '_') for m in math_envs]
    if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR']
    if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8']
    basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT']
    basic_float_vars = ['NaN', 'Infinity']

    if forwarded_json['Types']['preciseI64MathUsed'] or \
       forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \
       forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'):
      basic_vars += ['cttz_i8', 'ctlz_i8']

    asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)]
    # function tables
    def asm_coerce(value, sig):
      if sig == 'v': return value
      return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '')
        
    function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']]
    function_tables_impls = []
    for sig in last_forwarded_json['Functions']['tables'].iterkeys():
      args = ','.join(['a' + str(i) for i in range(1, len(sig))])
      arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))])
      coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))])
      ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0])
      function_tables_impls.append('''
  function dynCall_%s(index%s%s) {
    index = index|0;
    %s
    %s;
  }
''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret))
      args = ','.join(['a' + str(i) for i in range(1, len(sig))])
      args = 'index' + (',' if args else '') + args
      asm_setup += '''
function invoke_%s(%s) {
  try {
    %sModule.dynCall_%s(%s);
  } catch(e) {
    asm.setThrew(1);
  }
}
''' % (sig, args, 'return ' if sig[0] != 'v' else '', sig, args)
      basic_funcs.append('invoke_%s' % sig)

    # calculate exports
    exported_implemented_functions = list(exported_implemented_functions)
    exports = []
    if not simple:
      for export in exported_implemented_functions + asm_runtime_funcs + function_tables:
        exports.append("%s: %s" % (export, export))
      exports = '{ ' + ', '.join(exports) + ' }'
    else:
      exports = '_main'
    # calculate globals
    try:
      del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable
    except:
      pass
    # If no named globals, only need externals
    global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values()))
    global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2]
    def math_fix(g):
      return g if not g.startswith('Math_') else g.split('_')[1];
    asm_global_funcs = ''.join(['  var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \
                       ''.join(['  var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs])
    asm_global_vars = ''.join(['  var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \
                      ''.join(['  var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars])
    # sent data
    the_global = '{ ' + ', '.join([math_fix(s) + ': ' + s for s in fundamentals]) + ' }'
    sending = '{ ' + ', '.join([math_fix(s) + ': ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }'
    # received
    if not simple:
      receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm.' + s for s in exported_implemented_functions + function_tables])
    else:
      receiving = 'var _main = Module["_main"] = asm;'

    # finalize

    if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n  ')), len(exports), len(the_global), len(sending), len(receiving)

    funcs_js = ['''
%s
function asmPrintInt(x, y) {
  Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack);
}
function asmPrintFloat(x, y) {
  Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack);
}
// EMSCRIPTEN_START_ASM
var asm = (function(global, env, buffer) {
  'use asm';
  var HEAP8 = new global.Int8Array(buffer);
  var HEAP16 = new global.Int16Array(buffer);
  var HEAP32 = new global.Int32Array(buffer);
  var HEAPU8 = new global.Uint8Array(buffer);
  var HEAPU16 = new global.Uint16Array(buffer);
  var HEAPU32 = new global.Uint32Array(buffer);
  var HEAPF32 = new global.Float32Array(buffer);
  var HEAPF64 = new global.Float64Array(buffer);
''' % (asm_setup,) + '\n' + asm_global_vars + '''
  var __THREW__ = 0;
  var undef = 0;
  var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0;
''' + ''.join(['''
  var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + '''
// EMSCRIPTEN_START_FUNCS
  function stackAlloc(size) {
    size = size|0;
    var ret = 0;
    ret = STACKTOP;
    STACKTOP = (STACKTOP + size)|0;
    STACKTOP = ((STACKTOP + 3)>>2)<<2;
    return ret|0;
  }
  function stackSave() {
    return STACKTOP|0;
  }
  function stackRestore(top) {
    top = top|0;
    STACKTOP = top;
  }
  function setThrew(threw) {
    threw = threw|0;
    __THREW__ = threw;
  }
''' + ''.join(['''
  function setTempRet%d(value) {
    value = value|0;
    tempRet%d = value;
  }
''' % (i, i) for i in range(10)])] + funcs_js + ['''
  %s

  return %s;
})
// EMSCRIPTEN_END_ASM
(%s, %s, buffer);
%s;
Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) };
Runtime.stackSave = function() { return asm.stackSave() };
Runtime.stackRestore = function(top) { asm.stackRestore(top) };
''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n  '), exports, the_global, sending, receiving)]

    # Set function table masks
    def function_table_maskize(js):
      masks = {}
      default = None
      for sig, table in last_forwarded_json['Functions']['tables'].iteritems():
        masks[sig] = str(table.count(','))
        default = sig
      def fix(m):
        sig = m.groups(0)[0]
        return masks[sig]
      return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]]
    funcs_js = map(function_table_maskize, funcs_js)
  else:
    function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()])
    outfile.write(function_tables_defs)
    funcs_js = ['''
// EMSCRIPTEN_START_FUNCS
'''] + funcs_js + ['''
// EMSCRIPTEN_END_FUNCS
''']

  for funcs_js_item in funcs_js: # do this loop carefully to save memory
    funcs_js_item = indexize(funcs_js_item)
    funcs_js_item = blockaddrsize(funcs_js_item)
    outfile.write(funcs_js_item)
  funcs_js = None

  outfile.write(indexize(post))
  if DEBUG: print >> sys.stderr, '  emscript: phase 3 took %s seconds' % (time.time() - t)

  outfile.close()
Example #13
0
def emscript(infile,
             settings,
             outfile,
             libraries=[],
             compiler_engine=None,
             jcache=None,
             temp_files=None,
             DEBUG=None,
             DEBUG_CACHE=None):
    """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible

  Args:
    infile: The path to the input LLVM assembly file.
    settings: JSON-formatted settings that override the values
      defined in src/settings.js.
    outfile: The file where the output is written.
  """

    compiler = path_from_root('src', 'compiler.js')

    # Parallelization: We run 3 phases:
    #   1 aka 'pre'  : Process types and metadata and so forth, and generate the preamble.
    #   2 aka 'funcs': Process functions. We can parallelize this, working on each function independently.
    #   3 aka 'post' : Process globals, generate postamble and finishing touches.

    if DEBUG: print >> sys.stderr, 'emscript: ll=>js'

    if jcache: jcache.ensure()

    # Pre-scan ll and alter settings as necessary
    if DEBUG: t = time.time()
    ll = open(infile).read()
    scan(ll, settings)
    total_ll_size = len(ll)
    ll = None  # allow collection
    if DEBUG:
        print >> sys.stderr, '  emscript: scan took %s seconds' % (
            time.time() - t)

    # Split input into the relevant parts for each phase
    pre = []
    funcs = []  # split up functions here, for parallelism later
    func_idents = []
    meta = []  # needed by each function XXX

    if DEBUG: t = time.time()
    in_func = False
    ll_lines = open(infile).readlines()
    for line in ll_lines:
        if in_func:
            funcs[-1][1].append(line)
            if line.startswith('}'):
                in_func = False
                funcs[-1] = (funcs[-1][0], ''.join(funcs[-1][1]))
                pre.append(
                    line
                )  # pre needs it to, so we know about all implemented functions
        else:
            if line.startswith(';'): continue
            if line.startswith('define '):
                in_func = True
                funcs.append(
                    (line, [line]))  # use the entire line as the identifier
                pre.append(
                    line
                )  # pre needs it to, so we know about all implemented functions
            elif line.find(' = type { ') > 0:
                pre.append(line)  # type
            elif line.startswith('!'):
                if line.startswith('!llvm.module'):
                    continue  # we can ignore that
                meta.append(line)  # metadata
            else:
                pre.append(
                    line
                )  # pre needs it so we know about globals in pre and funcs. So emit globals there
    ll_lines = None
    meta = ''.join(meta)
    if DEBUG and len(meta) > 1024 * 1024:
        print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down'
    if DEBUG:
        print >> sys.stderr, '  emscript: split took %s seconds' % (
            time.time() - t)

    #if DEBUG:
    #  print >> sys.stderr, '========= pre ================\n'
    #  print >> sys.stderr, ''.join(pre)
    #  print >> sys.stderr, '========== funcs ===============\n'
    #  for func in funcs:
    #    print >> sys.stderr, '\n// ===\n\n', ''.join(func)
    #  print >> sys.stderr, '=========================\n'

    # Save settings to a file to work around v8 issue 1579
    settings_file = temp_files.get('.txt').name

    def save_settings():
        global settings_text
        settings_text = json.dumps(settings, sort_keys=True)
        s = open(settings_file, 'w')
        s.write(settings_text)
        s.close()

    save_settings()

    # Phase 1 - pre
    if DEBUG: t = time.time()
    pre_file = temp_files.get('.pre.ll').name
    pre_input = ''.join(pre) + '\n' + meta
    out = None
    if jcache:
        keys = [pre_input, settings_text, ','.join(libraries)]
        shortkey = jcache.get_shortkey(keys)
        if DEBUG_CACHE: print >> sys.stderr, 'shortkey', shortkey

        out = jcache.get(shortkey, keys)

        if DEBUG_CACHE and not out:
            dfpath = os.path.join(get_configuration().TEMP_DIR,
                                  "ems_" + shortkey)
            dfp = open(dfpath, 'w')
            dfp.write(pre_input)
            dfp.write("\n\n========================== settings_text\n\n")
            dfp.write(settings_text)
            dfp.write("\n\n========================== libraries\n\n")
            dfp.write("\n".join(libraries))
            dfp.close()
            print >> sys.stderr, '  cache miss, key data dumped to %s' % dfpath

        if out and DEBUG: print >> sys.stderr, '  loading pre from jcache'
    if not out:
        open(pre_file, 'w').write(pre_input)
        out = jsrun.run_js(compiler,
                           compiler_engine,
                           [settings_file, pre_file, 'pre'] + libraries,
                           stdout=subprocess.PIPE,
                           cwd=path_from_root('src'))
        assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?'
        if jcache:
            if DEBUG: print >> sys.stderr, '  saving pre to jcache'
            jcache.set(shortkey, keys, out)
    pre, forwarded_data = out.split('//FORWARDED_DATA:')
    forwarded_file = temp_files.get('.json').name
    open(forwarded_file, 'w').write(forwarded_data)
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 1 took %s seconds' % (
            time.time() - t)

    # Phase 2 - func

    cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
    assert cores >= 1
    if cores > 1:
        intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
        chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks)
        chunk_size += 3 * len(meta) + len(
            forwarded_data
        ) / 3  # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant)
        chunk_size = min(MAX_CHUNK_SIZE, chunk_size)
    else:
        chunk_size = MAX_CHUNK_SIZE  # if 1 core, just use the max chunk size

    if DEBUG: t = time.time()
    forwarded_json = json.loads(forwarded_data)
    indexed_functions = set()
    if settings.get('ASM_JS'):
        settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS']
        save_settings()

    chunks = cache_module.chunkify(
        funcs, chunk_size,
        jcache.get_cachename('emscript_files') if jcache else None)

    funcs = None

    if jcache:
        # load chunks from cache where we can # TODO: ignore small chunks
        cached_outputs = []

        def load_from_cache(chunk):
            keys = [settings_text, forwarded_data, chunk]
            shortkey = jcache.get_shortkey(
                keys)  # TODO: share shortkeys with later code
            out = jcache.get(shortkey,
                             keys)  # this is relatively expensive (pickling?)
            if out:
                cached_outputs.append(out)
                return False
            return True

        chunks = filter(load_from_cache, chunks)
        if len(cached_outputs) > 0:
            if out and DEBUG:
                print >> sys.stderr, '  loading %d funcchunks from jcache' % len(
                    cached_outputs)
        else:
            cached_outputs = []

    # TODO: minimize size of forwarded data from funcs to what we actually need

    if cores == 1 and total_ll_size < MAX_CHUNK_SIZE:
        assert len(
            chunks) == 1, 'no point in splitting up without multiple cores'

    if len(chunks) > 0:
        if DEBUG:
            print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (
                len(chunks),
                ('using %d cores' % cores) if len(chunks) > 1 else '',
                chunk_size / (1024 * 1024.), len(meta) /
                (1024 * 1024.), len(forwarded_data) /
                (1024 * 1024.), total_ll_size / (1024 * 1024.))

        commands = [(i, chunk, meta, settings_file, compiler, forwarded_file,
                     libraries, compiler_engine, temp_files, DEBUG)
                    for i, chunk in enumerate(chunks)]

        if len(chunks) > 1:
            pool = multiprocessing.Pool(processes=cores)
            outputs = pool.map(process_funcs, commands, chunksize=1)
        elif len(chunks) == 1:
            outputs = [process_funcs(commands[0])]

        commands = None

    else:
        outputs = []

    if jcache:
        # save chunks to cache
        for i in range(len(chunks)):
            chunk = chunks[i]
            keys = [settings_text, forwarded_data, chunk]
            shortkey = jcache.get_shortkey(keys)
            jcache.set(shortkey, keys, outputs[i])
        if out and DEBUG and len(chunks) > 0:
            print >> sys.stderr, '  saving %d funcchunks to jcache' % len(
                chunks)

    chunks = None

    if jcache: outputs += cached_outputs  # TODO: preserve order

    outputs = [output.split('//FORWARDED_DATA:') for output in outputs]
    for output in outputs:
        assert len(
            output
        ) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[
            0][-3000:]

    if DEBUG:
        print >> sys.stderr, '  emscript: phase 2 took %s seconds' % (
            time.time() - t)
    if DEBUG: t = time.time()

    # merge forwarded data
    if settings.get('ASM_JS'):
        all_exported_functions = set(
            settings['EXPORTED_FUNCTIONS'])  # both asm.js and otherwise
        for additional_export in settings[
                'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']:  # additional functions to export from asm, if they are implemented
            all_exported_functions.add('_' + additional_export)
        exported_implemented_functions = set()
    for func_js, curr_forwarded_data in outputs:
        curr_forwarded_json = json.loads(curr_forwarded_data)
        forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json[
            'Types']['preciseI64MathUsed'] or curr_forwarded_json['Types'][
                'preciseI64MathUsed']
        for key, value in curr_forwarded_json['Functions'][
                'blockAddresses'].iteritems():
            forwarded_json['Functions']['blockAddresses'][key] = value
        for key in curr_forwarded_json['Functions'][
                'indexedFunctions'].iterkeys():
            indexed_functions.add(key)
        if settings.get('ASM_JS'):
            export_bindings = settings['EXPORT_BINDINGS']
            for key in curr_forwarded_json['Functions'][
                    'implementedFunctions'].iterkeys():
                if key in all_exported_functions or (
                        export_bindings
                        and key.startswith('_emscripten_bind')):
                    exported_implemented_functions.add(key)
        for key, value in curr_forwarded_json['Functions'][
                'unimplementedFunctions'].iteritems():
            forwarded_json['Functions']['unimplementedFunctions'][key] = value

    if settings.get('ASM_JS'):
        parts = pre.split('// ASM_LIBRARY FUNCTIONS\n')
        if len(parts) > 1:
            pre = parts[0]
            outputs.append([parts[1]])
    funcs_js = [output[0] for output in outputs]

    outputs = None
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 2b took %s seconds' % (
            time.time() - t)
    if DEBUG: t = time.time()

    # calculations on merged forwarded data
    forwarded_json['Functions']['indexedFunctions'] = {}
    i = 2
    for indexed in indexed_functions:
        #print >> sys.stderr, 'indaxx', indexed, i
        forwarded_json['Functions']['indexedFunctions'][
            indexed] = i  # make sure not to modify this python object later - we use it in indexize
        i += 2
    forwarded_json['Functions']['nextIndex'] = i

    indexing = forwarded_json['Functions']['indexedFunctions']

    def indexize(js):
        return re.sub(r"'{{ FI_([\w\d_$]+) }}'",
                      lambda m: str(indexing.get(m.groups(0)[0]) or 0), js)

    blockaddrs = forwarded_json['Functions']['blockAddresses']

    def blockaddrsize(js):
        return re.sub(
            r'{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}',
            lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js)

    #if DEBUG: outfile.write('// pre\n')
    outfile.write(blockaddrsize(indexize(pre)))
    pre = None

    #if DEBUG: outfile.write('// funcs\n')

    # forward
    forwarded_data = json.dumps(forwarded_json)
    forwarded_file = temp_files.get('.2.json').name
    open(forwarded_file, 'w').write(indexize(forwarded_data))
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 2c took %s seconds' % (
            time.time() - t)

    # Phase 3 - post
    if DEBUG: t = time.time()
    post_file = temp_files.get('.post.ll').name
    open(post_file,
         'w').write('\n')  # no input, just processing of forwarded data
    out = jsrun.run_js(compiler,
                       compiler_engine,
                       [settings_file, post_file, 'post', forwarded_file] +
                       libraries,
                       stdout=subprocess.PIPE,
                       cwd=path_from_root('src'))
    post, last_forwarded_data = out.split(
        '//FORWARDED_DATA:'
    )  # if this fails, perhaps the process failed prior to printing forwarded data?
    last_forwarded_json = json.loads(last_forwarded_data)

    if settings.get('ASM_JS'):
        post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n')
        post = post_rest
        funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n']

        simple = os.environ.get('EMCC_SIMPLE_ASM')

        class Counter:
            i = 0

        pre_tables = last_forwarded_json['Functions']['tables']['pre']
        del last_forwarded_json['Functions']['tables']['pre']

        # Find function table calls without function tables generated for them
        for funcs_js_item in funcs_js:
            for use in set(re.findall(r'{{{ FTM_[\w\d_$]+ }}}',
                                      funcs_js_item)):
                sig = use[8:len(use) - 4]
                if sig not in last_forwarded_json['Functions']['tables']:
                    if DEBUG:
                        print >> sys.stderr, 'add empty function table', sig
                    last_forwarded_json['Functions']['tables'][
                        sig] = 'var FUNCTION_TABLE_' + sig + ' = [0,0];\n'

        def make_table(sig, raw):
            i = Counter.i
            Counter.i += 1
            bad = 'b' + str(i)
            params = ','.join(['p%d' % p for p in range(len(sig) - 1)])
            coercions = ';'.join([
                'p%d = %sp%d%s' % (p, '+' if sig[p + 1] != 'i' else '', p,
                                   '' if sig[p + 1] != 'i' else '|0')
                for p in range(len(sig) - 1)
            ]) + ';'
            ret = '' if sig[0] == 'v' else ('return %s0' %
                                            ('+' if sig[0] != 'i' else ''))
            return ('function %s(%s) { %s abort(%d); %s }' %
                    (bad, params, coercions, i, ret),
                    raw.replace('[0,', '[' + bad + ',').replace(
                        ',0,', ',' + bad +
                        ',').replace(',0,', ',' + bad + ',').replace(
                            ',0]',
                            ',' + bad + ']').replace(',0]',
                                                     ',' + bad + ']').replace(
                                                         ',0\n',
                                                         ',' + bad + '\n'))

        infos = [
            make_table(sig, raw) for sig, raw in
            last_forwarded_json['Functions']['tables'].iteritems()
        ]
        function_tables_defs = '\n'.join(
            [info[0]
             for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join(
                 [info[1] for info in infos])

        asm_setup = ''
        maths = [
            'Math.' + func for func in [
                'floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos',
                'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul'
            ]
        ]
        fundamentals = [
            'Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array',
            'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'
        ]
        math_envs = ['Math.min']  # TODO: move min to maths
        asm_setup += '\n'.join(
            ['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs])
        basic_funcs = [
            'abort', 'assert', 'asmPrintInt', 'asmPrintFloat',
            'copyTempDouble', 'copyTempFloat'
        ] + [m.replace('.', '_') for m in math_envs]
        if settings['SAFE_HEAP']:
            basic_funcs += [
                'SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'
            ]
        if settings['CHECK_HEAP_ALIGN']:
            basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8']
        basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT']
        basic_float_vars = ['NaN', 'Infinity']
        if forwarded_json['Types']['preciseI64MathUsed']:
            basic_funcs += [
                'i64Math_' + op
                for op in ['add', 'subtract', 'multiply', 'divide', 'modulo']
            ]
            asm_setup += '''
var i64Math_add = function(a, b, c, d) { i64Math.add(a, b, c, d) };
var i64Math_subtract = function(a, b, c, d) { i64Math.subtract(a, b, c, d) };
var i64Math_multiply = function(a, b, c, d) { i64Math.multiply(a, b, c, d) };
var i64Math_divide = function(a, b, c, d, e) { i64Math.divide(a, b, c, d, e) };
var i64Math_modulo = function(a, b, c, d, e) { i64Math.modulo(a, b, c, d, e) };
'''
        asm_runtime_funcs = [
            'stackAlloc', 'stackSave', 'stackRestore', 'setThrew'
        ] + ['setTempRet%d' % i for i in range(10)]

        # function tables
        def asm_coerce(value, sig):
            if sig == 'v': return value
            return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i'
                                                          else '')

        function_tables = [
            'dynCall_' + table
            for table in last_forwarded_json['Functions']['tables']
        ]
        function_tables_impls = []
        for sig in last_forwarded_json['Functions']['tables'].iterkeys():
            args = ','.join(['a' + str(i) for i in range(1, len(sig))])
            arg_coercions = ' '.join([
                'a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';'
                for i in range(1, len(sig))
            ])
            coerced_args = ','.join(
                [asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))])
            ret = ('return ' if sig[0] != 'v' else '') + asm_coerce(
                'FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' %
                (sig, sig, coerced_args), sig[0])
            function_tables_impls.append('''
  function dynCall_%s(index%s%s) {
    index = index|0;
    %s
    %s;
  }
''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret))

        # calculate exports
        exported_implemented_functions = list(exported_implemented_functions)
        exports = []
        if not simple:
            for export in exported_implemented_functions + asm_runtime_funcs + function_tables:
                exports.append("%s: %s" % (export, export))
            exports = '{ ' + ', '.join(exports) + ' }'
        else:
            exports = '_main'
        # calculate globals
        try:
            del forwarded_json['Variables']['globals'][
                '_llvm_global_ctors']  # not a true variable
        except:
            pass
        # If no named globals, only need externals
        global_vars = map(
            lambda g: g['name'],
            filter(
                lambda g: settings['NAMED_GLOBALS'] or g.get(
                    'external') or g.get('unIndexable'),
                forwarded_json['Variables']['globals'].values()))
        global_funcs = [
            '_' + x
            for x in forwarded_json['Functions']['libraryFunctions'].keys()
        ]

        def math_fix(g):
            return g if not g.startswith('Math_') else g.split('_')[1]
        asm_global_funcs = ''.join(['  var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \
                           ''.join(['  var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs])
        asm_global_vars = ''.join(['  var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \
                          ''.join(['  var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars])
        # sent data
        the_global = '{ ' + ', '.join(
            [math_fix(s) + ': ' + s for s in fundamentals]) + ' }'
        sending = '{ ' + ', '.join([
            math_fix(s) + ': ' + s for s in basic_funcs + global_funcs +
            basic_vars + basic_float_vars + global_vars
        ]) + ' }'
        # received
        if not simple:
            receiving = ';\n'.join([
                'var ' + s + ' = Module["' + s + '"] = asm.' + s
                for s in exported_implemented_functions + function_tables
            ])
        else:
            receiving = 'var _main = Module["_main"] = asm;'

        # finalize

        if DEBUG:
            print >> sys.stderr, 'asm text sizes', map(
                len, funcs_js), len(asm_setup), len(asm_global_vars), len(
                    asm_global_funcs), len(pre_tables), len(
                        '\n'.join(function_tables_impls)), len(
                            function_tables_defs.replace(
                                '\n', '\n  ')), len(exports), len(
                                    the_global), len(sending), len(receiving)

        funcs_js = [
            '''
%s
function asmPrintInt(x, y) {
  Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack);
}
function asmPrintFloat(x, y) {
  Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack);
}
// EMSCRIPTEN_START_ASM
var asm = (function(global, env, buffer) {
  'use asm';
  var HEAP8 = new global.Int8Array(buffer);
  var HEAP16 = new global.Int16Array(buffer);
  var HEAP32 = new global.Int32Array(buffer);
  var HEAPU8 = new global.Uint8Array(buffer);
  var HEAPU16 = new global.Uint16Array(buffer);
  var HEAPU32 = new global.Uint32Array(buffer);
  var HEAPF32 = new global.Float32Array(buffer);
  var HEAPF64 = new global.Float64Array(buffer);
''' % (asm_setup, ) + '\n' + asm_global_vars + '''
  var __THREW__ = 0;
  var undef = 0;
  var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0;
''' + ''.join(['''
  var tempRet%d = 0;''' % i
               for i in range(10)]) + '\n' + asm_global_funcs + '''
// EMSCRIPTEN_START_FUNCS
  function stackAlloc(size) {
    size = size|0;
    var ret = 0;
    ret = STACKTOP;
    STACKTOP = (STACKTOP + size)|0;
    STACKTOP = ((STACKTOP + 3)>>2)<<2;
    return ret|0;
  }
  function stackSave() {
    return STACKTOP|0;
  }
  function stackRestore(top) {
    top = top|0;
    STACKTOP = top;
  }
  function setThrew(threw) {
    threw = threw|0;
    __THREW__ = threw;
  }
''' + ''.join([
                   '''
  function setTempRet%d(value) {
    value = value|0;
    tempRet%d = value;
  }
''' % (i, i) for i in range(10)
               ])
        ] + funcs_js + [
            '''
  %s

  return %s;
})
// EMSCRIPTEN_END_ASM
(%s, %s, buffer);
%s;
Runtime.stackAlloc = function(size) { return asm.stackAlloc(size) };
Runtime.stackSave = function() { return asm.stackSave() };
Runtime.stackRestore = function(top) { asm.stackRestore(top) };
''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' +
        function_tables_defs.replace('\n', '\n  '), exports, the_global,
        sending, receiving)
        ]

        # Set function table masks
        def function_table_maskize(js):
            masks = {}
            default = None
            for sig, table in last_forwarded_json['Functions'][
                    'tables'].iteritems():
                masks[sig] = str(table.count(','))
                default = sig

            def fix(m):
                sig = m.groups(0)[0]
                return masks[sig]

            return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m),
                          js)  # masks[m.groups(0)[0]]

        funcs_js = map(function_table_maskize, funcs_js)
    else:
        function_tables_defs = '\n'.join([
            table for table in last_forwarded_json['Functions']
            ['tables'].itervalues()
        ])
        outfile.write(function_tables_defs)
        funcs_js = ['''
// EMSCRIPTEN_START_FUNCS
'''] + funcs_js + ['''
// EMSCRIPTEN_END_FUNCS
''']

    for funcs_js_item in funcs_js:  # do this loop carefully to save memory
        funcs_js_item = indexize(funcs_js_item)
        funcs_js_item = blockaddrsize(funcs_js_item)
        outfile.write(funcs_js_item)
    funcs_js = None

    outfile.write(indexize(post))
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 3 took %s seconds' % (
            time.time() - t)

    outfile.close()
Example #14
0
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
             jcache=None, temp_files=None, DEBUG=None, DEBUG_CACHE=None):
  """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible

  Args:
    infile: The path to the input LLVM assembly file.
    settings: JSON-formatted settings that override the values
      defined in src/settings.js.
    outfile: The file where the output is written.
  """

  compiler = path_from_root('src', 'compiler.js')

  # Parallelization: We run 3 phases:
  #   1 aka 'pre'  : Process types and metadata and so forth, and generate the preamble.
  #   2 aka 'funcs': Process functions. We can parallelize this, working on each function independently.
  #   3 aka 'post' : Process globals, generate postamble and finishing touches.

  if DEBUG: print >> sys.stderr, 'emscript: ll=>js'

  if jcache: jcache.ensure()

  # Pre-scan ll and alter settings as necessary
  if DEBUG: t = time.time()
  ll = open(infile).read()
  scan(ll, settings)
  total_ll_size = len(ll)
  ll = None # allow collection
  if DEBUG: print >> sys.stderr, '  emscript: scan took %s seconds' % (time.time() - t)

  # Split input into the relevant parts for each phase
  pre = []
  funcs = [] # split up functions here, for parallelism later
  meta = [] # needed by each function XXX

  if DEBUG: t = time.time()
  in_func = False
  ll_lines = open(infile).readlines()
  curr_func = None
  for line in ll_lines:
    if in_func:
      curr_func.append(line)
      if line.startswith('}'):
        in_func = False
        funcs.append((curr_func[0], ''.join(curr_func))) # use the entire line as the identifier
        # pre needs to know about all implemented functions, even for non-pre func
        pre.append(curr_func[0])
        pre.append(line)
        curr_func = None
    else:
      if line.startswith(';'): continue
      if line.startswith('define '):
        in_func = True
        curr_func = [line]
      elif line.find(' = type { ') > 0:
        pre.append(line) # type
      elif line.startswith('!'):
        if line.startswith('!llvm.module'): continue # we can ignore that
        meta.append(line) # metadata
      else:
        pre.append(line) # pre needs it so we know about globals in pre and funcs. So emit globals there
  ll_lines = None
  meta = ''.join(meta)
  if DEBUG and len(meta) > 1024*1024: print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down'
  if DEBUG: print >> sys.stderr, '  emscript: split took %s seconds' % (time.time() - t)

  if len(funcs) == 0:
    print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)'

  #if DEBUG:
  #  print >> sys.stderr, '========= pre ================\n'
  #  print >> sys.stderr, ''.join(pre)
  #  print >> sys.stderr, '========== funcs ===============\n'
  #  for func in funcs:
  #    print >> sys.stderr, '\n// ===\n\n', ''.join(func)
  #  print >> sys.stderr, '=========================\n'

  # Save settings to a file to work around v8 issue 1579
  settings_file = temp_files.get('.txt').name
  def save_settings():
    global settings_text
    settings_text = json.dumps(settings, sort_keys=True)
    s = open(settings_file, 'w')
    s.write(settings_text)
    s.close()
  save_settings()

  # Phase 1 - pre
  if DEBUG: t = time.time()
  pre_file = temp_files.get('.pre.ll').name
  pre_input = ''.join(pre) + '\n' + meta
  out = None
  if jcache:
    keys = [pre_input, settings_text, ','.join(libraries)]
    shortkey = jcache.get_shortkey(keys)
    if DEBUG_CACHE: print >>sys.stderr, 'shortkey', shortkey

    out = jcache.get(shortkey, keys)

    if DEBUG_CACHE and not out:
      dfpath = os.path.join(get_configuration().TEMP_DIR, "ems_" + shortkey)
      dfp = open(dfpath, 'w')
      dfp.write(pre_input)
      dfp.write("\n\n========================== settings_text\n\n")
      dfp.write(settings_text)
      dfp.write("\n\n========================== libraries\n\n")
      dfp.write("\n".join(libraries))
      dfp.close()
      print >>sys.stderr, '  cache miss, key data dumped to %s' % dfpath

    if out and DEBUG: print >> sys.stderr, '  loading pre from jcache'
  if not out:
    open(pre_file, 'w').write(pre_input)
    out = jsrun.run_js(compiler, compiler_engine, [settings_file, pre_file, 'pre'] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE,
                       cwd=path_from_root('src'))
    assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?'
    if jcache:
      if DEBUG: print >> sys.stderr, '  saving pre to jcache'
      jcache.set(shortkey, keys, out)
  pre, forwarded_data = out.split('//FORWARDED_DATA:')
  forwarded_file = temp_files.get('.json').name
  open(forwarded_file, 'w').write(forwarded_data)
  if DEBUG: print >> sys.stderr, '  emscript: phase 1 took %s seconds' % (time.time() - t)

  indexed_functions = set()
  forwarded_json = json.loads(forwarded_data)
  for key in forwarded_json['Functions']['indexedFunctions'].iterkeys():
    indexed_functions.add(key)

  # Phase 2 - func

  cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
  assert cores >= 1
  if cores > 1:
    intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
    chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks)
    chunk_size += 3*len(meta) + len(forwarded_data)/3 # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant)
    chunk_size = min(MAX_CHUNK_SIZE, chunk_size)
  else:
    chunk_size = MAX_CHUNK_SIZE # if 1 core, just use the max chunk size

  if DEBUG: t = time.time()
  if settings.get('ASM_JS'):
    settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS']
    save_settings()

  chunks = cache_module.chunkify(
    funcs, chunk_size,
    jcache.get_cachename('emscript_files') if jcache else None)

  funcs = None

  if jcache:
    # load chunks from cache where we can # TODO: ignore small chunks
    cached_outputs = []
    def load_from_cache(chunk):
      keys = [settings_text, forwarded_data, chunk]
      shortkey = jcache.get_shortkey(keys) # TODO: share shortkeys with later code
      out = jcache.get(shortkey, keys) # this is relatively expensive (pickling?)
      if out:
        cached_outputs.append(out)
        return False
      return True
    chunks = filter(load_from_cache, chunks)
    if len(cached_outputs) > 0:
      if out and DEBUG: print >> sys.stderr, '  loading %d funcchunks from jcache' % len(cached_outputs)
    else:
      cached_outputs = []

  # TODO: minimize size of forwarded data from funcs to what we actually need

  if len(chunks) > 0:
    if cores == 1 and total_ll_size < MAX_CHUNK_SIZE:
      assert len(chunks) == 1, 'no point in splitting up without multiple cores'

    if DEBUG: print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (len(chunks), ('using %d cores' % cores) if len(chunks) > 1 else '', chunk_size/(1024*1024.), len(meta)/(1024*1024.), len(forwarded_data)/(1024*1024.), total_ll_size/(1024*1024.))

    commands = [
      (i, chunk, meta, settings_file, compiler, forwarded_file, libraries, compiler_engine, temp_files, DEBUG)
      for i, chunk in enumerate(chunks)
    ]

    if len(chunks) > 1:
      pool = multiprocessing.Pool(processes=cores)
      outputs = pool.map(process_funcs, commands, chunksize=1)
    elif len(chunks) == 1:
      outputs = [process_funcs(commands[0])]

    commands = None

  else:
    outputs = []

  if jcache:
    # save chunks to cache
    for i in range(len(chunks)):
      chunk = chunks[i]
      keys = [settings_text, forwarded_data, chunk]
      shortkey = jcache.get_shortkey(keys)
      jcache.set(shortkey, keys, outputs[i])
    if out and DEBUG and len(chunks) > 0: print >> sys.stderr, '  saving %d funcchunks to jcache' % len(chunks)

  chunks = None

  if jcache: outputs += cached_outputs # TODO: preserve order

  outputs = [output.split('//FORWARDED_DATA:') for output in outputs]
  for output in outputs:
    assert len(output) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[0][-3000:]

  if DEBUG: print >> sys.stderr, '  emscript: phase 2 took %s seconds' % (time.time() - t)
  if DEBUG: t = time.time()

  # merge forwarded data
  if settings.get('ASM_JS'):
    all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise
    for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented
      all_exported_functions.add('_' + additional_export)
    exported_implemented_functions = set()
  for func_js, curr_forwarded_data in outputs:
    curr_forwarded_json = json.loads(curr_forwarded_data)
    forwarded_json['Types']['hasInlineJS'] = forwarded_json['Types']['hasInlineJS'] or curr_forwarded_json['Types']['hasInlineJS']
    forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json['Types']['preciseI64MathUsed'] or curr_forwarded_json['Types']['preciseI64MathUsed']
    for key, value in curr_forwarded_json['Functions']['blockAddresses'].iteritems():
      forwarded_json['Functions']['blockAddresses'][key] = value
    for key in curr_forwarded_json['Functions']['indexedFunctions'].iterkeys():
      indexed_functions.add(key)
    if settings.get('ASM_JS'):
      export_bindings = settings['EXPORT_BINDINGS']
      export_all = settings['EXPORT_ALL']
      for key in curr_forwarded_json['Functions']['implementedFunctions'].iterkeys():
        if key in all_exported_functions or export_all or (export_bindings and key.startswith('_emscripten_bind')):
          exported_implemented_functions.add(key)
    for key, value in curr_forwarded_json['Functions']['unimplementedFunctions'].iteritems():
      forwarded_json['Functions']['unimplementedFunctions'][key] = value
    for key, value in curr_forwarded_json['Functions']['neededTables'].iteritems():
      forwarded_json['Functions']['neededTables'][key] = value

  if settings.get('ASM_JS'):
    parts = pre.split('// ASM_LIBRARY FUNCTIONS\n')
    if len(parts) > 1:
      pre = parts[0]
      outputs.append([parts[1]])
  funcs_js = [output[0] for output in outputs]

  outputs = None
  if DEBUG: print >> sys.stderr, '  emscript: phase 2b took %s seconds' % (time.time() - t)
  if DEBUG: t = time.time()

  # calculations on merged forwarded data
  forwarded_json['Functions']['indexedFunctions'] = {}
  i = 2 # universal counter
  if settings['ASM_JS']: i += 2*settings['RESERVED_FUNCTION_POINTERS']
  table_counters = {} # table-specific counters
  alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS']
  sig = None
  for indexed in indexed_functions:
    if alias:
      sig = forwarded_json['Functions']['implementedFunctions'].get(indexed) or forwarded_json['Functions']['unimplementedFunctions'].get(indexed)
      assert sig, indexed
      if sig not in table_counters:
        table_counters[sig] = 2 + 2*settings['RESERVED_FUNCTION_POINTERS']
      curr = table_counters[sig]
      table_counters[sig] += 2
    else:
      curr = i
      i += 2
    #print >> sys.stderr, 'function indexing', indexed, curr, sig
    forwarded_json['Functions']['indexedFunctions'][indexed] = curr # make sure not to modify this python object later - we use it in indexize

  def split_32(x):
    x = int(x)
    return '%d,%d,%d,%d' % (x&255, (x >> 8)&255, (x >> 16)&255, (x >> 24)&255)

  indexing = forwarded_json['Functions']['indexedFunctions']
  def indexize(js):
    # In the global initial allocation, we need to split up into Uint8 format
    ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0", lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js)
    return re.sub(r"'{{ FI_([\w\d_$]+) }}'", lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret)

  blockaddrs = forwarded_json['Functions']['blockAddresses']
  def blockaddrsize(js):
    ret = re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0', lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js)
    return re.sub(r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?', lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret)

  pre = blockaddrsize(indexize(pre))

  if settings.get('ASM_JS'):
    # move postsets into the asm module
    class PostSets: js = ''
    def handle_post_sets(m):
      PostSets.js = m.group(0)
      return '\n'
    pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre)

  #if DEBUG: outfile.write('// pre\n')
  outfile.write(pre)
  pre = None

  #if DEBUG: outfile.write('// funcs\n')

  # forward
  forwarded_data = json.dumps(forwarded_json)
  forwarded_file = temp_files.get('.2.json').name
  open(forwarded_file, 'w').write(indexize(forwarded_data))
  if DEBUG: print >> sys.stderr, '  emscript: phase 2c took %s seconds' % (time.time() - t)

  # Phase 3 - post
  if DEBUG: t = time.time()
  post_file = temp_files.get('.post.ll').name
  open(post_file, 'w').write('\n') # no input, just processing of forwarded data
  out = jsrun.run_js(compiler, compiler_engine, [settings_file, post_file, 'post', forwarded_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE,
                     cwd=path_from_root('src'))
  post, last_forwarded_data = out.split('//FORWARDED_DATA:') # if this fails, perhaps the process failed prior to printing forwarded data?
  last_forwarded_json = json.loads(last_forwarded_data)

  if settings.get('ASM_JS'):
    post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n')
    post = post_rest

    # Move preAsms to their right place
    def move_preasm(m):
      contents = m.groups(0)[0]
      outfile.write(contents + '\n')
      return ''
    post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), post_funcs)

    funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n']

    simple = os.environ.get('EMCC_SIMPLE_ASM')
    class Counter:
      i = 0
    pre_tables = last_forwarded_json['Functions']['tables']['pre']
    del last_forwarded_json['Functions']['tables']['pre']

    def make_table(sig, raw):
      i = Counter.i
      Counter.i += 1
      bad = 'b' + str(i)
      params = ','.join(['p%d' % p for p in range(len(sig)-1)])
      coercions = ';'.join(['p%d = %sp%d%s' % (p, '+' if sig[p+1] != 'i' else '', p, '' if sig[p+1] != 'i' else '|0') for p in range(len(sig)-1)]) + ';'
      ret = '' if sig[0] == 'v' else ('return %s0' % ('+' if sig[0] != 'i' else ''))
      start = raw.index('[')
      end = raw.rindex(']')
      body = raw[start+1:end].split(',')
      for j in range(settings['RESERVED_FUNCTION_POINTERS']):
        body[2 + 2*j] = 'jsCall_%s_%s' % (sig, j)
      def fix_item(item):
        newline = '\n' in item
        return (bad if item.replace('\n', '') == '0' else item) + ('\n' if newline else '')
      body = ','.join(map(fix_item, body))
      return ('function %s(%s) { %s %s(%d); %s }' % (bad, params, coercions, 'abort' if not settings['ASSERTIONS'] else 'nullFunc', i, ret), raw[:start+1] + body + raw[end:])
    infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()]
    function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join([info[1] for info in infos])

    asm_setup = ''
    maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul']]
    fundamentals = ['Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array']
    math_envs = ['Math.min'] # TODO: move min to maths
    asm_setup += '\n'.join(['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs])

    if settings['TO_FLOAT32']: maths += ['Math.toFloat32']

    basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat'] + [m.replace('.', '_') for m in math_envs]
    if settings['RESERVED_FUNCTION_POINTERS'] > 0: basic_funcs.append('jsCall')
    if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR']
    if settings['CHECK_HEAP_ALIGN']: basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8']
    if settings['ASSERTIONS']:
      basic_funcs += ['nullFunc']
      asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n'

    basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT']
    basic_float_vars = ['NaN', 'Infinity']

    if forwarded_json['Types']['preciseI64MathUsed'] or \
       forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \
       forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'):
      basic_vars += ['cttz_i8', 'ctlz_i8']

    if settings.get('DLOPEN_SUPPORT'):
      for sig in last_forwarded_json['Functions']['tables'].iterkeys():
        basic_vars.append('F_BASE_%s' % sig)
        asm_setup += '  var F_BASE_%s = %s;\n' % (sig, 'FUNCTION_TABLE_OFFSET' if settings.get('SIDE_MODULE') else '0') + '\n'

    asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'setThrew'] + ['setTempRet%d' % i for i in range(10)]
    # function tables
    def asm_coerce(value, sig):
      if sig == 'v': return value
      return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i' else '')

    function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']]
    function_tables_impls = []
    for sig in last_forwarded_json['Functions']['tables'].iterkeys():
      args = ','.join(['a' + str(i) for i in range(1, len(sig))])
      arg_coercions = ' '.join(['a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';' for i in range(1, len(sig))])
      coerced_args = ','.join([asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))])
      ret = ('return ' if sig[0] != 'v' else '') + asm_coerce('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0])
      function_tables_impls.append('''
  function dynCall_%s(index%s%s) {
    index = index|0;
    %s
    %s;
  }
''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret))

      for i in range(settings['RESERVED_FUNCTION_POINTERS']):
        jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce('jsCall(%d%s%s)' % (i, ',' if coerced_args else '', coerced_args), sig[0])
        function_tables_impls.append('''
  function jsCall_%s_%s(%s) {
    %s
    %s;
  }

''' % (sig, i, args, arg_coercions, jsret))
      from tools import shared
      shared.Settings.copy(settings)
      asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n'
      basic_funcs.append('invoke_%s' % sig)
      if settings.get('DLOPEN_SUPPORT'):
        asm_setup += '\n' + shared.JS.make_extcall(sig) + '\n'
        basic_funcs.append('extCall_%s' % sig)

    # calculate exports
    exported_implemented_functions = list(exported_implemented_functions)
    exported_implemented_functions.append('runPostSets')
    exports = []
    if not simple:
      for export in exported_implemented_functions + asm_runtime_funcs + function_tables:
        exports.append("%s: %s" % (export, export))
      exports = '{ ' + ', '.join(exports) + ' }'
    else:
      exports = '_main'
    # calculate globals
    try:
      del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable
    except:
      pass
    # If no named globals, only need externals
    global_vars = map(lambda g: g['name'], filter(lambda g: settings['NAMED_GLOBALS'] or g.get('external') or g.get('unIndexable'), forwarded_json['Variables']['globals'].values()))
    global_funcs = ['_' + key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2]
    def math_fix(g):
      return g if not g.startswith('Math_') else g.split('_')[1]
    asm_global_funcs = ''.join(['  var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \
                       ''.join(['  var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs])
    asm_global_vars = ''.join(['  var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \
                      ''.join(['  var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars])
    # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules
    if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'):
      assert settings.get('TARGET_LE32'), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)'
      for key, value in forwarded_json['Variables']['globals'].iteritems():
        if value.get('linkable'):
          init = forwarded_json['Variables']['indexedGlobals'][key] + 8 # 8 is Runtime.GLOBAL_BASE / STATIC_BASE
          if settings.get('SIDE_MODULE'): init = '(H_BASE+' + str(init) + ')|0'
          asm_global_vars += '  var %s=%s;\n' % (key, str(init))

    # sent data
    the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }'
    sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }'
    # received
    if not simple:
      receiving = ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables])
    else:
      receiving = 'var _main = Module["_main"] = asm;'

    # finalize

    if DEBUG: print >> sys.stderr, 'asm text sizes', map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n  ')), len(exports), len(the_global), len(sending), len(receiving)

    funcs_js = ['''
%s
function asmPrintInt(x, y) {
  Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack);
}
function asmPrintFloat(x, y) {
  Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack);
}
// EMSCRIPTEN_START_ASM
var asm = (function(global, env, buffer) {
  %s
  var HEAP8 = new global.Int8Array(buffer);
  var HEAP16 = new global.Int16Array(buffer);
  var HEAP32 = new global.Int32Array(buffer);
  var HEAPU8 = new global.Uint8Array(buffer);
  var HEAPU16 = new global.Uint16Array(buffer);
  var HEAPU32 = new global.Uint32Array(buffer);
  var HEAPF32 = new global.Float32Array(buffer);
  var HEAPF64 = new global.Float64Array(buffer);
''' % (asm_setup, "'use asm';" if not forwarded_json['Types']['hasInlineJS'] and not settings['SIDE_MODULE'] else "'almost asm';") + '\n' + asm_global_vars + '''
  var __THREW__ = 0;
  var threwValue = 0;
  var setjmpId = 0;
  var undef = 0;
  var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0;
''' + ''.join(['''
  var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs + '''
// EMSCRIPTEN_START_FUNCS
function stackAlloc(size) {
  size = size|0;
  var ret = 0;
  ret = STACKTOP;
  STACKTOP = (STACKTOP + size)|0;
''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else 'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + '''
  return ret|0;
}
function stackSave() {
  return STACKTOP|0;
}
function stackRestore(top) {
  top = top|0;
  STACKTOP = top;
}
function setThrew(threw, value) {
  threw = threw|0;
  value = value|0;
  if ((__THREW__|0) == 0) {
    __THREW__ = threw;
    threwValue = value;
  }
}
function copyTempFloat(ptr) {
  ptr = ptr|0;
  HEAP8[tempDoublePtr] = HEAP8[ptr];
  HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0];
  HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0];
  HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0];
}
function copyTempDouble(ptr) {
  ptr = ptr|0;
  HEAP8[tempDoublePtr] = HEAP8[ptr];
  HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0];
  HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0];
  HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0];
  HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0];
  HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0];
  HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0];
  HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0];
}
''' + ''.join(['''
function setTempRet%d(value) {
  value = value|0;
  tempRet%d = value;
}
''' % (i, i) for i in range(10)])] + [PostSets.js + '\n'] + funcs_js + ['''
  %s

  return %s;
})
// EMSCRIPTEN_END_ASM
(%s, %s, buffer);
%s;
''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' + function_tables_defs.replace('\n', '\n  '), exports, the_global, sending, receiving)]

    if not settings.get('SIDE_MODULE'):
      funcs_js.append('''
Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) };
Runtime.stackSave = function() { return asm['stackSave']() };
Runtime.stackRestore = function(top) { asm['stackRestore'](top) };
''')

    # Set function table masks
    masks = {}
    max_mask = 0
    for sig, table in last_forwarded_json['Functions']['tables'].iteritems():
      mask = table.count(',')
      masks[sig] = str(mask)
      max_mask = max(mask, max_mask)
    def function_table_maskize(js, masks):
      def fix(m):
        sig = m.groups(0)[0]
        return masks[sig]
      return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]]
    funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js)

    if settings.get('DLOPEN_SUPPORT'):
      funcs_js.append('''
  asm.maxFunctionIndex = %(max_mask)d;
  DLFCN.registerFunctions(asm, %(max_mask)d+1, %(sigs)s, Module);
  Module.SYMBOL_TABLE = SYMBOL_TABLE;
''' % { 'max_mask': max_mask, 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) })

  else:
    function_tables_defs = '\n'.join([table for table in last_forwarded_json['Functions']['tables'].itervalues()])
    outfile.write(function_tables_defs)
    funcs_js = ['''
// EMSCRIPTEN_START_FUNCS
'''] + funcs_js + ['''
// EMSCRIPTEN_END_FUNCS
''']

  # Create symbol table for self-dlopen
  if settings.get('DLOPEN_SUPPORT'):
    symbol_table = {}
    for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems():
       if forwarded_json['Variables']['globals'][k]['named']:
         symbol_table[k] = str(v + forwarded_json['Runtime']['GLOBAL_BASE'])
    for raw in last_forwarded_json['Functions']['tables'].itervalues():
      if raw == '': continue
      table = map(string.strip, raw[raw.find('[')+1:raw.find(']')].split(","))
      for i in range(len(table)):
        value = table[i]
        if value != '0':
          if settings.get('SIDE_MODULE'):
            symbol_table[value] = 'FUNCTION_TABLE_OFFSET+' + str(i)
          else:
            symbol_table[value] = str(i)
    outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table).replace('"', ''))

  for funcs_js_item in funcs_js: # do this loop carefully to save memory
    funcs_js_item = indexize(funcs_js_item)
    funcs_js_item = blockaddrsize(funcs_js_item)
    outfile.write(funcs_js_item)
  funcs_js = None

  outfile.write(indexize(post))
  if DEBUG: print >> sys.stderr, '  emscript: phase 3 took %s seconds' % (time.time() - t)

  outfile.close()
Example #15
0
def emscript(infile,
             settings,
             outfile,
             libraries=[],
             compiler_engine=None,
             jcache=None,
             temp_files=None,
             DEBUG=None,
             DEBUG_CACHE=None):
    """Runs the emscripten LLVM-to-JS compiler. We parallelize as much as possible

  Args:
    infile: The path to the input LLVM assembly file.
    settings: JSON-formatted settings that override the values
      defined in src/settings.js.
    outfile: The file where the output is written.
  """

    compiler = path_from_root('src', 'compiler.js')

    # Parallelization: We run 3 phases:
    #   1 aka 'pre'  : Process types and metadata and so forth, and generate the preamble.
    #   2 aka 'funcs': Process functions. We can parallelize this, working on each function independently.
    #   3 aka 'post' : Process globals, generate postamble and finishing touches.

    if DEBUG: print >> sys.stderr, 'emscript: ll=>js'

    if jcache: jcache.ensure()

    # Pre-scan ll and alter settings as necessary
    if DEBUG: t = time.time()
    ll = open(infile).read()
    scan(ll, settings)
    total_ll_size = len(ll)
    ll = None  # allow collection
    if DEBUG:
        print >> sys.stderr, '  emscript: scan took %s seconds' % (
            time.time() - t)

    # Split input into the relevant parts for each phase
    pre = []
    funcs = []  # split up functions here, for parallelism later
    meta = []  # needed by each function XXX

    if DEBUG: t = time.time()
    in_func = False
    ll_lines = open(infile).readlines()
    curr_func = None
    for line in ll_lines:
        if in_func:
            curr_func.append(line)
            if line.startswith('}'):
                in_func = False
                funcs.append((curr_func[0], ''.join(curr_func)
                              ))  # use the entire line as the identifier
                # pre needs to know about all implemented functions, even for non-pre func
                pre.append(curr_func[0])
                pre.append(line)
                curr_func = None
        else:
            if line.startswith(';'): continue
            if line.startswith('define '):
                in_func = True
                curr_func = [line]
            elif line.find(' = type { ') > 0:
                pre.append(line)  # type
            elif line.startswith('!'):
                if line.startswith('!llvm.module'):
                    continue  # we can ignore that
                meta.append(line)  # metadata
            else:
                pre.append(
                    line
                )  # pre needs it so we know about globals in pre and funcs. So emit globals there
    ll_lines = None
    meta = ''.join(meta)
    if DEBUG and len(meta) > 1024 * 1024:
        print >> sys.stderr, 'emscript warning: large amounts of metadata, will slow things down'
    if DEBUG:
        print >> sys.stderr, '  emscript: split took %s seconds' % (
            time.time() - t)

    if len(funcs) == 0:
        print >> sys.stderr, 'No functions to process. Make sure you prevented LLVM from eliminating them as dead (use EXPORTED_FUNCTIONS if necessary, see the FAQ)'

    #if DEBUG:
    #  print >> sys.stderr, '========= pre ================\n'
    #  print >> sys.stderr, ''.join(pre)
    #  print >> sys.stderr, '========== funcs ===============\n'
    #  for func in funcs:
    #    print >> sys.stderr, '\n// ===\n\n', ''.join(func)
    #  print >> sys.stderr, '=========================\n'

    # Save settings to a file to work around v8 issue 1579
    settings_file = temp_files.get('.txt').name

    def save_settings():
        global settings_text
        settings_text = json.dumps(settings, sort_keys=True)
        s = open(settings_file, 'w')
        s.write(settings_text)
        s.close()

    save_settings()

    # Phase 1 - pre
    if DEBUG: t = time.time()
    pre_file = temp_files.get('.pre.ll').name
    pre_input = ''.join(pre) + '\n' + meta
    out = None
    if jcache:
        keys = [pre_input, settings_text, ','.join(libraries)]
        shortkey = jcache.get_shortkey(keys)
        if DEBUG_CACHE: print >> sys.stderr, 'shortkey', shortkey

        out = jcache.get(shortkey, keys)

        if DEBUG_CACHE and not out:
            dfpath = os.path.join(get_configuration().TEMP_DIR,
                                  "ems_" + shortkey)
            dfp = open(dfpath, 'w')
            dfp.write(pre_input)
            dfp.write("\n\n========================== settings_text\n\n")
            dfp.write(settings_text)
            dfp.write("\n\n========================== libraries\n\n")
            dfp.write("\n".join(libraries))
            dfp.close()
            print >> sys.stderr, '  cache miss, key data dumped to %s' % dfpath

        if out and DEBUG: print >> sys.stderr, '  loading pre from jcache'
    if not out:
        open(pre_file, 'w').write(pre_input)
        out = jsrun.run_js(compiler,
                           compiler_engine,
                           [settings_file, pre_file, 'pre'] + libraries,
                           stdout=subprocess.PIPE,
                           cwd=path_from_root('src'))
        assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?'
        if jcache:
            if DEBUG: print >> sys.stderr, '  saving pre to jcache'
            jcache.set(shortkey, keys, out)
    pre, forwarded_data = out.split('//FORWARDED_DATA:')
    forwarded_file = temp_files.get('.json').name
    open(forwarded_file, 'w').write(forwarded_data)
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 1 took %s seconds' % (
            time.time() - t)

    indexed_functions = set()
    forwarded_json = json.loads(forwarded_data)
    for key in forwarded_json['Functions']['indexedFunctions'].iterkeys():
        indexed_functions.add(key)

    # Phase 2 - func

    cores = int(os.environ.get('EMCC_CORES') or multiprocessing.cpu_count())
    assert cores >= 1
    if cores > 1:
        intended_num_chunks = int(round(cores * NUM_CHUNKS_PER_CORE))
        chunk_size = max(MIN_CHUNK_SIZE, total_ll_size / intended_num_chunks)
        chunk_size += 3 * len(meta) + len(
            forwarded_data
        ) / 3  # keep ratio of lots of function code to meta (expensive to process, and done in each parallel task) and forwarded data (less expensive but potentially significant)
        chunk_size = min(MAX_CHUNK_SIZE, chunk_size)
    else:
        chunk_size = MAX_CHUNK_SIZE  # if 1 core, just use the max chunk size

    if DEBUG: t = time.time()
    if settings.get('ASM_JS'):
        settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS']
        save_settings()

    chunks = cache_module.chunkify(
        funcs, chunk_size,
        jcache.get_cachename('emscript_files') if jcache else None)

    funcs = None

    if jcache:
        # load chunks from cache where we can # TODO: ignore small chunks
        cached_outputs = []

        def load_from_cache(chunk):
            keys = [settings_text, forwarded_data, chunk]
            shortkey = jcache.get_shortkey(
                keys)  # TODO: share shortkeys with later code
            out = jcache.get(shortkey,
                             keys)  # this is relatively expensive (pickling?)
            if out:
                cached_outputs.append(out)
                return False
            return True

        chunks = filter(load_from_cache, chunks)
        if len(cached_outputs) > 0:
            if out and DEBUG:
                print >> sys.stderr, '  loading %d funcchunks from jcache' % len(
                    cached_outputs)
        else:
            cached_outputs = []

    # TODO: minimize size of forwarded data from funcs to what we actually need

    if len(chunks) > 0:
        if cores == 1 and total_ll_size < MAX_CHUNK_SIZE:
            assert len(
                chunks) == 1, 'no point in splitting up without multiple cores'

        if DEBUG:
            print >> sys.stderr, '  emscript: phase 2 working on %d chunks %s (intended chunk size: %.2f MB, meta: %.2f MB, forwarded: %.2f MB, total: %.2f MB)' % (
                len(chunks),
                ('using %d cores' % cores) if len(chunks) > 1 else '',
                chunk_size / (1024 * 1024.), len(meta) /
                (1024 * 1024.), len(forwarded_data) /
                (1024 * 1024.), total_ll_size / (1024 * 1024.))

        commands = [(i, chunk, meta, settings_file, compiler, forwarded_file,
                     libraries, compiler_engine, temp_files, DEBUG)
                    for i, chunk in enumerate(chunks)]

        if len(chunks) > 1:
            pool = multiprocessing.Pool(processes=cores)
            outputs = pool.map(process_funcs, commands, chunksize=1)
        elif len(chunks) == 1:
            outputs = [process_funcs(commands[0])]

        commands = None

    else:
        outputs = []

    if jcache:
        # save chunks to cache
        for i in range(len(chunks)):
            chunk = chunks[i]
            keys = [settings_text, forwarded_data, chunk]
            shortkey = jcache.get_shortkey(keys)
            jcache.set(shortkey, keys, outputs[i])
        if out and DEBUG and len(chunks) > 0:
            print >> sys.stderr, '  saving %d funcchunks to jcache' % len(
                chunks)

    chunks = None

    if jcache: outputs += cached_outputs  # TODO: preserve order

    outputs = [output.split('//FORWARDED_DATA:') for output in outputs]
    for output in outputs:
        assert len(
            output
        ) == 2, 'Did not receive forwarded data in an output - process failed? We only got: ' + output[
            0][-3000:]

    if DEBUG:
        print >> sys.stderr, '  emscript: phase 2 took %s seconds' % (
            time.time() - t)
    if DEBUG: t = time.time()

    # merge forwarded data
    if settings.get('ASM_JS'):
        all_exported_functions = set(
            settings['EXPORTED_FUNCTIONS'])  # both asm.js and otherwise
        for additional_export in settings[
                'DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']:  # additional functions to export from asm, if they are implemented
            all_exported_functions.add('_' + additional_export)
        exported_implemented_functions = set()
    for func_js, curr_forwarded_data in outputs:
        curr_forwarded_json = json.loads(curr_forwarded_data)
        forwarded_json['Types']['preciseI64MathUsed'] = forwarded_json[
            'Types']['preciseI64MathUsed'] or curr_forwarded_json['Types'][
                'preciseI64MathUsed']
        for key, value in curr_forwarded_json['Functions'][
                'blockAddresses'].iteritems():
            forwarded_json['Functions']['blockAddresses'][key] = value
        for key in curr_forwarded_json['Functions'][
                'indexedFunctions'].iterkeys():
            indexed_functions.add(key)
        if settings.get('ASM_JS'):
            export_bindings = settings['EXPORT_BINDINGS']
            export_all = settings['EXPORT_ALL']
            for key in curr_forwarded_json['Functions'][
                    'implementedFunctions'].iterkeys():
                if key in all_exported_functions or export_all or (
                        export_bindings
                        and key.startswith('_emscripten_bind')):
                    exported_implemented_functions.add(key)
        for key, value in curr_forwarded_json['Functions'][
                'unimplementedFunctions'].iteritems():
            forwarded_json['Functions']['unimplementedFunctions'][key] = value
        for key, value in curr_forwarded_json['Functions'][
                'neededTables'].iteritems():
            forwarded_json['Functions']['neededTables'][key] = value

    if settings.get('ASM_JS'):
        parts = pre.split('// ASM_LIBRARY FUNCTIONS\n')
        if len(parts) > 1:
            pre = parts[0]
            outputs.append([parts[1]])
    funcs_js = [output[0] for output in outputs]

    outputs = None
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 2b took %s seconds' % (
            time.time() - t)
    if DEBUG: t = time.time()

    # calculations on merged forwarded data
    forwarded_json['Functions']['indexedFunctions'] = {}
    i = 2  # universal counter
    if settings['ASM_JS']: i += 2 * settings['RESERVED_FUNCTION_POINTERS']
    table_counters = {}  # table-specific counters
    alias = settings['ASM_JS'] and settings['ALIASING_FUNCTION_POINTERS']
    sig = None
    for indexed in indexed_functions:
        if alias:
            sig = forwarded_json['Functions']['implementedFunctions'].get(
                indexed) or forwarded_json['Functions'][
                    'unimplementedFunctions'].get(indexed)
            assert sig, indexed
            if sig not in table_counters:
                table_counters[
                    sig] = 2 + 2 * settings['RESERVED_FUNCTION_POINTERS']
            curr = table_counters[sig]
            table_counters[sig] += 2
        else:
            curr = i
            i += 2
        #print >> sys.stderr, 'function indexing', indexed, curr, sig
        forwarded_json['Functions']['indexedFunctions'][
            indexed] = curr  # make sure not to modify this python object later - we use it in indexize

    def split_32(x):
        x = int(x)
        return '%d,%d,%d,%d' % (x & 255, (x >> 8) & 255, (x >> 16) & 255,
                                (x >> 24) & 255)

    indexing = forwarded_json['Functions']['indexedFunctions']

    def indexize(js):
        # In the global initial allocation, we need to split up into Uint8 format
        ret = re.sub(r"\"?'?{{ FI_([\w\d_$]+) }}'?\"?,0,0,0",
                     lambda m: split_32(indexing.get(m.groups(0)[0]) or 0), js)
        return re.sub(r"'{{ FI_([\w\d_$]+) }}'",
                      lambda m: str(indexing.get(m.groups(0)[0]) or 0), ret)

    blockaddrs = forwarded_json['Functions']['blockAddresses']

    def blockaddrsize(js):
        ret = re.sub(
            r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?,0,0,0',
            lambda m: split_32(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), js)
        return re.sub(
            r'"?{{{ BA_([\w\d_$]+)\|([\w\d_$]+) }}}"?',
            lambda m: str(blockaddrs[m.groups(0)[0]][m.groups(0)[1]]), ret)

    pre = blockaddrsize(indexize(pre))

    if settings.get('ASM_JS'):
        # move postsets into the asm module
        class PostSets:
            js = ''

        def handle_post_sets(m):
            PostSets.js = m.group(0)
            return '\n'

        pre = re.sub(r'function runPostSets[^}]+}', handle_post_sets, pre)

    #if DEBUG: outfile.write('// pre\n')
    outfile.write(pre)
    pre = None

    #if DEBUG: outfile.write('// funcs\n')

    # forward
    forwarded_data = json.dumps(forwarded_json)
    forwarded_file = temp_files.get('.2.json').name
    open(forwarded_file, 'w').write(indexize(forwarded_data))
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 2c took %s seconds' % (
            time.time() - t)

    # Phase 3 - post
    if DEBUG: t = time.time()
    post_file = temp_files.get('.post.ll').name
    open(post_file,
         'w').write('\n')  # no input, just processing of forwarded data
    out = jsrun.run_js(compiler,
                       compiler_engine,
                       [settings_file, post_file, 'post', forwarded_file] +
                       libraries,
                       stdout=subprocess.PIPE,
                       cwd=path_from_root('src'))
    post, last_forwarded_data = out.split(
        '//FORWARDED_DATA:'
    )  # if this fails, perhaps the process failed prior to printing forwarded data?
    last_forwarded_json = json.loads(last_forwarded_data)

    if settings.get('ASM_JS'):
        post_funcs, post_rest = post.split('// EMSCRIPTEN_END_FUNCS\n')
        post = post_rest

        # Move preAsms to their right place
        def move_preasm(m):
            contents = m.groups(0)[0]
            outfile.write(contents + '\n')
            return ''

        post_funcs = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m),
                            post_funcs)

        funcs_js += ['\n' + post_funcs + '// EMSCRIPTEN_END_FUNCS\n']

        simple = os.environ.get('EMCC_SIMPLE_ASM')

        class Counter:
            i = 0

        pre_tables = last_forwarded_json['Functions']['tables']['pre']
        del last_forwarded_json['Functions']['tables']['pre']

        def make_table(sig, raw):
            i = Counter.i
            Counter.i += 1
            bad = 'b' + str(i)
            params = ','.join(['p%d' % p for p in range(len(sig) - 1)])
            coercions = ';'.join([
                'p%d = %sp%d%s' % (p, '+' if sig[p + 1] != 'i' else '', p,
                                   '' if sig[p + 1] != 'i' else '|0')
                for p in range(len(sig) - 1)
            ]) + ';'
            ret = '' if sig[0] == 'v' else ('return %s0' %
                                            ('+' if sig[0] != 'i' else ''))
            start = raw.index('[')
            end = raw.rindex(']')
            body = raw[start + 1:end].split(',')
            for j in range(settings['RESERVED_FUNCTION_POINTERS']):
                body[2 + 2 * j] = 'jsCall_%s_%s' % (sig, j)

            def fix_item(item):
                newline = '\n' in item
                return (bad if item.replace('\n', '') == '0' else
                        item) + ('\n' if newline else '')

            body = ','.join(map(fix_item, body))
            return ('function %s(%s) { %s %s(%d); %s }' %
                    (bad, params, coercions, 'abort'
                     if not settings['ASSERTIONS'] else 'nullFunc', i, ret),
                    raw[:start + 1] + body + raw[end:])

        infos = [
            make_table(sig, raw) for sig, raw in
            last_forwarded_json['Functions']['tables'].iteritems()
        ]
        function_tables_defs = '\n'.join(
            [info[0]
             for info in infos]) + '\n// EMSCRIPTEN_END_FUNCS\n' + '\n'.join(
                 [info[1] for info in infos])

        asm_setup = ''
        maths = [
            'Math.' + func for func in [
                'floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos',
                'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul'
            ]
        ]
        fundamentals = [
            'Math', 'Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array',
            'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array'
        ]
        math_envs = ['Math.min']  # TODO: move min to maths
        asm_setup += '\n'.join(
            ['var %s = %s;' % (f.replace('.', '_'), f) for f in math_envs])

        if settings['TO_FLOAT32']: maths += ['Math.toFloat32']

        basic_funcs = ['abort', 'assert', 'asmPrintInt', 'asmPrintFloat'
                       ] + [m.replace('.', '_') for m in math_envs]
        if settings['RESERVED_FUNCTION_POINTERS'] > 0:
            basic_funcs.append('jsCall')
        if settings['SAFE_HEAP']:
            basic_funcs += [
                'SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_HEAP_CLEAR'
            ]
        if settings['CHECK_HEAP_ALIGN']:
            basic_funcs += ['CHECK_ALIGN_2', 'CHECK_ALIGN_4', 'CHECK_ALIGN_8']
        if settings['ASSERTIONS']:
            basic_funcs += ['nullFunc']
            asm_setup += 'function nullFunc(x) { Module["printErr"]("Invalid function pointer called. Perhaps a miscast function pointer (check compilation warnings) or bad vtable lookup (maybe due to derefing a bad pointer, like NULL)?"); abort(x) }\n'

        basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT']
        basic_float_vars = ['NaN', 'Infinity']

        if forwarded_json['Types']['preciseI64MathUsed'] or \
           forwarded_json['Functions']['libraryFunctions'].get('llvm_cttz_i32') or \
           forwarded_json['Functions']['libraryFunctions'].get('llvm_ctlz_i32'):
            basic_vars += ['cttz_i8', 'ctlz_i8']

        asm_runtime_funcs = [
            'stackAlloc', 'stackSave', 'stackRestore', 'setThrew'
        ] + ['setTempRet%d' % i for i in range(10)]

        # function tables
        def asm_coerce(value, sig):
            if sig == 'v': return value
            return ('+' if sig != 'i' else '') + value + ('|0' if sig == 'i'
                                                          else '')

        function_tables = [
            'dynCall_' + table
            for table in last_forwarded_json['Functions']['tables']
        ]
        function_tables_impls = []
        for sig in last_forwarded_json['Functions']['tables'].iterkeys():
            args = ','.join(['a' + str(i) for i in range(1, len(sig))])
            arg_coercions = ' '.join([
                'a' + str(i) + '=' + asm_coerce('a' + str(i), sig[i]) + ';'
                for i in range(1, len(sig))
            ])
            coerced_args = ','.join(
                [asm_coerce('a' + str(i), sig[i]) for i in range(1, len(sig))])
            ret = ('return ' if sig[0] != 'v' else '') + asm_coerce(
                'FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' %
                (sig, sig, coerced_args), sig[0])
            function_tables_impls.append('''
  function dynCall_%s(index%s%s) {
    index = index|0;
    %s
    %s;
  }
''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret))

            for i in range(settings['RESERVED_FUNCTION_POINTERS']):
                jsret = ('return ' if sig[0] != 'v' else '') + asm_coerce(
                    'jsCall(%d%s%s)' %
                    (i, ',' if coerced_args else '', coerced_args), sig[0])
                function_tables_impls.append('''
  function jsCall_%s_%s(%s) {
    %s
    %s;
  }

''' % (sig, i, args, arg_coercions, jsret))
            from tools import shared
            asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n'
            basic_funcs.append('invoke_%s' % sig)

        # calculate exports
        exported_implemented_functions = list(exported_implemented_functions)
        exported_implemented_functions.append('runPostSets')
        exports = []
        if not simple:
            for export in exported_implemented_functions + asm_runtime_funcs + function_tables:
                exports.append("%s: %s" % (export, export))
            exports = '{ ' + ', '.join(exports) + ' }'
        else:
            exports = '_main'
        # calculate globals
        try:
            del forwarded_json['Variables']['globals'][
                '_llvm_global_ctors']  # not a true variable
        except:
            pass
        # If no named globals, only need externals
        global_vars = map(
            lambda g: g['name'],
            filter(
                lambda g: settings['NAMED_GLOBALS'] or g.get(
                    'external') or g.get('unIndexable'),
                forwarded_json['Variables']['globals'].values()))
        global_funcs = [
            '_' + key for key, value in forwarded_json['Functions']
            ['libraryFunctions'].iteritems() if value != 2
        ]

        def math_fix(g):
            return g if not g.startswith('Math_') else g.split('_')[1]
        asm_global_funcs = ''.join(['  var ' + g.replace('.', '_') + '=global.' + g + ';\n' for g in maths]) + \
                           ''.join(['  var ' + g + '=env.' + math_fix(g) + ';\n' for g in basic_funcs + global_funcs])
        asm_global_vars = ''.join(['  var ' + g + '=env.' + g + '|0;\n' for g in basic_vars + global_vars]) + \
                          ''.join(['  var ' + g + '=+env.' + g + ';\n' for g in basic_float_vars])
        # In linkable modules, we need to add some explicit globals for global variables that can be linked and used across modules
        if settings.get('MAIN_MODULE') or settings.get('SIDE_MODULE'):
            assert settings.get(
                'TARGET_LE32'
            ), 'TODO: support x86 target when linking modules (needs offset of 4 and not 8 here)'
            for key, value in forwarded_json['Variables']['globals'].iteritems(
            ):
                if value.get('linkable'):
                    init = forwarded_json['Variables']['indexedGlobals'][
                        key] + 8  # 8 is Runtime.GLOBAL_BASE / STATIC_BASE
                    if settings.get('SIDE_MODULE'):
                        init = '(H_BASE+' + str(init) + ')|0'
                    asm_global_vars += '  var %s=%s;\n' % (key, str(init))

        # sent data
        the_global = '{ ' + ', '.join(
            ['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }'
        sending = '{ ' + ', '.join([
            '"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs +
            basic_vars + basic_float_vars + global_vars
        ]) + ' }'
        # received
        if not simple:
            receiving = ';\n'.join([
                'var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]'
                for s in exported_implemented_functions + function_tables
            ])
        else:
            receiving = 'var _main = Module["_main"] = asm;'

        # finalize

        if DEBUG:
            print >> sys.stderr, 'asm text sizes', map(
                len, funcs_js), len(asm_setup), len(asm_global_vars), len(
                    asm_global_funcs), len(pre_tables), len(
                        '\n'.join(function_tables_impls)), len(
                            function_tables_defs.replace(
                                '\n', '\n  ')), len(exports), len(
                                    the_global), len(sending), len(receiving)

        funcs_js = [
            '''
%s
function asmPrintInt(x, y) {
  Module.print('int ' + x + ',' + y);// + ' ' + new Error().stack);
}
function asmPrintFloat(x, y) {
  Module.print('float ' + x + ',' + y);// + ' ' + new Error().stack);
}
// EMSCRIPTEN_START_ASM
var asm = (function(global, env, buffer) {
  'use asm';
  var HEAP8 = new global.Int8Array(buffer);
  var HEAP16 = new global.Int16Array(buffer);
  var HEAP32 = new global.Int32Array(buffer);
  var HEAPU8 = new global.Uint8Array(buffer);
  var HEAPU16 = new global.Uint16Array(buffer);
  var HEAPU32 = new global.Uint32Array(buffer);
  var HEAPF32 = new global.Float32Array(buffer);
  var HEAPF64 = new global.Float64Array(buffer);
''' % (asm_setup, ) + '\n' + asm_global_vars + '''
  var __THREW__ = 0;
  var threwValue = 0;
  var setjmpId = 0;
  var undef = 0;
  var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0;
''' + ''.join(['''
  var tempRet%d = 0;''' % i
               for i in range(10)]) + '\n' + asm_global_funcs + '''
// EMSCRIPTEN_START_FUNCS
  function stackAlloc(size) {
    size = size|0;
    var ret = 0;
    ret = STACKTOP;
    STACKTOP = (STACKTOP + size)|0;
''' + ('STACKTOP = ((STACKTOP + 3)>>2)<<2;' if settings['TARGET_X86'] else
        'STACKTOP = ((STACKTOP + 7)>>3)<<3;') + '''
    return ret|0;
  }
  function stackSave() {
    return STACKTOP|0;
  }
  function stackRestore(top) {
    top = top|0;
    STACKTOP = top;
  }
  function setThrew(threw, value) {
    threw = threw|0;
    value = value|0;
    if ((__THREW__|0) == 0) {
      __THREW__ = threw;
      threwValue = value;
    }
  }
  function copyTempFloat(ptr) {
    ptr = ptr|0;
    HEAP8[tempDoublePtr] = HEAP8[ptr];
    HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0];
    HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0];
    HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0];
  }
  function copyTempDouble(ptr) {
    ptr = ptr|0;
    HEAP8[tempDoublePtr] = HEAP8[ptr];
    HEAP8[tempDoublePtr+1|0] = HEAP8[ptr+1|0];
    HEAP8[tempDoublePtr+2|0] = HEAP8[ptr+2|0];
    HEAP8[tempDoublePtr+3|0] = HEAP8[ptr+3|0];
    HEAP8[tempDoublePtr+4|0] = HEAP8[ptr+4|0];
    HEAP8[tempDoublePtr+5|0] = HEAP8[ptr+5|0];
    HEAP8[tempDoublePtr+6|0] = HEAP8[ptr+6|0];
    HEAP8[tempDoublePtr+7|0] = HEAP8[ptr+7|0];
  }
''' + ''.join([
           '''
  function setTempRet%d(value) {
    value = value|0;
    tempRet%d = value;
  }
''' % (i, i) for i in range(10)
        ])
        ] + [PostSets.js + '\n'] + funcs_js + [
            '''
  %s

  return %s;
})
// EMSCRIPTEN_END_ASM
(%s, %s, buffer);
%s;
Runtime.stackAlloc = function(size) { return asm['stackAlloc'](size) };
Runtime.stackSave = function() { return asm['stackSave']() };
Runtime.stackRestore = function(top) { asm['stackRestore'](top) };
''' % (pre_tables + '\n'.join(function_tables_impls) + '\n' +
        function_tables_defs.replace('\n', '\n  '), exports, the_global,
        sending, receiving)
        ]

        # Set function table masks
        def function_table_maskize(js):
            masks = {}
            default = None
            for sig, table in last_forwarded_json['Functions'][
                    'tables'].iteritems():
                masks[sig] = str(table.count(','))
                default = sig

            def fix(m):
                sig = m.groups(0)[0]
                return masks[sig]

            return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m),
                          js)  # masks[m.groups(0)[0]]

        funcs_js = map(function_table_maskize, funcs_js)
    else:
        function_tables_defs = '\n'.join([
            table for table in last_forwarded_json['Functions']
            ['tables'].itervalues()
        ])
        outfile.write(function_tables_defs)
        funcs_js = ['''
// EMSCRIPTEN_START_FUNCS
'''] + funcs_js + ['''
// EMSCRIPTEN_END_FUNCS
''']

    # Create symbol table for self-dlopen
    if settings.get('DLOPEN_SUPPORT'):
        symbol_table = {}
        for k, v in forwarded_json['Variables']['indexedGlobals'].iteritems():
            if forwarded_json['Variables']['globals'][k]['named']:
                symbol_table[k] = v + forwarded_json['Runtime']['GLOBAL_BASE']
        for raw in last_forwarded_json['Functions']['tables'].itervalues():
            if raw == '': continue
            table = map(string.strip,
                        raw[raw.find('[') + 1:raw.find(']')].split(","))
            symbol_table.update(
                map(lambda x: (x[1], x[0]),
                    filter(lambda x: x[1] != '0', enumerate(table))))
        outfile.write("var SYMBOL_TABLE = %s;" % json.dumps(symbol_table))

    for funcs_js_item in funcs_js:  # do this loop carefully to save memory
        funcs_js_item = indexize(funcs_js_item)
        funcs_js_item = blockaddrsize(funcs_js_item)
        outfile.write(funcs_js_item)
    funcs_js = None

    outfile.write(indexize(post))
    if DEBUG:
        print >> sys.stderr, '  emscript: phase 3 took %s seconds' % (
            time.time() - t)

    outfile.close()
Example #16
0
def emscript(infile, settings, outfile, libraries=[], compiler_engine=None,
             temp_files=None, DEBUG=None, DEBUG_CACHE=None):
  """Runs the emscripten LLVM-to-JS compiler.

  Args:
    infile: The path to the input LLVM assembly file.
    settings: JSON-formatted settings that override the values
      defined in src/settings.js.
    outfile: The file where the output is written.
  """

  assert settings['ASM_JS'], 'fastcomp is asm.js-only (mode 1 or 2)'

  success = False

  try:

    # Overview:
    #   * Run LLVM backend to emit JS. JS includes function bodies, memory initializer,
    #     and various metadata
    #   * Run compiler.js on the metadata to emit the shell js code, pre/post-ambles,
    #     JS library dependencies, etc.

    temp_js = temp_files.get('.4.js').name
    backend_compiler = os.path.join(shared.LLVM_ROOT, 'llc')
    backend_args = [backend_compiler, infile, '-march=js', '-filetype=asm', '-o', temp_js]
    if settings['PRECISE_F32']:
      backend_args += ['-emscripten-precise-f32']
    if settings['USE_PTHREADS']:
      backend_args += ['-emscripten-enable-pthreads']
    if settings['WARN_UNALIGNED']:
      backend_args += ['-emscripten-warn-unaligned']
    if settings['RESERVED_FUNCTION_POINTERS'] > 0:
      backend_args += ['-emscripten-reserved-function-pointers=%d' % settings['RESERVED_FUNCTION_POINTERS']]
    if settings['ASSERTIONS'] > 0:
      backend_args += ['-emscripten-assertions=%d' % settings['ASSERTIONS']]
    if settings['ALIASING_FUNCTION_POINTERS'] == 0:
      backend_args += ['-emscripten-no-aliasing-function-pointers']
    if settings['EMULATED_FUNCTION_POINTERS']:
      backend_args += ['-emscripten-emulated-function-pointers']
    if settings['RELOCATABLE']:
      backend_args += ['-emscripten-relocatable']
      backend_args += ['-emscripten-global-base=0']
    elif settings['GLOBAL_BASE'] >= 0:
      backend_args += ['-emscripten-global-base=%d' % settings['GLOBAL_BASE']]
    backend_args += ['-O' + str(settings['OPT_LEVEL'])]
    if DEBUG:
      logging.debug('emscript: llvm backend: ' + ' '.join(backend_args))
      t = time.time()
    shared.jsrun.timeout_run(subprocess.Popen(backend_args, stdout=subprocess.PIPE))
    if DEBUG:
      logging.debug('  emscript: llvm backend took %s seconds' % (time.time() - t))
      t = time.time()

    # Split up output
    backend_output = open(temp_js).read()
    #if DEBUG: print >> sys.stderr, backend_output

    start_funcs_marker = '// EMSCRIPTEN_START_FUNCTIONS'
    end_funcs_marker = '// EMSCRIPTEN_END_FUNCTIONS'
    metadata_split_marker = '// EMSCRIPTEN_METADATA'

    start_funcs = backend_output.index(start_funcs_marker)
    end_funcs = backend_output.rindex(end_funcs_marker)
    metadata_split = backend_output.rindex(metadata_split_marker)

    funcs = backend_output[start_funcs+len(start_funcs_marker):end_funcs]
    metadata_raw = backend_output[metadata_split+len(metadata_split_marker):]
    #if DEBUG: print >> sys.stderr, "METAraw", metadata_raw
    try:
      metadata = cjson.decode(metadata_raw)
    except Exception, e:
      logging.error('emscript: failure to parse metadata output from compiler backend. raw output is: \n' + metadata_raw[:10000])
      raise e
    mem_init = backend_output[end_funcs+len(end_funcs_marker):metadata_split]
    #if DEBUG: print >> sys.stderr, "FUNCS", funcs
    #if DEBUG: print >> sys.stderr, "META", metadata
    #if DEBUG: print >> sys.stderr, "meminit", mem_init

    # if emulating pointer casts, force all tables to the size of the largest
    if settings['EMULATE_FUNCTION_POINTER_CASTS']:
      max_size = 0
      for k, v in metadata['tables'].iteritems():
        max_size = max(max_size, v.count(',')+1)
      for k, v in metadata['tables'].iteritems():
        curr = v.count(',')+1
        if curr < max_size:
          metadata['tables'][k] = v.replace(']', (',0'*(max_size - curr)) + ']')

    if settings['SIDE_MODULE']:
      for k in metadata['tables'].keys():
        metadata['tables'][k] = metadata['tables'][k].replace('var FUNCTION_TABLE_', 'var SIDE_FUNCTION_TABLE_')

    # function table masks

    table_sizes = {}
    for k, v in metadata['tables'].iteritems():
      table_sizes[k] = str(v.count(',')) # undercounts by one, but that is what we want
      #if settings['ASSERTIONS'] >= 2 and table_sizes[k] == 0:
      #  print >> sys.stderr, 'warning: no function pointers with signature ' + k + ', but there is a call, which will abort if it occurs (this can result from undefined behavior, check for compiler warnings on your source files and consider -Werror)'
    funcs = re.sub(r"#FM_(\w+)#", lambda m: table_sizes[m.groups(0)[0]], funcs)

    # fix +float into float.0, if not running js opts
    if not settings['RUNNING_JS_OPTS']:
      def fix_dot_zero(m):
        num = m.group(3)
        # TODO: handle 0x floats?
        if num.find('.') < 0:
          e = num.find('e');
          if e < 0:
            num += '.0'
          else:
            num = num[:e] + '.0' + num[e:]
        return m.group(1) + m.group(2) + num
      funcs = re.sub(r'([(=,+\-*/%<>:?] *)\+(-?)((0x)?[0-9a-f]*\.?[0-9]+([eE][-+]?[0-9]+)?)', lambda m: fix_dot_zero(m), funcs)

    # js compiler

    if DEBUG: logging.debug('emscript: js compiler glue')

    # Settings changes
    i64_funcs = ['i64Add', 'i64Subtract', '__muldi3', '__divdi3', '__udivdi3', '__remdi3', '__uremdi3']
    for i64_func in i64_funcs:
      if i64_func in metadata['declares']:
        settings['PRECISE_I64_MATH'] = 2
        break

    metadata['declares'] = filter(lambda i64_func: i64_func not in ['getHigh32', 'setHigh32', '__muldi3', '__divdi3', '__remdi3', '__udivdi3', '__uremdi3'], metadata['declares']) # FIXME: do these one by one as normal js lib funcs

    # Integrate info from backend
    if settings['SIDE_MODULE']:
      settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] = [] # we don't need any JS library contents in side modules
    settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] = list(
      set(settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE'] + map(shared.JS.to_nice_ident, metadata['declares'])).difference(
        map(lambda x: x[1:], metadata['implementedFunctions'])
      )
    ) + map(lambda x: x[1:], metadata['externs'])
    if metadata['simd']:
      settings['SIMD'] = 1
    if metadata['cantValidate'] and settings['ASM_JS'] != 2:
      logging.warning('disabling asm.js validation due to use of non-supported features: ' + metadata['cantValidate'])
      settings['ASM_JS'] = 2

    # Save settings to a file to work around v8 issue 1579
    settings_file = temp_files.get('.txt').name
    def save_settings():
      global settings_text
      settings_text = json.dumps(settings, sort_keys=True)
      s = open(settings_file, 'w')
      s.write(settings_text)
      s.close()
    save_settings()

    # Call js compiler
    if DEBUG: t = time.time()
    out = jsrun.run_js(path_from_root('src', 'compiler.js'), compiler_engine,
                       [settings_file] + libraries, stdout=subprocess.PIPE, stderr=STDERR_FILE,
                       cwd=path_from_root('src'), error_limit=300)
    assert '//FORWARDED_DATA:' in out, 'Did not receive forwarded data in pre output - process failed?'
    glue, forwarded_data = out.split('//FORWARDED_DATA:')

    if DEBUG:
      logging.debug('  emscript: glue took %s seconds' % (time.time() - t))
      t = time.time()

    last_forwarded_json = forwarded_json = json.loads(forwarded_data)

    # merge in information from llvm backend

    last_forwarded_json['Functions']['tables'] = metadata['tables']

    pre, post = glue.split('// EMSCRIPTEN_END_FUNCS')

    #print >> sys.stderr, 'glue:', pre, '\n\n||||||||||||||||\n\n', post, '...............'

    # memory and global initializers

    global_initializers = str(', '.join(map(lambda i: '{ func: function() { %s() } }' % i, metadata['initializers'])))

    if settings['SIMD'] == 1:
      pre = open(path_from_root(os.path.join('src', 'ecmascript_simd.js'))).read() + '\n\n' + pre

    staticbump = mem_init.count(',')+1
    while staticbump % 16 != 0: staticbump += 1
    pre = pre.replace('STATICTOP = STATIC_BASE + 0;', '''STATICTOP = STATIC_BASE + %d;
  /* global initializers */ %s __ATINIT__.push(%s);
  %s''' % (staticbump, 'if (!ENVIRONMENT_IS_PTHREAD)' if settings['USE_PTHREADS'] else '', global_initializers, mem_init)) # XXX wrong size calculation!

    if settings['SIDE_MODULE']:
      pre = pre.replace('Runtime.GLOBAL_BASE', 'gb').replace('{{{ STATIC_BUMP }}}', str(staticbump))

    funcs_js = [funcs]
    parts = pre.split('// ASM_LIBRARY FUNCTIONS\n')
    if len(parts) > 1:
      pre = parts[0]
      funcs_js.append(parts[1])

    # merge forwarded data
    settings['EXPORTED_FUNCTIONS'] = forwarded_json['EXPORTED_FUNCTIONS']
    all_exported_functions = set(settings['EXPORTED_FUNCTIONS']) # both asm.js and otherwise
    for additional_export in settings['DEFAULT_LIBRARY_FUNCS_TO_INCLUDE']: # additional functions to export from asm, if they are implemented
      all_exported_functions.add('_' + additional_export)
    if settings['EXPORT_FUNCTION_TABLES']:
      for table in last_forwarded_json['Functions']['tables'].values():
        for func in table.split('[')[1].split(']')[0].split(','):
          if func[0] == '_':
            all_exported_functions.add(func)
    exported_implemented_functions = set(metadata['exports'])
    export_bindings = settings['EXPORT_BINDINGS']
    export_all = settings['EXPORT_ALL']
    all_implemented = metadata['implementedFunctions'] + forwarded_json['Functions']['implementedFunctions'].keys() # XXX perf?
    for key in all_implemented:
      if key in all_exported_functions or export_all or (export_bindings and key.startswith('_emscripten_bind')):
        exported_implemented_functions.add(key)
    implemented_functions = set(metadata['implementedFunctions'])
    if settings['ASSERTIONS'] and settings.get('ORIGINAL_EXPORTED_FUNCTIONS'):
      original_exports = settings['ORIGINAL_EXPORTED_FUNCTIONS']
      if original_exports[0] == '@': original_exports = json.loads(open(original_exports[1:]).read())
      for requested in original_exports:
        if requested not in all_implemented and \
           requested != '_malloc': # special-case malloc, EXPORTED by default for internal use, but we bake in a trivial allocator and warn at runtime if used in ASSERTIONS
          logging.warning('function requested to be exported, but not implemented: "%s"', requested)

    asm_consts = [0]*len(metadata['asmConsts'])
    for k, v in metadata['asmConsts'].iteritems():
      const = v.encode('utf-8')
      if const[0] == '"' and const[-1] == '"':
        const = const[1:-1]
      const = '{ ' + const + ' }'
      i = 0
      args = []
      while ('$' + str(i)) in const:
        args.append('$' + str(i))
        i += 1
      const = 'function(' + ', '.join(args ) + ') ' + const
      asm_consts[int(k)] = const

    asm_const_funcs = []
    for arity in metadata['asmConstArities']:
      forwarded_json['Functions']['libraryFunctions']['_emscripten_asm_const_%d' % arity] = 1
      args = ['a%d' % i for i in range(arity)]
      all_args = ['code'] + args
      asm_const_funcs.append(r'''
function _emscripten_asm_const_%d(%s) {
 return ASM_CONSTS[code](%s);
}''' % (arity, ', '.join(all_args), ', '.join(args)))

    pre = pre.replace('// === Body ===', '// === Body ===\n' + '\nvar ASM_CONSTS = [' + ', '.join(asm_consts) + '];\n' + '\n'.join(asm_const_funcs) + '\n')

    #if DEBUG: outfile.write('// pre\n')
    outfile.write(pre)
    pre = None

    #if DEBUG: outfile.write('// funcs\n')

    # when emulating function pointer casts, we need to know what is the target of each pointer
    if settings['EMULATE_FUNCTION_POINTER_CASTS']:
      function_pointer_targets = {}
      for sig, table in last_forwarded_json['Functions']['tables'].iteritems():
        start = table.index('[')
        end = table.rindex(']')
        body = table[start+1:end].split(',')
        parsed = map(lambda x: x.strip(), body)
        for i in range(len(parsed)):
          if parsed[i] != '0':
            assert i not in function_pointer_targets
            function_pointer_targets[i] = [sig, str(parsed[i])]

    # Move preAsms to their right place
    def move_preasm(m):
      contents = m.groups(0)[0]
      outfile.write(contents + '\n')
      return ''
    if not settings['BOOTSTRAPPING_STRUCT_INFO'] and len(funcs_js) > 1:
      funcs_js[1] = re.sub(r'/\* PRE_ASM \*/(.*)\n', lambda m: move_preasm(m), funcs_js[1])

    class Counter:
      i = 0
      j = 0
    if 'pre' in last_forwarded_json['Functions']['tables']:
      pre_tables = last_forwarded_json['Functions']['tables']['pre']
      del last_forwarded_json['Functions']['tables']['pre']
    else:
      pre_tables = ''

    def unfloat(s):
      return 'd' if s == 'f' else s # lower float to double for ffis

    if settings['ASSERTIONS'] >= 2:
      debug_tables = {}

    def make_params(sig): return ','.join(['p%d' % p for p in range(len(sig)-1)])
    def make_coerced_params(sig): return ','.join([shared.JS.make_coercion('p%d', unfloat(sig[p+1]), settings) % p for p in range(len(sig)-1)])
    def make_coercions(sig): return ';'.join(['p%d = %s' % (p, shared.JS.make_coercion('p%d' % p, sig[p+1], settings)) for p in range(len(sig)-1)]) + ';'
    def make_func(name, code, params, coercions): return 'function %s(%s) { %s %s }' % (name, params, coercions, code)

    in_table = set()

    def make_table(sig, raw):
      params = make_params(sig)
      coerced_params = make_coerced_params(sig)
      coercions = make_coercions(sig)
      def make_bad(target=None):
        i = Counter.i
        Counter.i += 1
        if target is None: target = i
        name = 'b' + str(i)
        if not settings['ASSERTIONS']:
          code = 'abort(%s);' % target
        else:
          code = 'nullFunc_' + sig + '(%d);' % target
        if sig[0] != 'v':
          code += 'return %s' % shared.JS.make_initializer(sig[0], settings) + ';'
        return name, make_func(name, code, params, coercions)
      bad, bad_func = make_bad() # the default bad func
      if settings['ASSERTIONS'] <= 1:
        Counter.pre = [bad_func]
      else:
        Counter.pre = []
      start = raw.index('[')
      end = raw.rindex(']')
      body = raw[start+1:end].split(',')
      if settings['EMULATED_FUNCTION_POINTERS']:
        def receive(item):
          if item == '0':
            return item
          else:
            if item in all_implemented:
              in_table.add(item)
              return "asm['" + item + "']"
            else:
              return item # this is not implemented; it would normally be wrapped, but with emulation, we just use it directly outside
        body = map(receive, body)
      for j in range(settings['RESERVED_FUNCTION_POINTERS']):
        curr = 'jsCall_%s_%s' % (sig, j)
        body[settings['FUNCTION_POINTER_ALIGNMENT'] * (1 + j)] = curr
        implemented_functions.add(curr)
      Counter.j = 0
      def fix_item(item):
        j = Counter.j
        Counter.j += 1
        newline = Counter.j % 30 == 29
        if item == '0':
          if j > 0 and settings['EMULATE_FUNCTION_POINTER_CASTS'] and j in function_pointer_targets: # emulate all non-null pointer calls, if asked to
            proper_sig, proper_target = function_pointer_targets[j]
            if settings['EMULATED_FUNCTION_POINTERS']:
              if proper_target in all_implemented:
                proper_target = "asm['" + proper_target + "']"
            def make_emulated_param(i):
              if i >= len(sig): return shared.JS.make_initializer(proper_sig[i], settings) # extra param, just send a zero
              return shared.JS.make_coercion('p%d' % (i-1), proper_sig[i], settings, convert_from=sig[i])
            proper_code = proper_target + '(' + ','.join(map(lambda i: make_emulated_param(i+1), range(len(proper_sig)-1))) + ')'
            if proper_sig[0] != 'v':
              # proper sig has a return, which the wrapper may or may not use
              proper_code = shared.JS.make_coercion(proper_code, proper_sig[0], settings)
              if proper_sig[0] != sig[0]:
                # first coercion ensured we call the target ok; this one ensures we return the right type in the wrapper
                proper_code = shared.JS.make_coercion(proper_code, sig[0], settings, convert_from=proper_sig[0])
              if sig[0] != 'v':
                proper_code = 'return ' + proper_code
            else:
              # proper sig has no return, we may need a fake return
              if sig[0] != 'v':
                proper_code = 'return ' + shared.JS.make_initializer(sig[0], settings)
            name = 'fpemu_%s_%d' % (sig, j)
            wrapper = make_func(name, proper_code, params, coercions)
            Counter.pre.append(wrapper)
            return name if not newline else (name + '\n')
          if settings['ASSERTIONS'] <= 1:
            return bad if not newline else (bad + '\n')
          else:
            specific_bad, specific_bad_func = make_bad(j)
            Counter.pre.append(specific_bad_func)
            return specific_bad if not newline else (specific_bad + '\n')
        if item not in implemented_functions and not settings['EMULATED_FUNCTION_POINTERS']: # when emulating function pointers, we don't need wrappers
          # this is imported into asm, we must wrap it
          call_ident = item
          if call_ident in metadata['redirects']: call_ident = metadata['redirects'][call_ident]
          if not call_ident.startswith('_') and not call_ident.startswith('Math_'): call_ident = '_' + call_ident
          code = call_ident + '(' + coerced_params + ')'
          if sig[0] != 'v':
            # ffis cannot return float
            if sig[0] == 'f': code = '+' + code
            code = 'return ' + shared.JS.make_coercion(code, sig[0], settings)
          code += ';'
          Counter.pre.append(make_func(item + '__wrapper', code, params, coercions))
          return item + '__wrapper'
        return item if not newline else (item + '\n')
      if settings['ASSERTIONS'] >= 2:
        debug_tables[sig] = body
      body = ','.join(map(fix_item, body))
      return ('\n'.join(Counter.pre), ''.join([raw[:start+1], body, raw[end:]]))

    infos = [make_table(sig, raw) for sig, raw in last_forwarded_json['Functions']['tables'].iteritems()]
    Counter.pre = []

    function_tables_defs = '\n'.join([info[0] for info in infos]) + '\n'
    if not settings['EMULATED_FUNCTION_POINTERS']:
      function_tables_defs += '\n// EMSCRIPTEN_END_FUNCS\n'
    function_tables_defs += '\n'.join([info[1] for info in infos])

    asm_setup = ''

    if settings['ASSERTIONS'] >= 2:
      for sig in last_forwarded_json['Functions']['tables']:
        asm_setup += '\nvar debug_table_' + sig + ' = ' + json.dumps(debug_tables[sig]) + ';'

    maths = ['Math.' + func for func in ['floor', 'abs', 'sqrt', 'pow', 'cos', 'sin', 'tan', 'acos', 'asin', 'atan', 'atan2', 'exp', 'log', 'ceil', 'imul', 'min', 'clz32']]
    simdfloattypes = ['float32x4']
    simdinttypes = ['int32x4']
    simdtypes = simdfloattypes + simdinttypes
    simdfuncs = ['check', 'add', 'sub', 'neg', 'mul',
                 'equal', 'lessThan', 'greaterThan',
                 'notEqual', 'lessThanOrEqual', 'greaterThanOrEqual',
                 'select', 'and', 'or', 'xor', 'not',
                 'splat', 'swizzle', 'shuffle',
                 'withX', 'withY', 'withZ', 'withW',
                 'load', 'store', 'loadX', 'storeX', 'loadXY', 'storeXY', 'loadXYZ', 'storeXYZ']
    simdfloatfuncs = simdfuncs + ['div', 'min', 'max', 'minNum', 'maxNum', 'sqrt',
                                  'abs', 'fromInt32x4', 'fromInt32x4Bits',
                                  'reciprocalApproximation', 'reciprocalSqrtApproximation'];
    simdintfuncs = simdfuncs + ['fromFloat32x4', 'fromFloat32x4Bits',
                                'shiftRightArithmeticByScalar',
                                'shiftRightLogicalByScalar',
                                'shiftLeftByScalar'];
    fundamentals = ['Math']
    if settings['USE_PTHREADS']:
      fundamentals += ['SharedInt8Array', 'SharedInt16Array', 'SharedInt32Array', 'SharedUint8Array', 'SharedUint16Array', 'SharedUint32Array', 'SharedFloat32Array', 'SharedFloat64Array', 'Atomics']
    else:
      fundamentals += ['Int8Array', 'Int16Array', 'Int32Array', 'Uint8Array', 'Uint16Array', 'Uint32Array', 'Float32Array', 'Float64Array']
    fundamentals += ['NaN', 'Infinity']
    if metadata['simd']:
        fundamentals += ['SIMD']
    if settings['ALLOW_MEMORY_GROWTH']: fundamentals.append('byteLength')
    math_envs = []

    provide_fround = settings['PRECISE_F32'] or settings['SIMD']

    if provide_fround: maths += ['Math.fround']

    def get_function_pointer_error(sig):
      if settings['ASSERTIONS'] <= 1:
        extra = ' Module["printErr"]("Build with ASSERTIONS=2 for more info.");'
        pointer = ' '
      else:
        pointer = ' \'" + x + "\' '
        extra = ' Module["printErr"]("This pointer might make sense in another type signature: '
        # sort signatures, attempting to show most likely related ones first
        sigs = last_forwarded_json['Functions']['tables'].keys()
        def keyfunc(other):
          ret = 0
          minlen = min(len(other), len(sig))
          maxlen = min(len(other), len(sig))
          if other.startswith(sig) or sig.startswith(other): ret -= 1000 # prioritize prefixes, could be dropped params
          ret -= 133*difflib.SequenceMatcher(a=other, b=sig).ratio() # prioritize on diff similarity
          ret += 15*abs(len(other) - len(sig))/float(maxlen) # deprioritize the bigger the length difference is
          for i in range(minlen):
            if other[i] == sig[i]: ret -= 5/float(maxlen) # prioritize on identically-placed params
          ret += 20*len(other) # deprioritize on length
          return ret
        sigs.sort(key=keyfunc)
        for other in sigs:
          if other != sig:
            extra += other + ': " + debug_table_' + other + '[x] + "  '
        extra += '"); '
      return 'Module["printErr"]("Invalid function pointer' + pointer + 'called with signature \'' + sig + '\'. ' + \
             'Perhaps this is an invalid value (e.g. caused by calling a virtual method on a NULL pointer)? ' + \
             'Or calling a function with an incorrect type, which will fail? ' + \
             '(it is worth building your source files with -Werror (warnings are errors), as warnings can indicate undefined behavior which can cause this)' + \
             '"); ' + extra

    basic_funcs = ['abort', 'assert'] + [m.replace('.', '_') for m in math_envs]
    if settings['SAFE_HEAP']: basic_funcs += ['SAFE_HEAP_LOAD', 'SAFE_HEAP_STORE', 'SAFE_FT_MASK']
    if settings['ASSERTIONS']:
      if settings['ASSERTIONS'] >= 2: import difflib
      for sig in last_forwarded_json['Functions']['tables'].iterkeys():
        basic_funcs += ['nullFunc_' + sig]
        asm_setup += '\nfunction nullFunc_' + sig + '(x) { ' + get_function_pointer_error(sig) + 'abort(x) }\n'

    basic_vars = ['STACKTOP', 'STACK_MAX', 'tempDoublePtr', 'ABORT']
    basic_float_vars = []

    if metadata.get('preciseI64MathUsed'):
      basic_vars += ['cttz_i8']
    else:
      if forwarded_json['Functions']['libraryFunctions'].get('_llvm_cttz_i32'):
        basic_vars += ['cttz_i8']

    if settings['RELOCATABLE']:
      basic_vars += ['gb', 'fb']
      if not settings['SIDE_MODULE']:
        asm_setup += 'var gb = Runtime.GLOBAL_BASE, fb = 0;\n'

    asm_runtime_funcs = ['stackAlloc', 'stackSave', 'stackRestore', 'establishStackSpace', 'setThrew']
    if not settings['RELOCATABLE']:
      asm_runtime_funcs += ['setTempRet0', 'getTempRet0']
    else:
      basic_funcs += ['setTempRet0', 'getTempRet0']
      asm_setup += 'var setTempRet0 = Runtime.setTempRet0, getTempRet0 = Runtime.getTempRet0;\n'

    # See if we need ASYNCIFY functions
    # We might not need them even if ASYNCIFY is enabled
    need_asyncify = '_emscripten_alloc_async_context' in exported_implemented_functions
    if need_asyncify:
      basic_vars += ['___async', '___async_unwind', '___async_retval', '___async_cur_frame']
      asm_runtime_funcs += ['setAsync']

    if settings.get('EMTERPRETIFY'):
      asm_runtime_funcs += ['emterpret']
      if settings.get('EMTERPRETIFY_ASYNC'):
        asm_runtime_funcs += ['setAsyncState', 'emtStackSave']

    # function tables
    if not settings['EMULATED_FUNCTION_POINTERS']:
      function_tables = ['dynCall_' + table for table in last_forwarded_json['Functions']['tables']]
    else:
      function_tables = []
    function_tables_impls = []

    for sig in last_forwarded_json['Functions']['tables'].iterkeys():
      args = ','.join(['a' + str(i) for i in range(1, len(sig))])
      arg_coercions = ' '.join(['a' + str(i) + '=' + shared.JS.make_coercion('a' + str(i), sig[i], settings) + ';' for i in range(1, len(sig))])
      coerced_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings) for i in range(1, len(sig))])
      ret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('FUNCTION_TABLE_%s[index&{{{ FTM_%s }}}](%s)' % (sig, sig, coerced_args), sig[0], settings)
      if not settings['EMULATED_FUNCTION_POINTERS']:
        function_tables_impls.append('''
function dynCall_%s(index%s%s) {
  index = index|0;
  %s
  %s;
}
''' % (sig, ',' if len(sig) > 1 else '', args, arg_coercions, ret))
      else:
        function_tables_impls.append('''
var dynCall_%s = ftCall_%s;
''' % (sig, sig))

      ffi_args = ','.join([shared.JS.make_coercion('a' + str(i), sig[i], settings, ffi_arg=True) for i in range(1, len(sig))])
      for i in range(settings['RESERVED_FUNCTION_POINTERS']):
        jsret = ('return ' if sig[0] != 'v' else '') + shared.JS.make_coercion('jsCall_%s(%d%s%s)' % (sig, i, ',' if ffi_args else '', ffi_args), sig[0], settings, ffi_result=True)
        function_tables_impls.append('''
function jsCall_%s_%s(%s) {
  %s
  %s;
}

''' % (sig, i, args, arg_coercions, jsret))
      shared.Settings.copy(settings)
      asm_setup += '\n' + shared.JS.make_invoke(sig) + '\n'
      basic_funcs.append('invoke_%s' % sig)
      if settings.get('RESERVED_FUNCTION_POINTERS'):
        asm_setup += '\n' + shared.JS.make_jscall(sig) + '\n'
        basic_funcs.append('jsCall_%s' % sig)
      if settings.get('EMULATED_FUNCTION_POINTERS'):
        args = ['a%d' % i for i in range(len(sig)-1)]
        full_args = ['x'] + args
        table_access = 'FUNCTION_TABLE_' + sig
        if settings['SIDE_MODULE']:
          table_access = 'parentModule["' + table_access + '"]' # side module tables were merged into the parent, we need to access the global one
        prelude = '''
  if (x < 0 || x >= %s.length) { Module.printErr("Function table mask error (out of range)"); %s ; abort(x) }''' % (table_access, get_function_pointer_error(sig))
        asm_setup += '''
function ftCall_%s(%s) {%s
  return %s[x](%s);
}
''' % (sig, ', '.join(full_args), prelude, table_access, ', '.join(args))
        basic_funcs.append('ftCall_%s' % sig)

    def quote(prop):
      if settings['CLOSURE_COMPILER'] == 2:
        return "'" + prop + "'"
      else:
        return prop

    def access_quote(prop):
      if settings['CLOSURE_COMPILER'] == 2:
        return "['" + prop + "']"
      else:
        return '.' + prop

    # calculate exports
    exported_implemented_functions = list(exported_implemented_functions) + metadata['initializers']
    exported_implemented_functions.append('runPostSets')
    if settings['ALLOW_MEMORY_GROWTH']:
      exported_implemented_functions.append('_emscripten_replace_memory')
    all_exported = exported_implemented_functions + asm_runtime_funcs + function_tables
    exported_implemented_functions = list(set(exported_implemented_functions))
    if settings['EMULATED_FUNCTION_POINTERS']:
      all_exported = list(set(all_exported).union(in_table))
    exports = []
    for export in all_exported:
      exports.append(quote(export) + ": " + export)
    exports = '{ ' + ', '.join(exports) + ' }'
    # calculate globals
    try:
      del forwarded_json['Variables']['globals']['_llvm_global_ctors'] # not a true variable
    except:
      pass
    if not settings['RELOCATABLE']:
      global_vars = metadata['externs']
    else:
      global_vars = [] # linkable code accesses globals through function calls
    global_funcs = list(set([key for key, value in forwarded_json['Functions']['libraryFunctions'].iteritems() if value != 2]).difference(set(global_vars)).difference(implemented_functions))
    if settings['RELOCATABLE']:
      global_funcs += ['g$' + extern for extern in metadata['externs']]
      side = 'parent' if settings['SIDE_MODULE'] else ''
      def check(extern):
        if settings['ASSERTIONS']: return 'assert(' + side + 'Module["' + extern + '"]);'
        return ''
      for extern in metadata['externs']:
        asm_setup += 'var g$' + extern + ' = function() { ' + check(extern) + ' return ' + side + 'Module["' + extern + '"] };\n'
    def math_fix(g):
      return g if not g.startswith('Math_') else g.split('_')[1]
    asm_global_funcs = ''.join(['  var ' + g.replace('.', '_') + '=global' + access_quote(g) + ';\n' for g in maths]);
    asm_global_funcs += ''.join(['  var ' + g + '=env' + access_quote(math_fix(g)) + ';\n' for g in basic_funcs + global_funcs])
    if metadata['simd']:
      asm_global_funcs += ''.join(['  var SIMD_' + ty + '=global' + access_quote('SIMD') + access_quote(ty) + ';\n' for ty in simdtypes])
      asm_global_funcs += ''.join(['  var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdinttypes for g in simdintfuncs])
      asm_global_funcs += ''.join(['  var SIMD_' + ty + '_' + g + '=SIMD_' + ty + access_quote(g) + ';\n' for ty in simdfloattypes for g in simdfloatfuncs])
    if settings['USE_PTHREADS']:
#      asm_global_funcs += ''.join(['  var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'exchange', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor', 'fence']])
# TODO: Once bug https://bugzilla.mozilla.org/show_bug.cgi?id=1141986 is implemented, replace the following line with the above one!
      asm_global_funcs += ''.join(['  var Atomics_' + ty + '=global' + access_quote('Atomics') + access_quote(ty) + ';\n' for ty in ['load', 'store', 'compareExchange', 'add', 'sub', 'and', 'or', 'xor', 'fence']])
    asm_global_vars = ''.join(['  var ' + g + '=env' + access_quote(g) + '|0;\n' for g in basic_vars + global_vars])

    # sent data
    the_global = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in fundamentals]) + ' }'
    sending = '{ ' + ', '.join(['"' + math_fix(s) + '": ' + s for s in basic_funcs + global_funcs + basic_vars + basic_float_vars + global_vars]) + ' }'
    # received
    receiving = ''
    if settings['ASSERTIONS']:
      # assert on the runtime being in a valid state when calling into compiled code. The only exceptions are
      # some support code
      receiving = '\n'.join(['var real_' + s + ' = asm["' + s + '"]; asm["' + s + '''"] = function() {
assert(runtimeInitialized, 'you need to wait for the runtime to be ready (e.g. wait for main() to be called)');
assert(!runtimeExited, 'the runtime was exited (use NO_EXIT_RUNTIME to keep it alive after main() exits)');
return real_''' + s + '''.apply(null, arguments);
};
''' for s in exported_implemented_functions if s not in ['_memcpy', '_memset', 'runPostSets', '_emscripten_replace_memory']])

    if not settings['SWAPPABLE_ASM_MODULE']:
      receiving += ';\n'.join(['var ' + s + ' = Module["' + s + '"] = asm["' + s + '"]' for s in exported_implemented_functions + function_tables])
    else:
      receiving += 'Module["asm"] = asm;\n' + ';\n'.join(['var ' + s + ' = Module["' + s + '"] = function() { return Module["asm"]["' + s + '"].apply(null, arguments) }' for s in exported_implemented_functions + function_tables])
    receiving += ';\n'

    if settings['EXPORT_FUNCTION_TABLES']:
      for table in last_forwarded_json['Functions']['tables'].values():
        tableName = table.split()[1]
        table = table.replace('var ' + tableName, 'var ' + tableName + ' = Module["' + tableName + '"]')
        receiving += table + '\n'

    # finalize

    if DEBUG: logging.debug('asm text sizes' + str([map(len, funcs_js), len(asm_setup), len(asm_global_vars), len(asm_global_funcs), len(pre_tables), len('\n'.join(function_tables_impls)), len(function_tables_defs.replace('\n', '\n  ')), len(exports), len(the_global), len(sending), len(receiving)]))

    if not settings.get('EMULATED_FUNCTION_POINTERS'):
      final_function_tables = '\n'.join(function_tables_impls) + '\n' + function_tables_defs
    else:
      asm_setup += '\n' + '\n'.join(function_tables_impls) + '\n'
      receiving += '\n' + function_tables_defs + '\n' + ''.join(['Module["dynCall_%s"] = dynCall_%s\n' % (sig, sig) for sig in last_forwarded_json['Functions']['tables']])
      for sig in last_forwarded_json['Functions']['tables'].keys():
        name = 'FUNCTION_TABLE_' + sig
        fullname = name if not settings['SIDE_MODULE'] else ('SIDE_' + name)
        receiving += 'Module["' + name + '"] = ' + fullname + ';\n'
      final_function_tables = '\n// EMSCRIPTEN_END_FUNCS\n'

    if settings['RELOCATABLE']:
      receiving += '''
var NAMED_GLOBALS = { %s };
for (var named in NAMED_GLOBALS) {
  Module['_' + named] = gb + NAMED_GLOBALS[named];
}
Module['NAMED_GLOBALS'] = NAMED_GLOBALS;
''' % ', '.join('"' + k + '": ' + str(v) for k, v in metadata['namedGlobals'].iteritems())

      receiving += ''.join(["Module['%s'] = Module['%s']\n" % (k, v) for k, v in metadata['aliases'].iteritems()])

    funcs_js = ['''
%s
Module%s = %s;
Module%s = %s;
// EMSCRIPTEN_START_ASM
var asm = (function(global, env, buffer) {
  %s
  %s
''' % (asm_setup,
       access_quote('asmGlobalArg'), the_global,
       access_quote('asmLibraryArg'), sending,
       "'use asm';" if not metadata.get('hasInlineJS') and settings['ASM_JS'] == 1 else "'almost asm';", '''
  var HEAP8 = new global%s(buffer);
  var HEAP16 = new global%s(buffer);
  var HEAP32 = new global%s(buffer);
  var HEAPU8 = new global%s(buffer);
  var HEAPU16 = new global%s(buffer);
  var HEAPU32 = new global%s(buffer);
  var HEAPF32 = new global%s(buffer);
  var HEAPF64 = new global%s(buffer);
''' % (access_quote('SharedInt8Array' if settings['USE_PTHREADS'] else 'Int8Array'),
     access_quote('SharedInt16Array' if settings['USE_PTHREADS'] else 'Int16Array'),
     access_quote('SharedInt32Array' if settings['USE_PTHREADS'] else 'Int32Array'),
     access_quote('SharedUint8Array' if settings['USE_PTHREADS'] else 'Uint8Array'),
     access_quote('SharedUint16Array' if settings['USE_PTHREADS'] else 'Uint16Array'),
     access_quote('SharedUint32Array' if settings['USE_PTHREADS'] else 'Uint32Array'),
     access_quote('SharedFloat32Array' if settings['USE_PTHREADS'] else 'Float32Array'),
     access_quote('SharedFloat64Array' if settings['USE_PTHREADS'] else 'Float64Array'))
     if not settings['ALLOW_MEMORY_GROWTH'] else '''
  var Int8View = global%s;
  var Int16View = global%s;
  var Int32View = global%s;
  var Uint8View = global%s;
  var Uint16View = global%s;
  var Uint32View = global%s;
  var Float32View = global%s;
  var Float64View = global%s;
  var HEAP8 = new Int8View(buffer);
  var HEAP16 = new Int16View(buffer);
  var HEAP32 = new Int32View(buffer);
  var HEAPU8 = new Uint8View(buffer);
  var HEAPU16 = new Uint16View(buffer);
  var HEAPU32 = new Uint32View(buffer);
  var HEAPF32 = new Float32View(buffer);
  var HEAPF64 = new Float64View(buffer);
  var byteLength = global.byteLength;
''' % (access_quote('Int8Array'),
     access_quote('Int16Array'),
     access_quote('Int32Array'),
     access_quote('Uint8Array'),
     access_quote('Uint16Array'),
     access_quote('Uint32Array'),
     access_quote('Float32Array'),
     access_quote('Float64Array'))) + '\n' + asm_global_vars + ('''
  var __THREW__ = 0;
  var threwValue = 0;
  var setjmpId = 0;
  var undef = 0;
  var nan = global%s, inf = global%s;
  var tempInt = 0, tempBigInt = 0, tempBigIntP = 0, tempBigIntS = 0, tempBigIntR = 0.0, tempBigIntI = 0, tempBigIntD = 0, tempValue = 0, tempDouble = 0.0;
''' % (access_quote('NaN'), access_quote('Infinity'))) + ''.join(['''
  var tempRet%d = 0;''' % i for i in range(10)]) + '\n' + asm_global_funcs] + \
  ['  var tempFloat = %s;\n' % ('Math_fround(0)' if provide_fround else '0.0')] + \
  ['  var asyncState = 0;\n' if settings.get('EMTERPRETIFY_ASYNC') else ''] + \
  (['  const f0 = Math_fround(0);\n'] if provide_fround else []) + \
  ['' if not settings['ALLOW_MEMORY_GROWTH'] else '''
function _emscripten_replace_memory(newBuffer) {
  if ((byteLength(newBuffer) & 0xffffff || byteLength(newBuffer) <= 0xffffff) || byteLength(newBuffer) > 0x80000000) return false;
  HEAP8 = new Int8View(newBuffer);
  HEAP16 = new Int16View(newBuffer);
  HEAP32 = new Int32View(newBuffer);
  HEAPU8 = new Uint8View(newBuffer);
  HEAPU16 = new Uint16View(newBuffer);
  HEAPU32 = new Uint32View(newBuffer);
  HEAPF32 = new Float32View(newBuffer);
  HEAPF64 = new Float64View(newBuffer);
  buffer = newBuffer;
  return true;
}
'''] + ['''
// EMSCRIPTEN_START_FUNCS
function stackAlloc(size) {
  size = size|0;
  var ret = 0;
  ret = STACKTOP;
  STACKTOP = (STACKTOP + size)|0;
  STACKTOP = (STACKTOP + 15)&-16;
''' + ('if ((STACKTOP|0) >= (STACK_MAX|0)) abort();\n' if settings['ASSERTIONS'] else '') + '''
  return ret|0;
}
function stackSave() {
  return STACKTOP|0;
}
function stackRestore(top) {
  top = top|0;
  STACKTOP = top;
}
function establishStackSpace(stackBase, stackMax) {
  stackBase = stackBase|0;
  stackMax = stackMax|0;
  STACKTOP = stackBase;
  STACK_MAX = stackMax;
}
''' + ('''
function setAsync() {
  ___async = 1;
}''' if need_asyncify else '') + ('''
function emterpret(pc) { // this will be replaced when the emterpreter code is generated; adding it here allows validation until then
  pc = pc | 0;
  assert(0);
}
''' if settings['EMTERPRETIFY'] else '') + ('''
function setAsyncState(x) {
  x = x | 0;
  asyncState = x;
}
function emtStackSave() {
  return EMTSTACKTOP|0;
}
''' if settings['EMTERPRETIFY_ASYNC'] else '') + '''
function setThrew(threw, value) {
  threw = threw|0;
  value = value|0;
  if ((__THREW__|0) == 0) {
    __THREW__ = threw;
    threwValue = value;
  }
}
function copyTempFloat(ptr) {
  ptr = ptr|0;
  HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0];
  HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0];
  HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0];
  HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0];
}
function copyTempDouble(ptr) {
  ptr = ptr|0;
  HEAP8[tempDoublePtr>>0] = HEAP8[ptr>>0];
  HEAP8[tempDoublePtr+1>>0] = HEAP8[ptr+1>>0];
  HEAP8[tempDoublePtr+2>>0] = HEAP8[ptr+2>>0];
  HEAP8[tempDoublePtr+3>>0] = HEAP8[ptr+3>>0];
  HEAP8[tempDoublePtr+4>>0] = HEAP8[ptr+4>>0];
  HEAP8[tempDoublePtr+5>>0] = HEAP8[ptr+5>>0];
  HEAP8[tempDoublePtr+6>>0] = HEAP8[ptr+6>>0];
  HEAP8[tempDoublePtr+7>>0] = HEAP8[ptr+7>>0];
}
'''] + ['''
function setTempRet0(value) {
  value = value|0;
  tempRet0 = value;
}
function getTempRet0() {
  return tempRet0|0;
}
''' if not settings['RELOCATABLE'] else ''] + funcs_js + ['''
  %s

  return %s;
})
// EMSCRIPTEN_END_ASM
(%s, %s, buffer);
%s;
''' % (pre_tables + final_function_tables, exports,
       'Module' + access_quote('asmGlobalArg'),
       'Module' + access_quote('asmLibraryArg'),
       receiving)]

    if not settings.get('SIDE_MODULE'):
      funcs_js.append('''
Runtime.stackAlloc = asm['stackAlloc'];
Runtime.stackSave = asm['stackSave'];
Runtime.stackRestore = asm['stackRestore'];
Runtime.establishStackSpace = asm['establishStackSpace'];
''')
    if not settings['RELOCATABLE']:
      funcs_js.append('''
Runtime.setTempRet0 = asm['setTempRet0'];
Runtime.getTempRet0 = asm['getTempRet0'];
''')

    # Set function table masks
    masks = {}
    max_mask = 0
    for sig, table in last_forwarded_json['Functions']['tables'].iteritems():
      mask = table.count(',')
      masks[sig] = str(mask)
      max_mask = max(mask, max_mask)
    def function_table_maskize(js, masks):
      def fix(m):
        sig = m.groups(0)[0]
        return masks[sig]
      return re.sub(r'{{{ FTM_([\w\d_$]+) }}}', lambda m: fix(m), js) # masks[m.groups(0)[0]]
    funcs_js = map(lambda js: function_table_maskize(js, masks), funcs_js)

    if settings['SIDE_MODULE']:
      funcs_js.append('''
Runtime.registerFunctions(%(sigs)s, Module);
''' % { 'sigs': str(map(str, last_forwarded_json['Functions']['tables'].keys())) }) 

    for i in range(len(funcs_js)): # do this loop carefully to save memory
      if WINDOWS: funcs_js[i] = funcs_js[i].replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n!
      outfile.write(funcs_js[i])
    funcs_js = None

    if WINDOWS: post = post.replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n!
    outfile.write(post)

    outfile.close()

    if DEBUG: logging.debug('  emscript: final python processing took %s seconds' % (time.time() - t))

    success = True
Example #17
0
def main():
    data_files = []
    export_name = 'Module'
    leading = ''
    has_preloaded = False
    plugins = []
    jsoutput = None
    from_emcc = False
    force = True
    # If set to True, IndexedDB (IDBFS in library_idbfs.js) is used to locally
    # cache VFS XHR so that subsequent page loads can read the data from the
    # offline cache instead.
    use_preload_cache = False
    indexeddb_name = 'EM_PRELOAD_CACHE'
    # If set to True, the blob received from XHR is moved to the Emscripten HEAP,
    # optimizing for mmap() performance (if ALLOW_MEMORY_GROWTH=0).
    # If set to False, the XHR blob is kept intact, and fread()s etc. are performed
    # directly to that data. This optimizes for minimal memory usage and fread()
    # performance.
    heap_copy = True
    # If set to True, the package metadata is stored separately from js-output
    # file which makes js-output file immutable to the package content changes.
    # If set to False, the package metadata is stored inside the js-output file
    # which makes js-output file to mutate on each invocation of this packager tool.
    separate_metadata = False
    lz4 = False
    use_preload_plugins = False

    for arg in sys.argv[2:]:
        if arg == '--preload':
            has_preloaded = True
            leading = 'preload'
        elif arg == '--embed':
            leading = 'embed'
        elif arg == '--exclude':
            leading = 'exclude'
        elif arg == '--no-force':
            force = False
            leading = ''
        elif arg == '--use-preload-cache':
            use_preload_cache = True
            leading = ''
        elif arg.startswith('--indexedDB-name'):
            indexeddb_name = arg.split('=', 1)[1] if '=' in arg else None
            leading = ''
        elif arg == '--no-heap-copy':
            heap_copy = False
            leading = ''
        elif arg == '--separate-metadata':
            separate_metadata = True
            leading = ''
        elif arg == '--lz4':
            lz4 = True
            leading = ''
        elif arg == '--use-preload-plugins':
            use_preload_plugins = True
            leading = ''
        elif arg.startswith('--js-output'):
            jsoutput = arg.split('=', 1)[1] if '=' in arg else None
            leading = ''
        elif arg.startswith('--export-name'):
            if '=' in arg:
                export_name = arg.split('=', 1)[1]
            leading = ''
        elif arg.startswith('--from-emcc'):
            from_emcc = True
            leading = ''
        elif arg.startswith('--plugin'):
            plugin = open(arg.split('=', 1)[1], 'r').read()
            eval(plugin)  # should append itself to plugins
            leading = ''
        elif leading == 'preload' or leading == 'embed':
            mode = leading
            # position of @ if we're doing 'src@dst'. '__' is used to keep the index
            # same with the original if they escaped with '@@'.
            at_position = arg.replace('@@', '__').find('@')
            # '@@' in input string means there is an actual @ character, a single '@'
            # means the 'src@dst' notation.
            uses_at_notation = (at_position != -1)

            if uses_at_notation:
                srcpath = arg[0:at_position].replace('@@',
                                                     '@')  # split around the @
                dstpath = arg[at_position + 1:].replace('@@', '@')
            else:
                # Use source path as destination path.
                srcpath = dstpath = arg.replace('@@', '@')
            if os.path.isfile(srcpath) or os.path.isdir(srcpath):
                data_files.append({
                    'srcpath': srcpath,
                    'dstpath': dstpath,
                    'mode': mode,
                    'explicit_dst_path': uses_at_notation
                })
            else:
                print('Warning: ' + arg + ' does not exist, ignoring.',
                      file=sys.stderr)
        elif leading == 'exclude':
            excluded_patterns.append(arg)
        else:
            print('Unknown parameter:', arg, file=sys.stderr)
            sys.exit(1)

    if (not force) and not data_files:
        has_preloaded = False
    if not has_preloaded or jsoutput is None:
        assert not separate_metadata, (
            'cannot separate-metadata without both --preloaded files '
            'and a specified --js-output')

    if not from_emcc:
        print(
            'Remember to build the main file with  -s FORCE_FILESYSTEM=1  '
            'so that it includes support for loading this file package',
            file=sys.stderr)

    ret = ''
    # emcc.py will add this to the output itself, so it is only needed for
    # standalone calls
    if not from_emcc:
        ret = '''
  var Module = typeof %(EXPORT_NAME)s !== 'undefined' ? %(EXPORT_NAME)s : {};
  ''' % {
            "EXPORT_NAME": export_name
        }

    ret += '''
  if (!Module.expectedDataFileDownloads) {
    Module.expectedDataFileDownloads = 0;
    Module.finishedDataFileDownloads = 0;
  }
  Module.expectedDataFileDownloads++;
  (function() {
   var loadPackage = function(metadata) {
  '''

    code = '''
      function assert(check, msg) {
        if (!check) throw msg + new Error().stack;
      }
  '''

    for file_ in data_files:
        if not should_ignore(file_['srcpath']):
            if os.path.isdir(file_['srcpath']):
                add(file_['mode'], file_['srcpath'], file_['dstpath'])
            else:
                new_data_files.append(file_)
    data_files = [
        file_ for file_ in new_data_files
        if not os.path.isdir(file_['srcpath'])
    ]
    if len(data_files) == 0:
        print('Nothing to do!', file=sys.stderr)
        sys.exit(1)

    # Absolutize paths, and check that they make sense
    # os.getcwd() always returns the hard path with any symbolic links resolved,
    # even if we cd'd into a symbolic link.
    curr_abspath = os.path.abspath(os.getcwd())

    for file_ in data_files:
        if not file_['explicit_dst_path']:
            # This file was not defined with src@dst, so we inferred the destination
            # from the source. In that case, we require that the destination not be
            # under the current location
            path = file_['dstpath']
            # Use os.path.realpath to resolve any symbolic links to hard paths,
            # to match the structure in curr_abspath.
            abspath = os.path.realpath(os.path.abspath(path))
            if DEBUG:
                print(path, abspath, curr_abspath, file=sys.stderr)
            if not abspath.startswith(curr_abspath):
                print(
                    'Error: Embedding "%s" which is below the current directory '
                    '"%s". This is invalid since the current directory becomes the '
                    'root that the generated code will see' %
                    (path, curr_abspath),
                    file=sys.stderr)
                sys.exit(1)
            file_['dstpath'] = abspath[len(curr_abspath) + 1:]
            if os.path.isabs(path):
                print(
                    'Warning: Embedding an absolute file/directory name "%s" to the '
                    'virtual filesystem. The file will be made available in the '
                    'relative path "%s". You can use the explicit syntax '
                    '--preload-file srcpath@dstpath to explicitly specify the target '
                    'location the absolute source path should be directed to.'
                    % (path, file_['dstpath']),
                    file=sys.stderr)

    for file_ in data_files:
        # name in the filesystem, native and emulated
        file_['dstpath'] = file_['dstpath'].replace(os.path.sep, '/')
        # If user has submitted a directory name as the destination but omitted
        # the destination filename, use the filename from source file
        if file_['dstpath'].endswith('/'):
            file_['dstpath'] = file_['dstpath'] + os.path.basename(
                file_['srcpath'])
        # make destination path always relative to the root
        file_['dstpath'] = posixpath.normpath(
            os.path.join('/', file_['dstpath']))
        if DEBUG:
            print('Packaging file "%s" to VFS in path "%s".' %
                  (file_['srcpath'], file_['dstpath']),
                  file=sys.stderr)

    # Remove duplicates (can occur naively, for example preload dir/, preload dir/subdir/)
    seen = {}

    def was_seen(name):
        if seen.get(name):
            return True
        seen[name] = 1
        return False

    data_files = [
        file_ for file_ in data_files if not was_seen(file_['dstpath'])
    ]

    if AV_WORKAROUND:
        random.shuffle(data_files)

    # Apply plugins
    for file_ in data_files:
        for plugin in plugins:
            plugin(file_)

    metadata = {'files': []}

    # Set up folders
    partial_dirs = []
    for file_ in data_files:
        dirname = os.path.dirname(file_['dstpath'])
        dirname = dirname.lstrip(
            '/')  # absolute paths start with '/', remove that
        if dirname != '':
            parts = dirname.split('/')
            for i in range(len(parts)):
                partial = '/'.join(parts[:i + 1])
                if partial not in partial_dirs:
                    code += (
                        '''Module['FS_createPath']('/%s', '%s', true, true);\n'''
                        % ('/'.join(parts[:i]), parts[i]))
                    partial_dirs.append(partial)

    if has_preloaded:
        # Bundle all datafiles into one archive. Avoids doing lots of simultaneous
        # XHRs which has overhead.
        data = open(data_target, 'wb')
        start = 0
        for file_ in data_files:
            file_['data_start'] = start
            curr = open(file_['srcpath'], 'rb').read()
            file_['data_end'] = start + len(curr)
            if AV_WORKAROUND:
                curr += '\x00'
            start += len(curr)
            data.write(curr)
        data.close()
        # TODO: sha256sum on data_target
        if start > 256 * 1024 * 1024:
            print(
                'warning: file packager is creating an asset bundle of %d MB. '
                'this is very large, and browsers might have trouble loading it. '
                'see https://hacks.mozilla.org/2015/02/synchronous-execution-and-filesystem-access-in-emscripten/'
                % (start / (1024 * 1024)),
                file=sys.stderr)

        create_preloaded = '''
          Module['FS_createPreloadedFile'](this.name, null, byteArray, true, true, function() {
            Module['removeRunDependency']('fp ' + that.name);
          }, function() {
            if (that.audio) {
              Module['removeRunDependency']('fp ' + that.name); // workaround for chromium bug 124926 (still no audio with this, but at least we don't hang)
            } else {
              err('Preloading file ' + that.name + ' failed');
            }
          }, false, true); // canOwn this data in the filesystem, it is a slide into the heap that will never change
  '''
        create_data = '''
          Module['FS_createDataFile'](this.name, null, byteArray, true, true, true); // canOwn this data in the filesystem, it is a slide into the heap that will never change
          Module['removeRunDependency']('fp ' + that.name);
  '''

        # Data requests - for getting a block of data out of the big archive - have
        # a similar API to XHRs
        code += '''
      function DataRequest(start, end, audio) {
        this.start = start;
        this.end = end;
        this.audio = audio;
      }
      DataRequest.prototype = {
        requests: {},
        open: function(mode, name) {
          this.name = name;
          this.requests[name] = this;
          Module['addRunDependency']('fp ' + this.name);
        },
        send: function() {},
        onload: function() {
          var byteArray = this.byteArray.subarray(this.start, this.end);
          this.finish(byteArray);
        },
        finish: function(byteArray) {
          var that = this;
  %s
          this.requests[this.name] = null;
        }
      };
  %s
    ''' % (create_preloaded if use_preload_plugins else create_data, '''
          var files = metadata['files'];
          for (var i = 0; i < files.length; ++i) {
            new DataRequest(files[i]['start'], files[i]['end'], files[i]['audio']).open('GET', files[i]['filename']);
          }
  ''' if not lz4 else '')

    counter = 0
    for file_ in data_files:
        filename = file_['dstpath']
        dirname = os.path.dirname(filename)
        basename = os.path.basename(filename)
        if file_['mode'] == 'embed':
            # Embed
            data = list(bytearray(open(file_['srcpath'], 'rb').read()))
            code += '''var fileData%d = [];\n''' % counter
            if data:
                parts = []
                chunk_size = 10240
                start = 0
                while start < len(data):
                    parts.append(
                        '''fileData%d.push.apply(fileData%d, %s);\n''' %
                        (counter, counter, str(
                            data[start:start + chunk_size])))
                    start += chunk_size
                code += ''.join(parts)
            code += (
                '''Module['FS_createDataFile']('%s', '%s', fileData%d, true, true, false);\n'''
                % (dirname, basename, counter))
            counter += 1
        elif file_['mode'] == 'preload':
            # Preload
            counter += 1
            metadata['files'].append({
                'filename':
                file_['dstpath'],
                'start':
                file_['data_start'],
                'end':
                file_['data_end'],
                'audio':
                1 if filename[-4:] in AUDIO_SUFFIXES else 0,
            })
        else:
            assert 0

    if has_preloaded:
        if not lz4:
            # Get the big archive and split it up
            if heap_copy:
                use_data = '''
          // copy the entire loaded file into a spot in the heap. Files will refer to slices in that. They cannot be freed though
          // (we may be allocating before malloc is ready, during startup).
          var ptr = Module['getMemory'](byteArray.length);
          Module['HEAPU8'].set(byteArray, ptr);
          DataRequest.prototype.byteArray = Module['HEAPU8'].subarray(ptr, ptr+byteArray.length);
    '''
            else:
                use_data = '''
          // Reuse the bytearray from the XHR as the source for file reads.
          DataRequest.prototype.byteArray = byteArray;
    '''
            use_data += '''
            var files = metadata['files'];
            for (var i = 0; i < files.length; ++i) {
              DataRequest.prototype.requests[files[i].filename].onload();
            }
      '''
            use_data += (
                "          Module['removeRunDependency']('datafile_%s');\n" %
                shared.JS.escape_for_js_string(data_target))

        else:
            # LZ4FS usage
            temp = data_target + '.orig'
            shutil.move(data_target, temp)
            meta = run_js(shared.path_from_root('tools', 'lz4-compress.js'),
                          shared.NODE_JS, [
                              shared.path_from_root('src', 'mini-lz4.js'),
                              temp, data_target
                          ],
                          stdout=PIPE)
            os.unlink(temp)
            use_data = '''
            var compressedData = %s;
            compressedData['data'] = byteArray;
            assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with  -s LZ4=1  ?');
            LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData });
            Module['removeRunDependency']('datafile_%s');
      ''' % (meta, shared.JS.escape_for_js_string(data_target))

        package_uuid = uuid.uuid4()
        package_name = data_target
        remote_package_size = os.path.getsize(package_name)
        remote_package_name = os.path.basename(package_name)
        ret += r'''
      var PACKAGE_PATH;
      if (typeof window === 'object') {
        PACKAGE_PATH = window['encodeURIComponent'](window.location.pathname.toString().substring(0, window.location.pathname.toString().lastIndexOf('/')) + '/');
      } else if (typeof location !== 'undefined') {
        // worker
        PACKAGE_PATH = encodeURIComponent(location.pathname.toString().substring(0, location.pathname.toString().lastIndexOf('/')) + '/');
      } else {
        throw 'using preloaded data can only be done on a web page or in a web worker';
      }
      var PACKAGE_NAME = '%s';
      var REMOTE_PACKAGE_BASE = '%s';
      if (typeof Module['locateFilePackage'] === 'function' && !Module['locateFile']) {
        Module['locateFile'] = Module['locateFilePackage'];
        err('warning: you defined Module.locateFilePackage, that has been renamed to Module.locateFile (using your locateFilePackage for now)');
      }
      var REMOTE_PACKAGE_NAME = Module['locateFile'] ? Module['locateFile'](REMOTE_PACKAGE_BASE, '') : REMOTE_PACKAGE_BASE;
    ''' % (shared.JS.escape_for_js_string(data_target),
           shared.JS.escape_for_js_string(remote_package_name))
        metadata['remote_package_size'] = remote_package_size
        metadata['package_uuid'] = str(package_uuid)
        ret += '''
      var REMOTE_PACKAGE_SIZE = metadata['remote_package_size'];
      var PACKAGE_UUID = metadata['package_uuid'];
    '''

        if use_preload_cache:
            code += r'''
        var indexedDB = window.indexedDB || window.mozIndexedDB || window.webkitIndexedDB || window.msIndexedDB;
        var IDB_RO = "readonly";
        var IDB_RW = "readwrite";
        var DB_NAME = "''' + indexeddb_name + '''";
        var DB_VERSION = 1;
        var METADATA_STORE_NAME = 'METADATA';
        var PACKAGE_STORE_NAME = 'PACKAGES';
        function openDatabase(callback, errback) {
          try {
            var openRequest = indexedDB.open(DB_NAME, DB_VERSION);
          } catch (e) {
            return errback(e);
          }
          openRequest.onupgradeneeded = function(event) {
            var db = event.target.result;

            if(db.objectStoreNames.contains(PACKAGE_STORE_NAME)) {
              db.deleteObjectStore(PACKAGE_STORE_NAME);
            }
            var packages = db.createObjectStore(PACKAGE_STORE_NAME);

            if(db.objectStoreNames.contains(METADATA_STORE_NAME)) {
              db.deleteObjectStore(METADATA_STORE_NAME);
            }
            var metadata = db.createObjectStore(METADATA_STORE_NAME);
          };
          openRequest.onsuccess = function(event) {
            var db = event.target.result;
            callback(db);
          };
          openRequest.onerror = function(error) {
            errback(error);
          };
        };

        // This is needed as chromium has a limit on per-entry files in IndexedDB
        // https://cs.chromium.org/chromium/src/content/renderer/indexed_db/webidbdatabase_impl.cc?type=cs&sq=package:chromium&g=0&l=177
        // https://cs.chromium.org/chromium/src/out/Debug/gen/third_party/blink/public/mojom/indexeddb/indexeddb.mojom.h?type=cs&sq=package:chromium&g=0&l=60
        // We set the chunk size to 64MB to stay well-below the limit
        var CHUNK_SIZE = 64 * 1024 * 1024;

        function cacheRemotePackage(
          db,
          packageName,
          packageData,
          packageMeta,
          callback,
          errback
        ) {
          var transactionPackages = db.transaction([PACKAGE_STORE_NAME], IDB_RW);
          var packages = transactionPackages.objectStore(PACKAGE_STORE_NAME);
          var chunkSliceStart = 0;
          var nextChunkSliceStart = 0;
          var chunkCount = Math.ceil(packageData.byteLength / CHUNK_SIZE);
          var finishedChunks = 0;
          for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
            nextChunkSliceStart += CHUNK_SIZE;
            var putPackageRequest = packages.put(
              packageData.slice(chunkSliceStart, nextChunkSliceStart),
              'package/' + packageName + '/' + chunkId
            );
            chunkSliceStart = nextChunkSliceStart;
            putPackageRequest.onsuccess = function(event) {
              finishedChunks++;
              if (finishedChunks == chunkCount) {
                var transaction_metadata = db.transaction(
                  [METADATA_STORE_NAME],
                  IDB_RW
                );
                var metadata = transaction_metadata.objectStore(METADATA_STORE_NAME);
                var putMetadataRequest = metadata.put(
                  {
                    'uuid': packageMeta.uuid,
                    'chunkCount': chunkCount
                  },
                  'metadata/' + packageName
                );
                putMetadataRequest.onsuccess = function(event) {
                  callback(packageData);
                };
                putMetadataRequest.onerror = function(error) {
                  errback(error);
                };
              }
            };
            putPackageRequest.onerror = function(error) {
              errback(error);
            };
          }
        }

        /* Check if there's a cached package, and if so whether it's the latest available */
        function checkCachedPackage(db, packageName, callback, errback) {
          var transaction = db.transaction([METADATA_STORE_NAME], IDB_RO);
          var metadata = transaction.objectStore(METADATA_STORE_NAME);
          var getRequest = metadata.get('metadata/' + packageName);
          getRequest.onsuccess = function(event) {
            var result = event.target.result;
            if (!result) {
              return callback(false, null);
            } else {
              return callback(PACKAGE_UUID === result['uuid'], result);
            }
          };
          getRequest.onerror = function(error) {
            errback(error);
          };
        }

        function fetchCachedPackage(db, packageName, metadata, callback, errback) {
          var transaction = db.transaction([PACKAGE_STORE_NAME], IDB_RO);
          var packages = transaction.objectStore(PACKAGE_STORE_NAME);

          var chunksDone = 0;
          var totalSize = 0;
          var chunkCount = metadata['chunkCount'];
          var chunks = new Array(chunkCount);

          for (var chunkId = 0; chunkId < chunkCount; chunkId++) {
            var getRequest = packages.get('package/' + packageName + '/' + chunkId);
            getRequest.onsuccess = function(event) {
              // If there's only 1 chunk, there's nothing to concatenate it with so we can just return it now
              if (chunkCount == 1) {
                callback(event.target.result);
              } else {
                chunksDone++;
                totalSize += event.target.result.byteLength;
                chunks.push(event.target.result);
                if (chunksDone == chunkCount) {
                  if (chunksDone == 1) {
                    callback(event.target.result);
                  } else {
                    var tempTyped = new Uint8Array(totalSize);
                    var byteOffset = 0;
                    for (var chunkId in chunks) {
                      var buffer = chunks[chunkId];
                      tempTyped.set(new Uint8Array(buffer), byteOffset);
                      byteOffset += buffer.byteLength;
                      buffer = undefined;
                    }
                    chunks = undefined;
                    callback(tempTyped.buffer);
                    tempTyped = undefined;
                  }
                }
              }
            };
            getRequest.onerror = function(error) {
              errback(error);
            };
          }
        }
      '''

        ret += r'''
      function fetchRemotePackage(packageName, packageSize, callback, errback) {
        var xhr = new XMLHttpRequest();
        xhr.open('GET', packageName, true);
        xhr.responseType = 'arraybuffer';
        xhr.onprogress = function(event) {
          var url = packageName;
          var size = packageSize;
          if (event.total) size = event.total;
          if (event.loaded) {
            if (!xhr.addedTotal) {
              xhr.addedTotal = true;
              if (!Module.dataFileDownloads) Module.dataFileDownloads = {};
              Module.dataFileDownloads[url] = {
                loaded: event.loaded,
                total: size
              };
            } else {
              Module.dataFileDownloads[url].loaded = event.loaded;
            }
            var total = 0;
            var loaded = 0;
            var num = 0;
            for (var download in Module.dataFileDownloads) {
            var data = Module.dataFileDownloads[download];
              total += data.total;
              loaded += data.loaded;
              num++;
            }
            total = Math.ceil(total * Module.expectedDataFileDownloads/num);
            if (Module['setStatus']) Module['setStatus']('Downloading data... (' + loaded + '/' + total + ')');
          } else if (!Module.dataFileDownloads) {
            if (Module['setStatus']) Module['setStatus']('Downloading data...');
          }
        };
        xhr.onerror = function(event) {
          throw new Error("NetworkError for: " + packageName);
        }
        xhr.onload = function(event) {
          if (xhr.status == 200 || xhr.status == 304 || xhr.status == 206 || (xhr.status == 0 && xhr.response)) { // file URLs can return 0
            var packageData = xhr.response;
            callback(packageData);
          } else {
            throw new Error(xhr.statusText + " : " + xhr.responseURL);
          }
        };
        xhr.send(null);
      };

      function handleError(error) {
        console.error('package error:', error);
      };
    '''

        code += r'''
      function processPackageData(arrayBuffer) {
        Module.finishedDataFileDownloads++;
        assert(arrayBuffer, 'Loading data file failed.');
        assert(arrayBuffer instanceof ArrayBuffer, 'bad input to processPackageData');
        var byteArray = new Uint8Array(arrayBuffer);
        var curr;
        %s
      };
      Module['addRunDependency']('datafile_%s');
    ''' % (use_data, shared.JS.escape_for_js_string(data_target))
        # use basename because from the browser's point of view,
        # we need to find the datafile in the same dir as the html file

        code += r'''
      if (!Module.preloadResults) Module.preloadResults = {};
    '''

        if use_preload_cache:
            code += r'''
        function preloadFallback(error) {
          console.error(error);
          console.error('falling back to default preload behavior');
          fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, processPackageData, handleError);
        };

        openDatabase(
          function(db) {
            checkCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME,
              function(useCached, metadata) {
                Module.preloadResults[PACKAGE_NAME] = {fromCache: useCached};
                if (useCached) {
                  fetchCachedPackage(db, PACKAGE_PATH + PACKAGE_NAME, metadata, processPackageData, preloadFallback);
                } else {
                  fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE,
                    function(packageData) {
                      cacheRemotePackage(db, PACKAGE_PATH + PACKAGE_NAME, packageData, {uuid:PACKAGE_UUID}, processPackageData,
                        function(error) {
                          console.error(error);
                          processPackageData(packageData);
                        });
                    }
                  , preloadFallback);
                }
              }
            , preloadFallback);
          }
        , preloadFallback);

        if (Module['setStatus']) Module['setStatus']('Downloading...');
      '''
        else:
            # Not using preload cache, so we might as well start the xhr ASAP,
            # potentially before JS parsing of the main codebase if it's after us.
            # Only tricky bit is the fetch is async, but also when runWithFS is called
            # is async, so we handle both orderings.
            ret += r'''
        var fetchedCallback = null;
        var fetched = Module['getPreloadedPackage'] ? Module['getPreloadedPackage'](REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE) : null;

        if (!fetched) fetchRemotePackage(REMOTE_PACKAGE_NAME, REMOTE_PACKAGE_SIZE, function(data) {
          if (fetchedCallback) {
            fetchedCallback(data);
            fetchedCallback = null;
          } else {
            fetched = data;
          }
        }, handleError);
      '''

            code += r'''
        Module.preloadResults[PACKAGE_NAME] = {fromCache: false};
        if (fetched) {
          processPackageData(fetched);
          fetched = null;
        } else {
          fetchedCallback = processPackageData;
        }
      '''

    ret += '''
    function runWithFS() {
  '''
    ret += code
    ret += '''
    }
    if (Module['calledRun']) {
      runWithFS();
    } else {
      if (!Module['preRun']) Module['preRun'] = [];
      Module["preRun"].push(runWithFS); // FS is not initialized yet, wait for it
    }
  '''

    if separate_metadata:
        _metadata_template = '''
    Module['removeRunDependency']('%(metadata_file)s');
   }

   function runMetaWithFS() {
    Module['addRunDependency']('%(metadata_file)s');
    var REMOTE_METADATA_NAME = Module['locateFile'] ? Module['locateFile']('%(metadata_file)s', '') : '%(metadata_file)s';
    var xhr = new XMLHttpRequest();
    xhr.onreadystatechange = function() {
     if (xhr.readyState === 4 && xhr.status === 200) {
       loadPackage(JSON.parse(xhr.responseText));
     }
    }
    xhr.open('GET', REMOTE_METADATA_NAME, true);
    xhr.overrideMimeType('application/json');
    xhr.send(null);
   }

   if (Module['calledRun']) {
    runMetaWithFS();
   } else {
    if (!Module['preRun']) Module['preRun'] = [];
    Module["preRun"].push(runMetaWithFS);
   }
  ''' % {
            'metadata_file': os.path.basename(jsoutput + '.metadata')
        }

    else:
        _metadata_template = '''
   }
   loadPackage(%s);
  ''' % json.dumps(metadata)

    ret += '''%s
  })();
  ''' % _metadata_template

    if force or len(data_files):
        if jsoutput is None:
            print(ret)
        else:
            # Overwrite the old jsoutput file (if exists) only when its content
            # differs from the current generated one, otherwise leave the file
            # untouched preserving its old timestamp
            if os.path.isfile(jsoutput):
                f = open(jsoutput, 'r+')
                old = f.read()
                if old != ret:
                    f.seek(0)
                    f.write(ret)
                    f.truncate()
            else:
                f = open(jsoutput, 'w')
                f.write(ret)
            f.close()
            if separate_metadata:
                f = open(jsoutput + '.metadata', 'w')
                json.dump(metadata, f, separators=(',', ':'))
                f.close()

    return 0
Example #18
0
  '''
    use_data += '''
          var files = metadata.files;
          for (var i = 0; i < files.length; ++i) {
            DataRequest.prototype.requests[files[i].filename].onload();
          }
    '''
    use_data += ("          Module['removeRunDependency']('datafile_%s');\n"
                 % shared.JS.escape_for_js_string(data_target))

  else:
    # LZ4FS usage
    temp = data_target + '.orig'
    shutil.move(data_target, temp)
    meta = run_js(shared.path_from_root('tools', 'lz4-compress.js'),
                  shared.NODE_JS,
                  [shared.path_from_root('src', 'mini-lz4.js'),
                   temp, data_target], stdout=PIPE)
    os.unlink(temp)
    use_data = '''
          var compressedData = %s;
          compressedData.data = byteArray;
          assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with  -s LZ4=1  ?');
          LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData });
          Module['removeRunDependency']('datafile_%s');
    ''' % (meta, shared.JS.escape_for_js_string(data_target))

  package_uuid = uuid.uuid4()
  package_name = data_target
  statinfo = os.stat(package_name)
  remote_package_size = statinfo.st_size
  remote_package_name = os.path.basename(package_name)
Example #19
0
          var files = metadata.files;
          for (var i = 0; i < files.length; ++i) {
            DataRequest.prototype.requests[files[i].filename].onload();
          }
    '''
        use_data += (
            "          Module['removeRunDependency']('datafile_%s');\n" %
            shared.JS.escape_for_js_string(data_target))

    else:
        # LZ4FS usage
        temp = data_target + '.orig'
        shutil.move(data_target, temp)
        meta = run_js(
            shared.path_from_root('tools', 'lz4-compress.js'),
            shared.NODE_JS,
            [shared.path_from_root('src', 'mini-lz4.js'), temp, data_target],
            stdout=PIPE)
        os.unlink(temp)
        use_data = '''
          var compressedData = %s;
          compressedData.data = byteArray;
          assert(typeof LZ4 === 'object', 'LZ4 not present - was your app build with  -s LZ4=1  ?');
          LZ4.loadPackage({ 'metadata': metadata, 'compressedData': compressedData });
          Module['removeRunDependency']('datafile_%s');
    ''' % (meta, shared.JS.escape_for_js_string(data_target))

    package_uuid = uuid.uuid4()
    package_name = data_target
    remote_package_size = os.path.getsize(package_name)
    remote_package_name = os.path.basename(package_name)
Example #20
0
 def run(self, args):
   return jsrun.run_js(self.filename, engine=self.engine, args=args, stderr=PIPE, full_output=True, assert_returncode=None)