def test_emcc_ports(self):
restore_and_set_up()
# listing ports
out = self.do([EMCC, '--show-ports'])
self.assertContained('Available ports:', out)
self.assertContained('SDL2', out)
self.assertContained('SDL2_image', out)
self.assertContained('SDL2_net', out)
# using ports
RETRIEVING_MESSAGE = 'retrieving port'
BUILDING_MESSAGE = 'generating port'
PORTS_DIR = system_libs.Ports.get_dir()
for i in [0, 1]:
self.do([EMCC, '--clear-cache'])
print(i)
if i == 0:
try_delete(PORTS_DIR)
else:
self.do([EMCC, '--clear-ports'])
self.assertNotExists(PORTS_DIR)
# Building a file that doesn't need ports should not trigger anything
output = self.do([EMCC, test_file('hello_world_sdl.cpp')])
self.assertNotContained(RETRIEVING_MESSAGE, output)
self.assertNotContained(BUILDING_MESSAGE, output)
self.assertNotExists(PORTS_DIR)
def first_use():
output = self.do([
EMCC,
test_file('hello_world_sdl.cpp'), '-s', 'USE_SDL=2'
])
self.assertContained(RETRIEVING_MESSAGE, output)
self.assertContained(BUILDING_MESSAGE, output)
self.assertExists(PORTS_DIR)
def second_use():
# Using it again avoids retrieve and build
output = self.do([
EMCC,
test_file('hello_world_sdl.cpp'), '-s', 'USE_SDL=2'
])
self.assertNotContained(RETRIEVING_MESSAGE, output)
self.assertNotContained(BUILDING_MESSAGE, output)
# Building a file that need a port does trigger stuff
first_use()
second_use()
# if the url doesn't match, we retrieve and rebuild
with open(os.path.join(PORTS_DIR, 'sdl2', '.emscripten_url'),
'w') as f:
f.write('foo')
first_use()
second_use()
def test_vanilla(self):
restore_and_set_up()
self.clear_cache()
def make_fake(report):
with open(config_file, 'a') as f:
f.write('LLVM_ROOT = "' + self.in_dir('fake', 'bin') + '"\n')
# BINARYEN_ROOT needs to exist in the config, even though this test
# doesn't actually use it.
f.write('BINARYEN_ROOT= "%s"\n' % self.in_dir('fake', 'bin'))
make_fake_clang(self.in_dir('fake', 'bin', 'clang'),
EXPECTED_LLVM_VERSION)
make_fake_llc(self.in_dir('fake', 'bin', 'llc'), report)
make_fake_tool(self.in_dir('fake', 'bin', 'wasm-ld'),
EXPECTED_LLVM_VERSION)
# fake llc output
def test_with_fake(report, expected):
make_fake(report)
with env_modify({'EMCC_DEBUG': '1'}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'],
expected)
test_with_fake(
'got js backend! JavaScript (asm.js, emscripten) backend',
'LLVM has not been built with the WebAssembly backend')
try_delete(CANONICAL_TEMP_DIR)
def test_llvm_fastcomp(self):
WARNING = 'fastcomp in use, but LLVM has not been built with the JavaScript backend as a target'
restore_and_set_up()
# Should see js backend during sanity check
self.assertTrue(shared.check_llvm())
output = self.check_working(EMCC)
self.assertNotIn(WARNING, output)
# Fake incorrect llc output, no mention of js backend
restore_and_set_up()
with open(CONFIG_FILE, 'a') as f:
f.write('LLVM_ROOT = "' + self.in_dir('fake', 'bin') + '"')
# print '1', open(CONFIG_FILE).read()
make_fake_clang(self.in_dir('fake', 'bin', 'clang'),
expected_llvm_version())
make_fake_llc(self.in_dir('fake', 'bin', 'llc'),
'no j-s backend for you!')
self.check_working(EMCC, WARNING)
# fake some more
for fake in [
'llvm-link', 'llvm-ar', 'opt', 'llvm-as', 'llvm-dis',
'llvm-nm', 'lli'
]:
open(self.in_dir('fake', 'bin', fake), 'w').write('.')
try_delete(SANITY_FILE)
self.check_working(EMCC, WARNING)
def test_closure_compiler(self):
CLOSURE_FATAL = 'fatal: closure compiler'
CLOSURE_WARNING = 'does not exist'
# Sanity check should find closure
restore_and_set_up()
output = self.check_working(EMCC)
self.assertNotContained(CLOSURE_FATAL, output)
self.assertNotContained(CLOSURE_WARNING, output)
# Append a bad path for closure, will warn
f = open(CONFIG_FILE, 'a')
f.write('CLOSURE_COMPILER = "/tmp/nowhere/nothingtoseehere/kjadsfkjwelkjsdfkqgas/nonexistent.txt"\n')
f.close()
output = self.check_working(EMCC, CLOSURE_WARNING)
# And if you actually try to use the bad path, will be fatal
f = open(CONFIG_FILE, 'a')
f.write('CLOSURE_COMPILER = "/tmp/nowhere/nothingtoseehere/kjadsfkjwelkjsdfkqgas/nonexistent.txt"\n')
f.close()
output = self.check_working([EMCC, '-s', '--closure', '1'] + MINIMAL_HELLO_WORLD + ['-O2'], CLOSURE_FATAL)
# With a working path, all is well
restore_and_set_up()
try_delete('a.out.js')
output = self.check_working([EMCC, '-s', '--closure', '1'] + MINIMAL_HELLO_WORLD + ['-O2'], '')
assert os.path.exists('a.out.js'), output
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [
shared.CLANG_CXX, 'g++', 'clang++'
]: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
shared.run_process([
compiler,
shared.path_from_root('tools', 'optimizer',
'parser.cpp'),
shared.path_from_root('tools', 'optimizer',
'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-shared.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-main.cpp'), '-O3',
'-std=c++11', '-fno-exceptions', '-fno-rtti', '-o',
output
] + args,
stdout=log_output,
stderr=log_output)
except Exception as e:
logging.debug(str(e))
continue # perhaps the later compilers will succeed
# success
return output
shared.exit_with_error('Failed to build native optimizer')
def generate_minimal_runtime_html(target, options, js_target, target_basename):
logger.debug('generating HTML for minimal runtime')
shell = utils.read_file(options.shell_path)
if settings.SINGLE_FILE:
# No extra files needed to download in a SINGLE_FILE build.
shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', '')
else:
shell = shell.replace('{{{ DOWNLOAD_JS_AND_WASM_FILES }}}', generate_minimal_runtime_load_statement(target_basename))
temp_files = shared.configuration.get_temp_files()
with temp_files.get_file(suffix='.js') as shell_temp:
utils.write_file(shell_temp, shell)
shell = shared.read_and_preprocess(shell_temp)
if re.search(r'{{{\s*SCRIPT\s*}}}', shell):
shared.exit_with_error('--shell-file "' + options.shell_path + '": MINIMAL_RUNTIME uses a different kind of HTML page shell file than the traditional runtime! Please see $EMSCRIPTEN/src/shell_minimal_runtime.html for a template to use as a basis.')
shell = shell.replace('{{{ TARGET_BASENAME }}}', target_basename)
shell = shell.replace('{{{ EXPORT_NAME }}}', settings.EXPORT_NAME)
shell = shell.replace('{{{ PTHREAD_WORKER_FILE }}}', settings.PTHREAD_WORKER_FILE)
# In SINGLE_FILE build, embed the main .js file into the .html output
if settings.SINGLE_FILE:
js_contents = utils.read_file(js_target)
shared.try_delete(js_target)
else:
js_contents = ''
shell = shell.replace('{{{ JS_CONTENTS_IN_SINGLE_FILE_BUILD }}}', js_contents)
shell = line_endings.convert_line_endings(shell, '\n', options.output_eol)
# Force UTF-8 output for consistency across platforms and with the web.
with open(target, 'wb') as f:
f.write(shell.encode('utf-8'))
def process_funcs(args):
i, ll, settings_file, compiler, forwarded_file, libraries = args
funcs_file = temp_files.get('.func_%d.ll' % i).name
open(funcs_file, 'w').write(ll)
out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
shared.try_delete(funcs_file)
return out.split('//FORWARDED_DATA:')
def test_js_engine_path(self):
# Test that running JS commands works for node, d8, and jsc and is not path dependent
restore_and_set_up()
sample_script = path_from_root('tests', 'print_args.js')
# Fake some JS engines
# Note that the path contains 'd8'.
test_path = self.in_dir('fake', 'abcd8765')
ensure_dir(test_path)
jsengines = [('d8', config.V8_ENGINE),
('d8_g', config.V8_ENGINE),
('js', config.SPIDERMONKEY_ENGINE),
('node', config.NODE_JS),
('nodejs', config.NODE_JS)]
for filename, engine in jsengines:
try_delete(SANITY_FILE)
if type(engine) is list:
engine = engine[0]
if not engine:
print('WARNING: Not testing engine %s, not configured.' % (filename))
continue
print(filename, engine)
test_engine_path = os.path.join(test_path, filename)
with open(test_engine_path, 'w') as f:
f.write('#!/bin/sh\n')
f.write('exec %s $@\n' % (engine))
make_executable(test_engine_path)
out = self.run_js(sample_script, engine=test_engine_path, args=['--foo'])
self.assertEqual('0: --foo', out.strip())
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
self.old_env = os.environ
os.environ = self.env.copy()
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
env_init = self.env.copy()
# Note that we need to pass in all the flags here because some build
# systems (like zlib) if they see a CFLAGS it will override all their
# default flags, including optimizations.
env_init['CFLAGS'] = ' '.join(LLVM_FEATURE_FLAGS + [OPTIMIZATIONS] + self.extra_args)
emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=env_init)
final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
EMCC, filename,
OPTIMIZATIONS,
'-s', 'INITIAL_MEMORY=256MB',
'-s', 'FILESYSTEM=0',
'--closure', '1',
'-s', 'MINIMAL_RUNTIME=1',
'-s', 'BENCHMARK=%d' % (1 if IGNORE_COMPILATION and not has_output_parser else 0),
'-o', final
] + shared_args + emcc_args + LLVM_FEATURE_FLAGS + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd]
if PROFILING:
cmd += ['--profiling-funcs']
self.cmd = cmd
run_process(cmd, env=self.env)
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def try_js(args=[]):
shared.try_delete(filename + '.js')
js_args = [shared.EMCC, fullname, '-o', filename + '.js'
] + [opts] + llvm_opts + CSMITH_CFLAGS + args + ['-w']
if TEST_BINARYEN:
if random.random() < 0.5:
js_args += ['-g']
if random.random() < 0.5:
# pick random passes
BINARYEN_EXTRA_PASSES = [
"code-pushing",
"duplicate-function-elimination",
"dce",
"remove-unused-brs",
"remove-unused-names",
"local-cse",
"optimize-instructions",
"post-emscripten",
"precompute",
"simplify-locals",
"simplify-locals-nostructure",
"vacuum",
"coalesce-locals",
"reorder-locals",
"merge-blocks",
"remove-unused-module-elements",
"memory-packing",
]
passes = []
while 1:
passes.append(random.choice(BINARYEN_EXTRA_PASSES))
if random.random() < 0.1:
break
js_args += [
'-s', 'BINARYEN_EXTRA_PASSES="' + ','.join(passes) + '"'
]
if random.random() < 0.5:
js_args += ['-s', 'ALLOW_MEMORY_GROWTH=1']
if random.random(
) < 0.5 and 'ALLOW_MEMORY_GROWTH=1' not in js_args and 'BINARYEN=1' not in js_args:
js_args += ['-s', 'MAIN_MODULE=1']
if random.random() < 0.25:
js_args += ['-s', 'INLINING_LIMIT=1'
] # inline nothing, for more call interaction
if random.random() < 0.5:
js_args += ["--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2"]
if random.random() < 0.5:
js_args += ['-s', 'ASSERTIONS=1']
print('(compile)', ' '.join(js_args))
short_args = [shared.EMCC, fail_output_name] + js_args[5:]
escaped_short_args = map(lambda x: ("'" + x + "'")
if '"' in x else x, short_args)
open(fullname,
'a').write('\n// ' + ' '.join(escaped_short_args) + '\n\n')
try:
shared.run_process(js_args)
assert os.path.exists(filename + '.js')
return js_args
except Exception:
return False
def erase():
dirname = Ports.get_dir()
shared.try_delete(dirname)
if os.path.exists(dirname):
logger.warning(
'could not delete ports dir %s - try to delete it manually' %
dirname)
def install_header_dir(src_dir, target=None):
if not target:
target = os.path.basename(src_dir)
dest = os.path.join(Ports.get_include_dir(), target)
shared.try_delete(dest)
logger.debug(f'installing headers: {dest}')
shutil.copytree(src_dir, dest)
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
self.old_env = os.environ
os.environ = self.env.copy()
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
emcc_args = emcc_args + lib_builder(
'js_' + llvm_root, native=False, env_init=self.env.copy())
final = os.path.dirname(filename) + os.path.sep + self.name + (
'_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
PYTHON, EMCC, filename, OPTIMIZATIONS, '-s', 'TOTAL_MEMORY=256MB',
'-s', 'FILESYSTEM=0', '--closure', '1', '-s', 'MINIMAL_RUNTIME=0',
'-s',
'BENCHMARK=%d' %
(1 if IGNORE_COMPILATION and not has_output_parser else 0), '-o',
final
] + shared_args + emcc_args + LLVM_FEATURE_FLAGS + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [
arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd
]
if PROFILING:
cmd += ['--profiling-funcs']
self.cmd = cmd
run_process(cmd, env=self.env)
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [
shared.CLANG, 'g++', 'clang++'
]: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
out, err = subprocess.Popen(
[
compiler,
shared.path_from_root('tools', 'optimizer',
'parser.cpp'),
shared.path_from_root('tools', 'optimizer',
'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-shared.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-main.cpp'),
'-O3', '-std=c++11', '-fno-exceptions',
'-fno-rtti', '-o', output
] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
outs.append(out)
errs.append(err)
except OSError:
if compiler == shared.CLANG:
raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
if os.path.exists(output): return output
raise NativeOptimizerCreationException()
def test_closure_compiler(self):
CLOSURE_FATAL = 'fatal: closure compiler'
CLOSURE_WARNING = 'does not exist'
# Sanity check should find closure
restore_and_set_up()
output = self.check_working(EMCC)
self.assertNotContained(CLOSURE_FATAL, output)
self.assertNotContained(CLOSURE_WARNING, output)
# Append a bad path for closure, will warn
f = open(CONFIG_FILE, 'a')
f.write(
'CLOSURE_COMPILER = "/tmp/nowhere/nothingtoseehere/kjadsfkjwelkjsdfkqgas/nonexistent.txt"\n'
)
f.close()
output = self.check_working(EMCC, CLOSURE_WARNING)
# And if you actually try to use the bad path, will be fatal
f = open(CONFIG_FILE, 'a')
f.write(
'CLOSURE_COMPILER = "/tmp/nowhere/nothingtoseehere/kjadsfkjwelkjsdfkqgas/nonexistent.txt"\n'
)
f.close()
output = self.check_working([EMCC, '-s', '--closure', '1'] +
MINIMAL_HELLO_WORLD + ['-O2'],
CLOSURE_FATAL)
# With a working path, all is well
restore_and_set_up()
try_delete('a.out.js')
output = self.check_working([EMCC, '-s', '--closure', '1'] +
MINIMAL_HELLO_WORLD + ['-O2'], '')
self.assertExists('a.out.js', output)
def process_funcs(args):
i, funcs, meta, settings_file, compiler, forwarded_file, libraries = args
ll = ''.join(funcs) + '\n' + meta
funcs_file = temp_files.get('.func_%d.ll' % i).name
open(funcs_file, 'w').write(ll)
out = shared.run_js(compiler, compiler_engine, [settings_file, funcs_file, 'funcs', forwarded_file] + libraries, stdout=subprocess.PIPE, cwd=path_from_root('src'))
shared.try_delete(funcs_file)
return out
def run():
if shared.Settings.WASM_BACKEND:
# The wasm backend does suffer from the same probllem as fastcomp so doesn't
# need the filename hashing.
cmd = [shared.LLVM_AR] + sys.argv[1:]
return shared.run_process(cmd, stdin=sys.stdin, check=False).returncode
try:
args = substitute_response_files(sys.argv)
except IOError as e:
shared.exit_with_error(e)
newargs = [shared.LLVM_AR] + args[1:]
tmpdir = None
response_filename = None
# The 3 argmuent form of ar doesn't involve other files. For example
# 'ar x libfoo.a'.
if len(newargs) > 3:
tmpdir = tempfile.mkdtemp(prefix='emar-')
cmd = newargs[1]
if 'r' in cmd or 'q' in cmd:
# We are adding files to the archive.
# Normally the output file is then arg 2, except in the case were the
# a or b modifiers are used in which case its arg 3.
if 'a' in cmd or 'b' in cmd:
out_arg_index = 3
else:
out_arg_index = 2
# Add a hash to colliding basename, to make them unique.
for j in range(out_arg_index + 1, len(newargs)):
orig_name = newargs[j]
full_name = os.path.abspath(orig_name)
basename = os.path.basename(full_name)
h = hashlib.md5(full_name.encode('utf-8')).hexdigest()[:8]
parts = basename.split('.')
parts[0] += '_' + h
newname = '.'.join(parts)
full_newname = os.path.join(tmpdir, newname)
shutil.copyfile(orig_name, full_newname)
newargs[j] = full_newname
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
response_filename = create_response_file(
newargs[3:], shared.get_emscripten_temp_dir())
newargs = newargs[:3] + ['@' + response_filename]
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
rtn = shared.run_process(newargs, stdin=sys.stdin, check=False).returncode
if tmpdir:
shutil.rmtree(tmpdir)
shared.try_delete(response_filename)
return rtn
def run():
try:
args = substitute_response_files(sys.argv)
except IOError as e:
shared.exit_with_error(e)
newargs = [shared.LLVM_AR] + args[1:]
to_delete = []
# The 3 argmuent form of ar doesn't involve other files. For example
# 'ar x libfoo.a'.
if len(newargs) > 3:
cmd = newargs[1]
if 'r' in cmd or 'q' in cmd:
# We are adding files to the archive.
# Normally the output file is then arg 2, except in the case were the
# a or b modifiers are used in which case its arg 3.
if 'a' in cmd or 'b' in cmd:
out_arg_index = 3
else:
out_arg_index = 2
# Add a hash to colliding basename, to make them unique.
for j in range(out_arg_index + 1, len(newargs)):
orig_name = newargs[j]
full_name = os.path.abspath(orig_name)
dirname = os.path.dirname(full_name)
basename = os.path.basename(full_name)
h = hashlib.md5(full_name.encode('utf-8')).hexdigest()[:8]
parts = basename.split('.')
parts[0] += '_' + h
newname = '.'.join(parts)
full_newname = os.path.join(dirname, newname)
try:
shutil.copyfile(orig_name, full_newname)
newargs[j] = full_newname
to_delete.append(full_newname)
except Exception:
# it is ok to fail here, we just don't get hashing
pass
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
response_filename = create_response_file(
newargs[3:], shared.get_emscripten_temp_dir())
to_delete += [response_filename]
newargs = newargs[:3] + ['@' + response_filename]
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
rtn = shared.run_process(newargs, stdin=sys.stdin, check=False).returncode
for d in to_delete:
shared.try_delete(d)
return rtn
def run():
args = substitute_response_files(sys.argv)
newargs = [shared.LLVM_AR] + args[1:]
to_delete = []
# The 3 argment form of ar doesn't involve other files. For example
# 'ar x libfoo.a'.
if len(newargs) > 3:
cmd = newargs[1]
if 'r' in cmd:
# we are adding files to the archive.
# normally the output file is then arg 2, except in the case were the
# a or b modifiers are used in which case its arg 3.
if 'a' in cmd or 'b' in cmd:
new_member_args_start = 4
else:
new_member_args_start = 3
# we add a hash to each input, to make them unique as
# possible, as llvm-ar cannot extract duplicate names
# (and only the basename is used!)
for j in range(new_member_args_start, len(newargs)):
orig_name = newargs[j]
full_name = os.path.abspath(orig_name)
dir_name = os.path.dirname(full_name)
base_name = os.path.basename(full_name)
h = hashlib.md5(full_name.encode('utf-8')).hexdigest()[:8]
parts = base_name.split('.')
parts[0] += '_' + h
newname = '.'.join(parts)
full_newname = os.path.join(dir_name, newname)
if not os.path.exists(full_newname):
try: # it is ok to fail here, we just don't get hashing
shutil.copyfile(orig_name, full_newname)
newargs[j] = full_newname
to_delete.append(full_newname)
except:
pass
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
response_filename = create_response_file(newargs[3:], shared.get_emscripten_temp_dir())
to_delete += [response_filename]
newargs = newargs[:3] + ['@' + response_filename]
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
try:
return shared.run_process(newargs, stdin=sys.stdin, check=False).returncode
finally:
for d in to_delete:
shared.try_delete(d)
def test_enet(self):
# this is also a good test of raw usage of emconfigure and emmake
shared.try_delete('enet')
shutil.copytree(test_file('third_party', 'enet'), 'enet')
with utils.chdir('enet'):
self.run_process([path_from_root('emconfigure'), './configure', '--disable-shared'])
self.run_process([path_from_root('emmake'), 'make'])
enet = [self.in_dir('enet', '.libs', 'libenet.a'), '-I' + self.in_dir('enet', 'include')]
with CompiledServerHarness(test_file('sockets/test_enet_server.c'), enet, 49210) as harness:
self.btest_exit(test_file('sockets/test_enet_client.c'), args=enet + ['-DSOCKK=%d' % harness.listen_port])
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
cheerp_args = [
'-fno-math-errno',
]
cheerp_args += self.args
self.parent = parent
if lib_builder:
# build as "native" (so no emcc env stuff), but with all the cheerp stuff
# set in the env
cheerp_args = cheerp_args + lib_builder(
self.name,
native=True,
env_init={
'CC': CHEERP_BIN + 'clang',
'CXX': CHEERP_BIN + 'clang++',
'AR': CHEERP_BIN + '../libexec/cheerp-unknown-none-ar',
'LD': CHEERP_BIN + 'clang',
'NM': CHEERP_BIN + 'llvm-nm',
'LDSHARED': CHEERP_BIN + 'clang',
'RANLIB':
CHEERP_BIN + '../libexec/cheerp-unknown-none-ranlib',
'CXXFLAGS': "-Wno-c++11-narrowing",
'CHEERP_PREFIX': CHEERP_BIN + '../',
})
if PROFILING:
cheerp_args += ['-cheerp-pretty-code'
] # get function names, like emcc --profiling
final = os.path.dirname(filename) + os.path.sep + self.name + (
'_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
dirs_to_delete = []
cheerp_args += ['-cheerp-preexecute']
try:
# print(cheerp_args)
if filename.endswith('.c'):
compiler = CHEERP_BIN + '/clang'
else:
compiler = CHEERP_BIN + '/clang++'
cmd = [compiler] + cheerp_args + [
'-cheerp-linear-heap-size=256',
'-cheerp-secondary-output-file=' +
final.replace('.js', '.wasm'), filename, '-o', final
] + shared_args
# print(' '.join(cmd))
run_process(cmd, stdout=PIPE, stderr=PIPE)
self.filename = final
if self.binaryen_opts:
run_binaryen_opts(final.replace('.js', '.wasm'),
self.binaryen_opts)
finally:
for dir_ in dirs_to_delete:
try_delete(dir_)
def test_emcc_caching(self):
INCLUDING_MESSAGE = 'including X'
BUILDING_MESSAGE = 'building X for cache'
ERASING_MESSAGE = 'clearing cache'
EMCC_CACHE = Cache.dirname
restore_and_set_up()
Cache.erase()
assert not os.path.exists(EMCC_CACHE)
with env_modify({'EMCC_DEBUG': '1'}):
# Building a file that *does* need something *should* trigger cache
# generation, but only the first time
for filename, libname in [('hello_libcxx.cpp', 'libcxx')]:
for i in range(3):
print(filename, libname, i)
self.clear()
output = self.do([EMCC, '-O' + str(i), '-s', '--llvm-lto', '0', path_from_root('tests', filename), '--save-bc', 'a.bc', '-s', 'DISABLE_EXCEPTION_CATCHING=0'])
# print '\n\n\n', output
assert INCLUDING_MESSAGE.replace('X', libname) in output
if libname == 'libc':
assert INCLUDING_MESSAGE.replace('X', 'libcxx') not in output # we don't need libcxx in this code
else:
assert INCLUDING_MESSAGE.replace('X', 'libc') in output # libcxx always forces inclusion of libc
assert (BUILDING_MESSAGE.replace('X', libname) in output) == (i == 0), 'Must only build the first time'
self.assertContained('hello, world!', run_js('a.out.js'))
assert os.path.exists(EMCC_CACHE)
full_libname = libname + '.bc' if libname != 'libcxx' else libname + '.a'
assert os.path.exists(os.path.join(EMCC_CACHE, full_libname))
try_delete(CANONICAL_TEMP_DIR)
restore_and_set_up()
def ensure_cache():
self.do([PYTHON, EMCC, '-O2', path_from_root('tests', 'hello_world.c')])
# Manual cache clearing
ensure_cache()
self.assertTrue(os.path.exists(EMCC_CACHE))
output = self.do([PYTHON, EMCC, '--clear-cache'])
self.assertIn(ERASING_MESSAGE, output)
self.assertFalse(os.path.exists(EMCC_CACHE))
self.assertIn(SANITY_MESSAGE, output)
# Changing LLVM_ROOT, even without altering .emscripten, clears the cache
ensure_cache()
with env_modify({'LLVM': 'waka'}):
self.assertTrue(os.path.exists(EMCC_CACHE))
output = self.do([PYTHON, EMCC])
self.assertIn(ERASING_MESSAGE, output)
self.assertFalse(os.path.exists(EMCC_CACHE))
def test_emcc(self):
SANITY_FAIL_MESSAGE = 'sanity check failed to run'
# emcc should check sanity if no ${EM_CONFIG}_sanity
restore_and_set_up()
time.sleep(1)
assert not os.path.exists(
SANITY_FILE) # restore is just the settings, not the sanity
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
# EMCC should have checked sanity successfully
old_sanity = open(SANITY_FILE).read()
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc run again should not sanity check, because the sanity file is newer
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# incorrect sanity contents mean we *must* check
open(SANITY_FILE, 'w').write('wakawaka')
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
# correct sanity contents mean we need not check
open(SANITY_FILE, 'w').write(old_sanity)
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
# but with EMCC_DEBUG=1 we should check
with env_modify({'EMCC_DEBUG': '1'}):
output = self.check_working(EMCC)
try_delete(CANONICAL_TEMP_DIR)
self.assertContained(SANITY_MESSAGE, output)
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
# also with -v, with or without inputs
output = self.check_working([EMCC, '-v'], SANITY_MESSAGE)
output = self.check_working([EMCC, '-v'] + MINIMAL_HELLO_WORLD + [],
SANITY_MESSAGE)
# Make sure the test runner didn't do anything to the setup
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc should also check sanity if the file is outdated
open(CONFIG_FILE, 'a').write('# extra stuff\n')
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
emcc_args = emcc_args + lib_builder(
'js_' + llvm_root, native=False, env_init=self.env.copy())
open('hardcode.py', 'w').write('''
def process(filename):
js = open(filename).read()
replaced = js.replace("run();", "run(%s.concat(Module[\\"arguments\\"]));")
assert js != replaced
open(filename, 'w').write(replaced)
import sys
process(sys.argv[1])
''' % str(args[:-1])) # do not hardcode in the last argument, the default arg
final = os.path.dirname(filename) + os.path.sep + self.name + (
'_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
PYTHON,
EMCC,
filename,
OPTIMIZATIONS,
'--js-transform',
'python hardcode.py',
'-s',
'TOTAL_MEMORY=256*1024*1024',
'-s',
'FILESYSTEM=0',
# '--profiling',
'--closure',
'1',
'-s',
'BENCHMARK=%d' %
(1 if IGNORE_COMPILATION and not has_output_parser else 0),
'-o',
final
] + shared_args + emcc_args + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [
arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd
]
output = run_process(cmd, stdout=PIPE, stderr=PIPE,
env=self.env).stdout
assert os.path.exists(
final
), 'Failed to compile file: ' + output + ' (looked for ' + final + ')'
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def process_funcs(args):
i, ll, settings_file, compiler, forwarded_file, libraries = args
funcs_file = temp_files.get('.func_%d.ll' % i).name
open(funcs_file, 'w').write(ll)
out = shared.run_js(compiler,
compiler_engine,
[settings_file, funcs_file, 'funcs', forwarded_file] +
libraries,
stdout=subprocess.PIPE,
cwd=path_from_root('src'))
shared.try_delete(funcs_file)
return out.split('//FORWARDED_DATA:')
def process_funcs(args):
i, funcs, meta, settings_file, compiler, forwarded_file, libraries = args
ll = ''.join(funcs) + '\n' + meta
funcs_file = temp_files.get('.func_%d.ll' % i).name
open(funcs_file, 'w').write(ll)
out = shared.run_js(compiler,
compiler_engine,
[settings_file, funcs_file, 'funcs', forwarded_file] +
libraries,
stdout=subprocess.PIPE,
cwd=path_from_root('src'))
shared.try_delete(funcs_file)
return out
def test_enet(self):
# this is also a good test of raw usage of emconfigure and emmake
shared.try_delete(self.in_dir('enet'))
shutil.copytree(path_from_root('tests', 'enet'), self.in_dir('enet'))
with chdir(self.in_dir('enet')):
run_process([PYTHON, path_from_root('emconfigure'), './configure'])
run_process([PYTHON, path_from_root('emmake'), 'make'])
enet = [self.in_dir('enet', '.libs', 'libenet.a'), '-I' + path_from_root('tests', 'enet', 'include')]
for harness in [
CompiledServerHarness(os.path.join('sockets', 'test_enet_server.c'), enet, 49210)
]:
with harness:
self.btest(os.path.join('sockets', 'test_enet_client.c'), expected='0', args=enet + ['-DSOCKK=%d' % harness.listen_port])
def test_enet(self):
# this is also a good test of raw usage of emconfigure and emmake
shared.try_delete('enet')
shutil.copytree(path_from_root('tests', 'enet'), 'enet')
with chdir('enet'):
run_process([path_from_root('emconfigure'), './configure'])
run_process([path_from_root('emmake'), 'make'])
enet = [self.in_dir('enet', '.libs', 'libenet.a'), '-I' + path_from_root('tests', 'enet', 'include')]
for harness in [
CompiledServerHarness(os.path.join('sockets', 'test_enet_server.c'), enet, 49210)
]:
with harness:
self.btest(os.path.join('sockets', 'test_enet_client.c'), expected='0', args=enet + ['-DSOCKK=%d' % harness.listen_port])
def run():
DEBUG = os.environ.get('EMCC_DEBUG')
if DEBUG == "0":
DEBUG = None
newargs = [shared.LLVM_AR] + sys.argv[1:]
if DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
to_delete = []
if len(newargs) > 2:
if 'r' in newargs[1]:
# we are adding files to the archive.
# find the .a; everything after it is an input file.
# we add a hash to each input, to make them unique as
# possible, as llvm-ar cannot extract duplicate names
# (and only the basename is used!)
i = 1
while i < len(newargs):
if newargs[i].endswith('.a'):
import hashlib, shutil
for j in range(i+1, len(newargs)):
orig_name = newargs[j]
full_name = os.path.abspath(orig_name)
dir_name = os.path.dirname(full_name)
base_name = os.path.basename(full_name)
h = hashlib.md5(full_name.encode('utf-8')).hexdigest()[:8]
parts = base_name.split('.')
parts[0] += '_' + h
newname = '.'.join(parts)
full_newname = os.path.join(dir_name, newname)
if not os.path.exists(full_newname):
try: # it is ok to fail here, we just don't get hashing
shutil.copyfile(orig_name, full_newname)
newargs[j] = full_newname
to_delete.append(full_newname)
except:
pass
break
i += 1
if DEBUG:
print('Invoking ' + str(newargs), file=sys.stderr)
try:
return subprocess.call(newargs, stdin=sys.stdin)
finally:
for d in to_delete:
shared.try_delete(d)
def run():
try:
me, main, side, out = sys.argv[:4]
except:
print('usage: emlink.py [main module] [side module] [output name]', file=sys.stderr)
sys.exit(1)
print('Main module:', main)
print('Side module:', side)
print('Output:', out)
shared.try_delete(out)
main = AsmModule(main)
side = AsmModule(side)
side.relocate_into(main)
main.write(out)
def run():
try:
me, main, side, out = sys.argv[:4]
except:
print >> sys.stderr, 'usage: emlink.py [main module] [side module] [output name]'
sys.exit(1)
print 'Main module:', main
print 'Side module:', side
print 'Output:', out
shared.try_delete(out)
main = AsmModule(main)
side = AsmModule(side)
side.relocate_into(main)
main.write(out)
def run():
try:
me, main, side, out = sys.argv[:4]
except ValueError:
print('usage: emlink.py [main module] [side module] [output name]', file=sys.stderr)
sys.exit(1)
print('Main module:', main)
print('Side module:', side)
print('Output:', out)
shared.try_delete(out)
main = AsmModule(main)
side = AsmModule(side)
side.relocate_into(main)
main.write(out)
def test_node(self):
NODE_WARNING = 'node version appears too old'
NODE_WARNING_2 = 'cannot check node version'
restore_and_set_up()
# Clang should report the version number we expect, and emcc should not warn
assert shared.check_node_version()
output = self.check_working(EMCC)
self.assertNotContained(NODE_WARNING, output)
# Fake a different node version
restore_and_set_up()
with open(config_file, 'a') as f:
f.write('NODE_JS = "' + self.in_dir('fake', 'nodejs') + '"')
ensure_dir('fake')
for version, succeed in [('v0.8.0', False),
('v4.1.0', False),
('v4.1.1', True),
('v4.2.3-pre', True),
('cheez', False)]:
print(version, succeed)
try_delete(SANITY_FILE)
f = open(self.in_dir('fake', 'nodejs'), 'w')
f.write('#!/bin/sh\n')
f.write('''if [ $1 = "--version" ]; then
echo "%s"
else
%s $@
fi
''' % (version, ' '.join(config.NODE_JS)))
f.close()
make_executable(self.in_dir('fake', 'nodejs'))
if not succeed:
if version[0] == 'v':
self.check_working(EMCC, NODE_WARNING)
else:
self.check_working(EMCC, NODE_WARNING_2)
else:
output = self.check_working(EMCC)
self.assertNotContained(NODE_WARNING, output)
def test_llvm(self):
LLVM_WARNING = 'LLVM version appears incorrect'
restore_and_set_up()
# Clang should report the version number we expect, and emcc should not warn
assert shared.check_llvm_version()
output = self.check_working(EMCC)
self.assertNotContained(LLVM_WARNING, output)
# Fake a different llvm version
restore_and_set_up()
with open(config_file, 'a') as f:
f.write('LLVM_ROOT = "' + self.in_dir('fake') + '"')
real_version_x, real_version_y = (
int(x) for x in EXPECTED_LLVM_VERSION.split('.'))
make_fake_llc(self.in_dir('fake', 'llc'),
'wasm32 - WebAssembly 32-bit')
make_fake_tool(self.in_dir('fake', 'wasm-ld'), EXPECTED_LLVM_VERSION)
for inc_x in range(-2, 3):
for inc_y in range(-2, 3):
try_delete(SANITY_FILE)
expected_x = real_version_x + inc_x
expected_y = real_version_y + inc_y
if expected_x < 0 or expected_y < 0:
continue # must be a valid llvm version
print("mod LLVM version: %d %d -> %d %d" %
(real_version_x, real_version_y, expected_x, expected_y))
make_fake_clang(self.in_dir('fake', 'clang'),
'%s.%s' % (expected_x, expected_y))
make_fake_tool(self.in_dir('fake', 'llvm-ar'),
'%s.%s' % (expected_x, expected_y))
make_fake_tool(self.in_dir('fake', 'llvm-nm'),
'%s.%s' % (expected_x, expected_y))
did_modify = inc_x != 0 or inc_y != 0
if did_modify:
output = self.check_working(EMCC, LLVM_WARNING)
else:
output = self.check_working(EMCC)
self.assertNotContained(LLVM_WARNING, output)
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
out, err = subprocess.Popen([compiler,
shared.path_from_root('tools', 'optimizer', 'parser.cpp'),
shared.path_from_root('tools', 'optimizer', 'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'),
shared.path_from_root('tools', 'optimizer', 'optimizer-shared.cpp'),
shared.path_from_root('tools', 'optimizer', 'optimizer-main.cpp'),
'-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
outs.append(out)
errs.append(err)
except OSError:
if compiler == shared.CLANG: raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
if os.path.exists(output): return output
raise NativeOptimizerCreationException()
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
emcc_args = emcc_args or []
self.filename = filename
llvm_root = self.env.get('LLVM') or config.LLVM_ROOT
if lib_builder:
env_init = self.env.copy()
# Note that we need to pass in all the flags here because some build
# systems (like zlib) if they see a CFLAGS it will override all their
# default flags, including optimizations.
env_init['CFLAGS'] = ' '.join(LLVM_FEATURE_FLAGS + [OPTIMIZATIONS] + self.extra_args)
emcc_args += lib_builder('js_' + llvm_root, native=False, env_init=env_init)
final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
EMCC, filename,
OPTIMIZATIONS,
'-sINITIAL_MEMORY=256MB',
'-sENVIRONMENT=node,shell',
'-sBENCHMARK=%d' % (1 if IGNORE_COMPILATION and not has_output_parser else 0),
'-o', final
] + LLVM_FEATURE_FLAGS
if shared_args:
cmd += shared_args
if common.EMTEST_FORCE64:
cmd += ['--profiling']
else:
cmd += ['--closure=1', '-sMINIMAL_RUNTIME']
# add additional emcc args at the end, which may override other things
# above, such as minimal runtime
cmd += emcc_args + self.extra_args
if '-sFORCE_FILESYSTEM' not in cmd:
cmd += ['-sFILESYSTEM=0']
if PROFILING:
cmd += ['--profiling-funcs']
self.cmd = cmd
run_process(cmd, env=self.env)
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def prep():
restore_and_set_up()
print('clearing ports...')
print(self.do([PYTHON, EMCC, '--clear-ports']))
wipe()
self.do([PYTHON, EMCC]) # first run stage
try_delete(tag_file)
# if BINARYEN_ROOT is set, we don't build the port. Check we do build it if not
if binaryen_root_in_config:
config = open(CONFIG_FILE).read()
assert '''BINARYEN_ROOT = os.path.expanduser(os.getenv('BINARYEN', ''))''' in config, config # setup created it to be ''
print('created config:')
print(config)
restore_and_set_up()
config = open(CONFIG_FILE).read()
config = config.replace('BINARYEN_ROOT', '''BINARYEN_ROOT = os.path.expanduser(os.getenv('BINARYEN', '')) # ''')
else:
restore_and_set_up()
config = open(CONFIG_FILE).read()
config = config.replace('BINARYEN_ROOT', '#')
print('modified config:')
print(config)
open(CONFIG_FILE, 'w').write(config)
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=self.env.copy())
open('hardcode.py', 'w').write('''
def process(filename):
js = open(filename).read()
replaced = js.replace("run();", "run(%s.concat(Module[\\"arguments\\"]));")
assert js != replaced
open(filename, 'w').write(replaced)
import sys
process(sys.argv[1])
''' % str(args[:-1])) # do not hardcode in the last argument, the default arg
final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
PYTHON, EMCC, filename,
OPTIMIZATIONS,
'--js-transform', 'python hardcode.py',
'-s', 'TOTAL_MEMORY=256*1024*1024',
'-s', 'FILESYSTEM=0',
# '--profiling',
'--closure', '1',
'-s', 'BENCHMARK=%d' % (1 if IGNORE_COMPILATION and not has_output_parser else 0),
'-o', final
] + shared_args + emcc_args + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd]
output = run_process(cmd, stdout=PIPE, stderr=PIPE, env=self.env).stdout
assert os.path.exists(final), 'Failed to compile file: ' + output + ' (looked for ' + final + ')'
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
#!/usr/bin/env python2 ''' Fast static linker for emscripten outputs. Specifically this links asm.js modules. See https://github.com/kripken/emscripten/wiki/Linking ''' import sys from tools import shared from tools.asm_module import AsmModule try: me, main, side, out = sys.argv[:4] except: print >> sys.stderr, 'usage: emlink.py [main module] [side module] [output name]' sys.exit(1) print 'Main module:', main print 'Side module:', side print 'Output:', out shared.try_delete(out) main = AsmModule(main) side = AsmModule(side) side.relocate_into(main) main.write(out)
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
suffix = filename.split('.')[-1]
cheerp_temp = filename + '.cheerp.' + suffix
code = open(filename).read()
if 'int main()' in code:
main_args = ''
else:
main_args = 'argc, (%(const)s char**)argv' % {
'const': 'const' if 'const char *argv' in code else ''
}
open(cheerp_temp, 'w').write('''
%(code)s
void webMain() {
// TODO: how to read from commandline?
volatile int argc = 2;
typedef char** charStarStar;
volatile charStarStar argv;
argv[0] = "./cheerp.exe";
argv[1] = "%(arg)s";
volatile int exit_code = main(%(main_args)s);
}
''' % {
'arg': args[-1],
'code': code,
'main_args': main_args
})
cheerp_args = [
'-target', 'cheerp',
'-cheerp-mode=wasm'
]
cheerp_args += self.args
self.parent = parent
if lib_builder:
# build as "native" (so no emcc env stuff), but with all the cheerp stuff
# set in the env
cheerp_args = cheerp_args + lib_builder(self.name, native=True, env_init={
'CC': CHEERP_BIN + 'clang',
'CXX': CHEERP_BIN + 'clang++',
'AR': CHEERP_BIN + 'llvm-ar',
'LD': CHEERP_BIN + 'clang',
'NM': CHEERP_BIN + 'llvm-nm',
'LDSHARED': CHEERP_BIN + 'clang',
'RANLIB': CHEERP_BIN + 'llvm-ranlib',
'CFLAGS': ' '.join(cheerp_args),
'CXXFLAGS': ' '.join(cheerp_args),
})
# cheerp_args += ['-cheerp-pretty-code'] # get function names, like emcc --profiling
final = os.path.dirname(filename) + os.path.sep + 'cheerp_' + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
dirs_to_delete = []
try:
for arg in cheerp_args[:]:
if arg.endswith('.a'):
info = self.handle_static_lib(arg)
cheerp_args += info['files']
dirs_to_delete += [info['dir']]
cheerp_args = [arg for arg in cheerp_args if not arg.endswith('.a')]
# print(cheerp_args)
cmd = [CHEERP_BIN + 'clang++'] + cheerp_args + [
'-cheerp-linear-heap-size=256',
'-cheerp-wasm-loader=' + final,
cheerp_temp,
'-Wno-writable-strings', # for how we set up webMain
'-o', final + '.wasm'
] + shared_args
# print(' '.join(cmd))
run_process(cmd)
self.filename = final
Building.get_binaryen()
if self.binaryen_opts:
run_binaryen_opts(final + '.wasm', self.binaryen_opts)
finally:
for dir_ in dirs_to_delete:
try_delete(dir_)
outfile.write(funcs_js[i])
funcs_js = None
if WINDOWS: post = post.replace('\r\n', '\n') # Normalize to UNIX line endings, otherwise writing to text file will duplicate \r\n to \r\r\n!
outfile.write(post)
outfile.close()
if DEBUG: logging.debug(' emscript: final python processing took %s seconds' % (time.time() - t))
success = True
finally:
if not success:
outfile.close()
shared.try_delete(outfile.name) # remove partial output
if os.environ.get('EMCC_FAST_COMPILER') == '0':
logging.critical('Non-fastcomp compiler is no longer available, please use fastcomp or an older version of emscripten')
sys.exit(1)
def main(args, compiler_engine, cache, temp_files, DEBUG, DEBUG_CACHE):
# Prepare settings for serialization to JSON.
settings = {}
for setting in args.settings:
name, value = setting.strip().split('=', 1)
settings[name] = json.loads(value)
# libraries
libraries = args.libraries[0].split(',') if len(args.libraries) > 0 else []
def test_vanilla(self):
restore_and_set_up()
Cache.erase()
with env_modify({'EMCC_DEBUG': '1'}):
# see that we test vanilla status, and just once
TESTING = 'testing for asm.js target'
self.check_working(EMCC, TESTING)
for i in range(3):
output = self.check_working(EMCC, 'check tells us to use')
assert TESTING not in output
# if env var tells us, do what it says
with env_modify({'EMCC_WASM_BACKEND': '1'}):
self.check_working(EMCC, 'EMCC_WASM_BACKEND tells us to use wasm backend')
with env_modify({'EMCC_WASM_BACKEND': '0'}):
self.check_working(EMCC, 'EMCC_WASM_BACKEND tells us to use asm.js backend')
def make_fake(report):
with open(CONFIG_FILE, 'a') as f:
f.write('LLVM_ROOT = "' + path_from_root('tests', 'fake', 'bin') + '"\n')
# BINARYEN_ROOT needs to exist in the config, even though this test
# doesn't actually use it.
f.write('BINARYEN_ROOT= "%s"\n' % path_from_root('tests', 'fake', 'bin'))
with open(path_from_root('tests', 'fake', 'bin', 'llc'), 'w') as f:
f.write('#!/bin/sh\n')
f.write('echo "llc fake output\nRegistered Targets:\n%s"' % report)
os.chmod(path_from_root('tests', 'fake', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
with open(path_from_root('tests', 'fake', 'bin', 'wasm-ld'), 'w') as f:
f.write('#!/bin/sh\n')
f.write('exit 0\n')
os.chmod(path_from_root('tests', 'fake', 'bin', 'wasm-ld'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
with env_modify({'EMCC_DEBUG': '1'}):
make_fake('wasm32-unknown-unknown-elf')
# see that we request the right backend from llvm
with env_modify({'EMCC_WASM_BACKEND': '1'}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], 'wasm32-unknown-unknown-elf')
make_fake('asmjs-unknown-emscripten')
with env_modify({'EMCC_WASM_BACKEND': '0'}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], 'asmjs-unknown-emscripten')
# check the current installed one is ok
restore_and_set_up()
self.check_working(EMCC)
output = self.check_working(EMCC, 'check tells us to use')
if 'wasm backend' in output:
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], 'wasm32-unknown-unknown-elf')
else:
assert 'asm.js backend' in output
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], 'asmjs-unknown-emscripten')
# fake llc output
try_delete(path_from_root('tests', 'fake'))
os.makedirs(path_from_root('tests', 'fake', 'bin'))
def test_with_fake(report, expected):
make_fake(report)
with env_modify({'EMCC_DEBUG': '1'}):
output = self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], expected)
self.assertContained('config file changed since we checked vanilla', output)
test_with_fake('got js backend! JavaScript (asm.js, emscripten) backend', 'check tells us to use asm.js backend')
test_with_fake('got wasm32 backend! WebAssembly 32-bit', 'check tells us to use wasm backend')
# use LLVM env var to modify LLVM between vanilla checks
assert not os.environ.get('LLVM'), 'we need to modify LLVM env var for this'
f = open(CONFIG_FILE, 'a')
f.write('LLVM_ROOT = os.getenv("LLVM", "' + path_from_root('tests', 'fake1', 'bin') + '")\n')
f.close()
safe_ensure_dirs(path_from_root('tests', 'fake1', 'bin'))
f = open(path_from_root('tests', 'fake1', 'bin', 'llc'), 'w')
f.write('#!/bin/sh\n')
f.write('echo "llc fake1 output\nRegistered Targets:\n%s"' % 'got js backend! JavaScript (asm.js, emscripten) backend')
f.close()
os.chmod(path_from_root('tests', 'fake1', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
safe_ensure_dirs(path_from_root('tests', 'fake2', 'bin'))
f = open(path_from_root('tests', 'fake2', 'bin', 'llc'), 'w')
f.write('#!/bin/sh\n')
f.write('echo "llc fake2 output\nRegistered Targets:\n%s"' % 'got wasm32 backend! WebAssembly 32-bit')
f.close()
os.chmod(path_from_root('tests', 'fake2', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
with env_modify({'EMCC_DEBUG': '1'}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], 'use asm.js backend')
with env_modify({'LLVM': path_from_root('tests', 'fake2', 'bin')}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-c'], 'regenerating vanilla check since other llvm')
try_delete(CANONICAL_TEMP_DIR)
return # TODO: the rest of this
# check separate cache dirs are used
restore_and_set_up()
self.check_working([EMCC], '')
root_cache = os.path.expanduser('~/.emscripten_cache')
if os.path.exists(os.path.join(root_cache, 'asmjs')):
shutil.rmtree(os.path.join(root_cache, 'asmjs'))
if os.path.exists(os.path.join(root_cache, 'wasm')):
shutil.rmtree(os.path.join(root_cache, 'wasm'))
with env_modify({'EMCC_WASM_BACKEND': '1'}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD, '')
assert os.path.exists(os.path.join(root_cache, 'wasm'))
with env_modify({'EMCC_WASM_BACKEND': '0'}):
self.check_working([EMCC] + MINIMAL_HELLO_WORLD, '')
assert os.path.exists(os.path.join(root_cache, 'asmjs'))
shutil.rmtree(os.path.join(root_cache, 'asmjs'))
self.check_working([EMCC] + MINIMAL_HELLO_WORLD, '')
assert os.path.exists(os.path.join(root_cache, 'asmjs'))
def test_embuilder(self):
restore_and_set_up()
tests = [
([PYTHON, EMBUILDER], ['Emscripten System Builder Tool', 'build libc', 'native_optimizer'], True, []),
([PYTHON, EMBUILDER, 'build', 'waka'], 'ERROR', False, []),
([PYTHON, EMBUILDER, 'build', 'struct_info'], ['building and verifying struct_info', 'success'], True, ['generated_struct_info.json']),
([PYTHON, EMBUILDER, 'build', 'libc'], ['building and verifying libc', 'success'], True, ['libc.bc']),
([PYTHON, EMBUILDER, 'build', 'libc-mt'], ['building and verifying libc-mt', 'success'], True, ['libc-mt.bc']),
([PYTHON, EMBUILDER, 'build', 'libc-extras'], ['building and verifying libc-extras', 'success'], True, ['libc-extras.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc'], ['building and verifying dlmalloc', 'success'], True, ['dlmalloc.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc_debug'], ['building and verifying dlmalloc', 'success'], True, ['dlmalloc_debug.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc_threadsafe'], ['building and verifying dlmalloc_threadsafe', 'success'], True, ['dlmalloc_threadsafe.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc_threadsafe_debug'], ['building and verifying dlmalloc', 'success'], True, ['dlmalloc_threadsafe_debug.bc']),
([PYTHON, EMBUILDER, 'build', 'emmalloc'], ['building and verifying emmalloc', 'success'], True, ['emmalloc.bc']),
([PYTHON, EMBUILDER, 'build', 'emmalloc_debug'], ['building and verifying emmalloc', 'success'], True, ['emmalloc_debug.bc']),
([PYTHON, EMBUILDER, 'build', 'pthreads'], ['building and verifying pthreads', 'success'], True, ['pthreads.bc']),
([PYTHON, EMBUILDER, 'build', 'libcxx'], ['success'], True, ['libcxx.a']),
([PYTHON, EMBUILDER, 'build', 'libcxx_noexcept'], ['success'], True, ['libcxx_noexcept.a']),
([PYTHON, EMBUILDER, 'build', 'libcxxabi'], ['success'], True, ['libcxxabi.bc']),
([PYTHON, EMBUILDER, 'build', 'gl'], ['success'], True, ['gl.bc']),
([PYTHON, EMBUILDER, 'build', 'native_optimizer'], ['success'], True, ['optimizer.2.exe']),
([PYTHON, EMBUILDER, 'build', 'zlib'], ['building and verifying zlib', 'success'], True, ['zlib.bc']),
([PYTHON, EMBUILDER, 'build', 'libpng'], ['building and verifying libpng', 'success'], True, ['libpng.bc']),
([PYTHON, EMBUILDER, 'build', 'bullet'], ['building and verifying bullet', 'success'], True, ['bullet.bc']),
([PYTHON, EMBUILDER, 'build', 'vorbis'], ['building and verifying vorbis', 'success'], True, ['vorbis.bc']),
([PYTHON, EMBUILDER, 'build', 'ogg'], ['building and verifying ogg', 'success'], True, ['ogg.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2'], ['success'], True, ['sdl2.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-gfx'], ['success'], True, ['sdl2-gfx.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-image'], ['success'], True, ['sdl2-image.bc']),
([PYTHON, EMBUILDER, 'build', 'freetype'], ['building and verifying freetype', 'success'], True, ['freetype.bc']),
([PYTHON, EMBUILDER, 'build', 'harfbuzz'], ['building and verifying harfbuzz', 'success'], True, ['harfbuzz.bc']),
([PYTHON, EMBUILDER, 'build', 'icu'], ['building and verifying icu', 'success'], True, ['icu.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-ttf'], ['building and verifying sdl2-ttf', 'success'], True, ['sdl2-ttf.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-net'], ['building and verifying sdl2-net', 'success'], True, ['sdl2-net.bc']),
([PYTHON, EMBUILDER, 'build', 'binaryen'], ['building and verifying binaryen', 'success'], True, []),
([PYTHON, EMBUILDER, 'build', 'cocos2d'], ['building and verifying cocos2d', 'success'], True, ['libCocos2d.bc']),
([PYTHON, EMBUILDER, 'build', 'wasm-libc'], ['building and verifying wasm-libc', 'success'], True, ['wasm-libc.bc']),
]
if Settings.WASM_BACKEND:
tests.append(([PYTHON, EMBUILDER, 'build', 'wasm_compiler_rt'], ['building and verifying wasm_compiler_rt', 'success'], True, ['wasm_compiler_rt.a']),)
Cache.erase()
for command, expected, success, result_libs in tests:
print(command)
for lib in result_libs:
if os.path.sep in lib:
dirname = os.path.dirname(Cache.get_path(lib))
print(' erase dir', dirname)
try_delete(dirname)
else:
print(' erase file', lib)
try_delete(Cache.get_path(lib))
proc = run_process(command, stdout=PIPE, stderr=STDOUT, check=False)
out = proc.stdout
if success and proc.returncode != 0:
print(out)
assert (proc.returncode == 0) == success
if not isinstance(expected, list):
expected = [expected]
for ex in expected:
print(' seek', ex)
assert ex in out, out
for lib in result_libs:
print(' verify', lib)
assert os.path.exists(Cache.get_path(lib))
def create_optimizer_cmake():
shared.logging.debug('building native optimizer via CMake: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
if NATIVE_OPTIMIZER == '1':
cmake_build_type = 'RelWithDebInfo'
elif NATIVE_OPTIMIZER == '2':
cmake_build_type = 'Release'
elif NATIVE_OPTIMIZER == 'g':
cmake_build_type = 'Debug'
build_path = shared.Cache.get_path('optimizer_build_' + cmake_build_type)
shared.try_delete(os.path.join(build_path, 'CMakeCache.txt'))
log_output = None if DEBUG else subprocess.PIPE
if not os.path.exists(build_path):
os.mkdir(build_path)
if WINDOWS:
# Poor man's check for whether or not we should attempt 64 bit build
if os.environ.get('ProgramFiles(x86)'):
cmake_generators = [
'Visual Studio 15 2017 Win64',
'Visual Studio 15 2017',
'Visual Studio 14 2015 Win64',
'Visual Studio 14 2015',
'Visual Studio 12 Win64', # The year component is omitted for compatibility with older CMake.
'Visual Studio 12',
'Visual Studio 11 Win64',
'Visual Studio 11',
'MinGW Makefiles',
'Unix Makefiles',
]
else:
cmake_generators = [
'Visual Studio 15 2017',
'Visual Studio 14 2015',
'Visual Studio 12',
'Visual Studio 11',
'MinGW Makefiles',
'Unix Makefiles',
]
else:
cmake_generators = ['Unix Makefiles']
for cmake_generator in cmake_generators:
# Delete CMakeCache.txt so that we can switch to a new CMake generator.
shared.try_delete(os.path.join(build_path, 'CMakeCache.txt'))
proc = subprocess.Popen(['cmake', '-G', cmake_generator, '-DCMAKE_BUILD_TYPE='+cmake_build_type, shared.path_from_root('tools', 'optimizer')], cwd=build_path, stdin=log_output, stdout=log_output, stderr=log_output)
proc.communicate()
if proc.returncode == 0:
make = ['cmake', '--build', build_path]
if 'Visual Studio' in cmake_generator:
make += ['--config', cmake_build_type, '--', '/nologo', '/verbosity:minimal']
proc = subprocess.Popen(make, cwd=build_path, stdin=log_output, stdout=log_output, stderr=log_output)
proc.communicate()
if proc.returncode == 0:
if WINDOWS and 'Visual Studio' in cmake_generator:
shutil.copyfile(os.path.join(build_path, cmake_build_type, 'optimizer.exe'), output)
else:
shutil.copyfile(os.path.join(build_path, 'optimizer'), output)
return output
raise NativeOptimizerCreationException()
def test_native_optimizer(self):
restore_and_set_up()
def build():
return self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-O2', '-s', 'WASM=0'], 'running js post-opts')
def test():
self.assertContained('hello, world!', run_js('a.out.js'))
with env_modify({'EMCC_DEBUG': '1'}):
# basic usage or lack of usage
for native in [None, 0, 1]:
print('phase 1, part', native)
Cache.erase()
try:
if native is not None:
os.environ['EMCC_NATIVE_OPTIMIZER'] = str(native)
output = build()
assert ('js optimizer using native' in output) == (not not (native or native is None)), output
test()
if native or native is None: # None means use the default, which is to use the native optimizer
assert 'building native optimizer' in output, output
# compile again, no rebuild of optimizer
output = build()
assert 'building native optimizer' not in output
assert 'js optimizer using native' in output
test()
finally:
if native is not None:
del os.environ['EMCC_NATIVE_OPTIMIZER']
# force a build failure, see we fall back to non-native
for native in [1, 'g']:
with env_modify({'EMCC_NATIVE_OPTIMIZER': str(native)}):
print('phase 2, part', native)
Cache.erase()
try:
# break it
f = path_from_root('tools', 'optimizer', 'optimizer-main.cpp')
src = open(f).read()
bad = src.replace('main', '!waka waka<')
assert bad != src
open(f, 'w').write(bad)
# first try
output = build()
assert 'failed to build native optimizer' in output, output
if native == 1:
assert 'to see compiler errors, build with EMCC_NATIVE_OPTIMIZER=g' in output
assert 'waka waka' not in output
else:
assert 'output from attempt' in output, output
assert 'waka waka' in output, output
assert 'js optimizer using native' not in output
test() # still works, without native optimizer
# second try, see previous failure
output = build()
assert 'failed to build native optimizer' not in output
assert 'seeing that optimizer could not be built' in output
test() # still works, without native optimizer
# clear cache, try again
Cache.erase()
output = build()
assert 'failed to build native optimizer' in output
test() # still works, without native optimizer
finally:
open(f, 'w').write(src)
Cache.erase()
# now it should work again
output = build()
assert 'js optimizer using native' in output
test() # still works
try_delete(CANONICAL_TEMP_DIR)
DEBUG = os.environ.get('IDL_VERBOSE') is '1'
if DEBUG: print("Debug print ON, CHECKS=%s" % CHECKS)
class Dummy(object):
def __init__(self, init):
for k, v in init.items():
self.__dict__[k] = v
def getExtendedAttribute(self, name):
return None
input_file = sys.argv[1]
output_base = sys.argv[2]
shared.try_delete(output_base + '.cpp')
shared.try_delete(output_base + '.js')
p = WebIDL.Parser()
p.parse(r'''
interface VoidPtr {
};
''' + open(input_file).read())
data = p.finish()
interfaces = {}
implements = {}
enums = {}
for thing in data:
if isinstance(thing, WebIDL.IDLInterface):
def test_llvm_fastcomp(self):
WARNING = 'fastcomp in use, but LLVM has not been built with the JavaScript backend as a target'
WARNING2 = 'you can fall back to the older (pre-fastcomp) compiler core, although that is not recommended, see http://kripken.github.io/emscripten-site/docs/building_from_source/LLVM-Backend.html'
restore_and_set_up()
# Should see js backend during sanity check
assert shared.check_fastcomp()
output = self.check_working(EMCC)
self.assertNotIn(WARNING, output)
self.assertNotIn(WARNING2, output)
# Fake incorrect llc output, no mention of js backend
restore_and_set_up()
with open(CONFIG_FILE, 'a') as f:
f.write('LLVM_ROOT = "' + path_from_root('tests', 'fake', 'bin') + '"')
# print '1', open(CONFIG_FILE).read()
try_delete(path_from_root('tests', 'fake'))
os.makedirs(path_from_root('tests', 'fake', 'bin'))
with open(path_from_root('tests', 'fake', 'bin', 'llc'), 'w') as f:
f.write('#!/bin/sh\n')
f.write('echo "llc fake output\nRegistered Targets:\nno j-s backend for you!"')
os.chmod(path_from_root('tests', 'fake', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
output = self.check_working(EMCC, WARNING)
output = self.check_working(EMCC, WARNING2)
# fake some more
for fake in ['llvm-link', 'clang', 'clang++', 'llvm-ar', 'opt', 'llvm-as', 'llvm-dis', 'llvm-nm', 'lli']:
open(path_from_root('tests', 'fake', 'bin', fake), 'w').write('.')
try_delete(SANITY_FILE)
output = self.check_working(EMCC, WARNING)
# make sure sanity checks notice there is no source dir with version #
open(path_from_root('tests', 'fake', 'bin', 'llc'), 'w').write('#!/bin/sh\necho "Registered Targets: there IZ a js backend: JavaScript (asm.js, emscripten) backend"')
open(path_from_root('tests', 'fake', 'bin', 'clang++'), 'w').write('#!/bin/sh\necho "clang version %s (blah blah)" >&2\necho "..." >&2\n' % expected_llvm_version())
os.chmod(path_from_root('tests', 'fake', 'bin', 'llc'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
os.chmod(path_from_root('tests', 'fake', 'bin', 'clang++'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
try_delete(SANITY_FILE)
output = self.check_working(EMCC, 'did not see a source tree above or next to the LLVM root directory')
VERSION_WARNING = 'Emscripten, llvm and clang repo versions do not match, this is dangerous'
BUILD_VERSION_WARNING = 'Emscripten, llvm and clang build versions do not match, this is dangerous'
# add version number
open(path_from_root('tests', 'fake', 'emscripten-version.txt'), 'w').write('waka')
try_delete(SANITY_FILE)
output = self.check_working(EMCC, VERSION_WARNING)
os.makedirs(path_from_root('tests', 'fake', 'tools', 'clang'))
open(path_from_root('tests', 'fake', 'tools', 'clang', 'emscripten-version.txt'), 'w').write(EMSCRIPTEN_VERSION)
try_delete(SANITY_FILE)
output = self.check_working(EMCC, VERSION_WARNING)
open(path_from_root('tests', 'fake', 'emscripten-version.txt'), 'w').write(EMSCRIPTEN_VERSION)
try_delete(SANITY_FILE)
output = self.check_working(EMCC)
self.assertNotIn(VERSION_WARNING, output)
open(path_from_root('tests', 'fake', 'tools', 'clang', 'emscripten-version.txt'), 'w').write('waka')
try_delete(SANITY_FILE)
output = self.check_working(EMCC, VERSION_WARNING)
# restore clang version to ok, and fake the *build* versions
open(path_from_root('tests', 'fake', 'tools', 'clang', 'emscripten-version.txt'), 'w').write(EMSCRIPTEN_VERSION)
output = self.check_working(EMCC)
self.assertNotIn(VERSION_WARNING, output)
fake = '#!/bin/sh\necho "clang version %s (blah blah) (emscripten waka : waka)"\necho "..."\n' % expected_llvm_version()
open(path_from_root('tests', 'fake', 'bin', 'clang'), 'w').write(fake)
open(path_from_root('tests', 'fake', 'bin', 'clang++'), 'w').write(fake)
os.chmod(path_from_root('tests', 'fake', 'bin', 'clang'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
os.chmod(path_from_root('tests', 'fake', 'bin', 'clang++'), stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
try_delete(SANITY_FILE)
output = self.check_working(EMCC, BUILD_VERSION_WARNING)
self.assertNotIn(VERSION_WARNING, output)
# break clang repo version again, see it hides the build warning
open(path_from_root('tests', 'fake', 'tools', 'clang', 'emscripten-version.txt'), 'w').write('waka')
output = self.check_working(EMCC, VERSION_WARNING)
self.assertNotIn(BUILD_VERSION_WARNING, output)
restore_and_set_up()
self.check_working([EMCC] + MINIMAL_HELLO_WORLD + ['-s', 'ASM_JS=0'], '''Compiler settings are incompatible with fastcomp. You can fall back to the older compiler core, although that is not recommended''')
def test_emcc_ports(self):
restore_and_set_up()
# listing ports
out = self.do([PYTHON, EMCC, '--show-ports'])
assert 'Available ports:' in out, out
assert 'SDL2' in out, out
assert 'SDL2_image' in out, out
assert 'SDL2_net' in out, out
# using ports
RETRIEVING_MESSAGE = 'retrieving port'
BUILDING_MESSAGE = 'generating port'
from tools import system_libs
PORTS_DIR = system_libs.Ports.get_dir()
for compiler in [EMCC]:
print(compiler)
for i in [0, 1]:
self.do([PYTHON, EMCC, '--clear-cache'])
print(i)
if i == 0:
try_delete(PORTS_DIR)
else:
self.do([PYTHON, compiler, '--clear-ports'])
assert not os.path.exists(PORTS_DIR)
# Building a file that doesn't need ports should not trigger anything
# (avoid wasm to avoid the binaryen port)
output = self.do([compiler, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'WASM=0'])
print('no', output)
assert RETRIEVING_MESSAGE not in output, output
assert BUILDING_MESSAGE not in output
assert not os.path.exists(PORTS_DIR)
# Building a file that need a port does trigger stuff
output = self.do([compiler, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'WASM=0', '-s', 'USE_SDL=2'])
print('yes', output)
assert RETRIEVING_MESSAGE in output, output
assert BUILDING_MESSAGE in output, output
assert os.path.exists(PORTS_DIR)
def second_use():
# Using it again avoids retrieve and build
output = self.do([compiler, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'WASM=0', '-s', 'USE_SDL=2'])
assert RETRIEVING_MESSAGE not in output, output
assert BUILDING_MESSAGE not in output, output
second_use()
# if the version isn't sufficient, we retrieve and rebuild
subdir = os.listdir(os.path.join(PORTS_DIR, 'sdl2'))[0]
os.rename(os.path.join(PORTS_DIR, 'sdl2', subdir), os.path.join(PORTS_DIR, 'sdl2', 'old-subdir'))
import zipfile
z = zipfile.ZipFile(os.path.join(PORTS_DIR, 'sdl2' + '.zip'), 'w')
if not os.path.exists('old-sub'):
os.mkdir('old-sub')
open(os.path.join('old-sub', 'a.txt'), 'w').write('waka')
open(os.path.join('old-sub', 'b.txt'), 'w').write('waka')
z.write(os.path.join('old-sub', 'a.txt'))
z.write(os.path.join('old-sub', 'b.txt'))
z.close()
output = self.do([compiler, path_from_root('tests', 'hello_world_sdl.cpp'), '-s', 'WASM=0', '-s', 'USE_SDL=2'])
assert RETRIEVING_MESSAGE in output, output
assert BUILDING_MESSAGE in output, output
assert os.path.exists(PORTS_DIR)
second_use()
if random.random() < 0.5: extra_args += ['--no-math64']
extra_args += ['--no-bitfields'] # due to pnacl bug 4027, "LLVM ERROR: can't convert calls with illegal types"
#if random.random() < 0.5: extra_args += ['--float'] # XXX hits undefined behavior on float=>int conversions (too big to fit)
if random.random() < 0.5: extra_args += ['--max-funcs', str(random.randint(10, 30))]
suffix = '.c'
COMP = shared.CLANG_CC
fullname = filename + suffix
check_call([CSMITH, '--no-volatiles', '--no-packed-struct'] + extra_args,
#['--max-block-depth', '2', '--max-block-size', '2', '--max-expr-complexity', '2', '--max-funcs', '2'],
stdout=open(fullname, 'w'))
print '1) Generate source... %.2f K' % (len(open(fullname).read())/1024.)
tried += 1
print '2) Compile natively'
shared.try_delete(filename)
try:
shared.check_execute([COMP, '-m32', opts, fullname, '-o', filename + '1'] + CSMITH_CFLAGS + ['-w']) # + shared.EMSDK_OPTS
except CalledProcessError as e:
print 'Failed to compile natively using clang'
notes['invalid'] += 1
continue
shared.check_execute([COMP, '-m32', opts, '-emit-llvm', '-c', fullname, '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS + ['-w'])
shared.check_execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'], stderr=PIPE)
shutil.move(filename + '.bc.run', filename + '2')
shared.check_execute([COMP, fullname, '-o', filename + '3'] + CSMITH_CFLAGS + ['-w'])
print '3) Run natively'
try:
correct1 = shared.jsrun.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct1 or len(correct1) < 10: raise Exception('segfault')
def test_emcc(self):
SANITY_FAIL_MESSAGE = 'sanity check failed to run'
# emcc should check sanity if no ${EM_CONFIG}_sanity
restore_and_set_up()
time.sleep(1)
assert not os.path.exists(SANITY_FILE) # restore is just the settings, not the sanity
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
assert os.path.exists(SANITY_FILE) # EMCC should have checked sanity successfully
assert mtime(SANITY_FILE) > mtime(CONFIG_FILE)
assert generate_sanity() == open(SANITY_FILE).read()
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc run again should not sanity check, because the sanity file is newer
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# correct sanity contents mean we need not check
open(SANITY_FILE, 'w').write(generate_sanity())
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
# incorrect sanity contents mean we *must* check
open(SANITY_FILE, 'w').write('wakawaka')
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
# but with EMCC_DEBUG=1 we should check
with env_modify({'EMCC_DEBUG': '1'}):
output = self.check_working(EMCC)
try_delete(CANONICAL_TEMP_DIR)
self.assertContained(SANITY_MESSAGE, output)
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
# also with -v, with or without inputs
output = self.check_working([EMCC, '-v'], SANITY_MESSAGE)
output = self.check_working([EMCC, '-v'] + MINIMAL_HELLO_WORLD + [], SANITY_MESSAGE)
# Make sure the test runner didn't do anything to the setup
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc should also check sanity if the file is outdated
time.sleep(0.1)
restore_and_set_up()
assert mtime(SANITY_FILE) < mtime(CONFIG_FILE)
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
assert mtime(SANITY_FILE) >= mtime(CONFIG_FILE)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc should be configurable directly from EM_CONFIG without any config file
restore_and_set_up()
config = open(CONFIG_FILE, 'r').read()
open('main.cpp', 'w').write('''
#include <stdio.h>
int main() {
printf("hello from emcc with no config file\\n");
return 0;
}
''')
wipe()
with env_modify({'EM_CONFIG': config}):
run_process([PYTHON, EMCC, 'main.cpp', '-o', 'a.out.js'])
self.assertContained('hello from emcc with no config file', run_js('a.out.js'))
def try_js(args=[]):
shared.try_delete(filename + '.js')
js_args = [shared.PYTHON, shared.EMCC, fullname, '-o', filename + '.js'] + [opts] + llvm_opts + CSMITH_CFLAGS + args + ['-w']
if TEST_BINARYEN:
js_args += ['-s', 'BINARYEN=1', '-s', 'BINARYEN_TRAP_MODE="js"']
if random.random() < 0.5:
js_args += ['-g']
if random.random() < 0.1:
if random.random() < 0.5:
js_args += ['--js-opts', '0']
else:
js_args += ['--js-opts', '1']
if random.random() < 0.5:
# pick random passes
BINARYEN_PASSES = [
"code-pushing",
"duplicate-function-elimination",
"dce",
"remove-unused-brs",
"remove-unused-names",
"local-cse",
"optimize-instructions",
"post-emscripten",
"precompute",
"simplify-locals",
"simplify-locals-nostructure",
"vacuum",
"coalesce-locals",
"reorder-locals",
"merge-blocks",
"remove-unused-module-elements",
"memory-packing",
]
passes = []
while 1:
passes.append(random.choice(BINARYEN_PASSES))
if random.random() < 0.1:
break
js_args += ['-s', 'BINARYEN_PASSES="' + ','.join(passes) + '"']
if random.random() < 0.5:
js_args += ['-s', 'ALLOW_MEMORY_GROWTH=1']
if random.random() < 0.5 and 'ALLOW_MEMORY_GROWTH=1' not in js_args and 'BINARYEN=1' not in js_args:
js_args += ['-s', 'MAIN_MODULE=1']
if random.random() < 0.25:
js_args += ['-s', 'INLINING_LIMIT=1'] # inline nothing, for more call interaction
if random.random() < 0.01:
js_args += ['-s', 'EMTERPRETIFY=1']
if random.random() < 0.5:
if random.random() < 0.5:
js_args += ['-s', 'EMTERPRETIFY_BLACKLIST=["_main"]'] # blacklist main and all inlined into it, but interpret the rest, tests mixing
else:
js_args += ['-s', 'EMTERPRETIFY_WHITELIST=["_main"]'] # the opposite direction
if random.random() < 0.5:
js_args += ['-s', 'EMTERPRETIFY_ASYNC=1']
if random.random() < 0.5:
js_args += ["--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2"]
if random.random() < 0.5:
js_args += ['-s', 'ASSERTIONS=1']
print '(compile)', ' '.join(js_args)
short_args = [shared.PYTHON, shared.EMCC, fail_output_name] + js_args[5:]
escaped_short_args = map(lambda x : ("'" + x + "'") if '"' in x else x, short_args)
open(fullname, 'a').write('\n// ' + ' '.join(escaped_short_args) + '\n\n')
try:
shared.check_execute(js_args)
assert os.path.exists(filename + '.js')
return js_args
except:
return False
def wipe(): try_delete(CONFIG_FILE) try_delete(SANITY_FILE)
def relocate_into(self, main):
# heap initializer
if self.staticbump > 0:
new_mem_init = self.mem_init_js[:self.mem_init_js.rfind(', ')] + ', Runtime.GLOBAL_BASE+%d)' % main.staticbump
main.pre_js = re.sub(shared.JS.memory_staticbump_pattern, 'STATICTOP = STATIC_BASE + %d;\n' % (main.staticbump + side.staticbump) + new_mem_init, main.pre_js, count=1)
# Find function name replacements TODO: do not rename duplicate names with duplicate contents, just merge them
replacements = {}
for func in self.funcs:
rep = func
while rep in main.funcs:
rep += '_'
replacements[func] = rep
#print >> sys.stderr, 'replacements:', replacements
# sendings: add invokes for new tables
all_sendings = main.sendings
added_sending = False
for table in self.tables:
if table not in main.tables:
sig = table[table.rfind('_')+1:]
all_sendings['invoke_%s' % sig] = shared.JS.make_invoke(sig, named=False)
added_sending = True
# imports
all_imports = main.imports
for key, value in self.imports.iteritems():
if key in self.funcs or key in main.funcs: continue # external function in one module, implemented in the other
value_concrete = '.' not in value # env.key means it is an import, an external value, and not a concrete one
main_value = main.imports.get(key)
main_value_concrete = main_value and '.' not in main_value
if value_concrete and main_value_concrete: continue # standard global var
if not main_value or value_concrete:
if '+' in value:
# relocate
value = value.replace('(', '').replace(')', '').replace('| 0', '').replace('|0', '').replace(' ', '')
left, right = value.split('+')
assert left == 'H_BASE'
value = str(main.staticbump + int(right))
all_imports[key] = value
if (value_concrete or main_value_concrete) and key in all_sendings:
del all_sendings[key] # import of external value no longer needed
main.imports_js = '\n'.join(['var %s = %s;' % (key, value) for key, value in all_imports.iteritems()]) + '\n'
if added_sending:
sendings_js = ', '.join(['%s: %s' % (key, value) for key, value in all_sendings.iteritems()])
sendings_start = main.post_js.find('}, { ')+5
sendings_end = main.post_js.find(' }, buffer);')
main.post_js = main.post_js[:sendings_start] + sendings_js + main.post_js[sendings_end:]
# check for undefined references to global variables
def check_import(key, value):
if value.startswith('+') or value.endswith('|0'): # ignore functions
if key not in all_sendings:
print >> sys.stderr, 'warning: external variable %s is still not defined after linking' % key
for key, value in all_imports.iteritems(): check_import(key, value)
# tables
f_bases = {}
f_sizes = {}
for table, data in self.tables.iteritems():
main.tables[table] = self.merge_tables(table, main.tables.get(table), data, replacements, f_bases, f_sizes)
main.combine_tables()
#print >> sys.stderr, 'f bases', f_bases
# relocate
temp = shared.Building.js_optimizer(self.filename, ['asm', 'relocate', 'last'], extra_info={
'replacements': replacements,
'fBases': f_bases,
'hBase': main.staticbump
})
#print >> sys.stderr, 'relocated side into', temp
relocated_funcs = AsmModule(temp)
shared.try_delete(temp)
main.extra_funcs_js = relocated_funcs.funcs_js.replace(js_optimizer.start_funcs_marker, '\n')
# update function table uses
ft_marker = 'FUNCTION_TABLE_'
def update_fts(what):
updates = []
i = 1 # avoid seeing marker in recursion
while 1:
i = what.find(ft_marker, i)
if i < 0: break;
start = i
end = what.find('[', start)
table = what[i:end]
if table not in f_sizes:
# table was not modified
i += len(ft_marker)
continue
nesting = 1
while nesting > 0:
next = what.find(']', end+1)
nesting -= 1
nesting += what.count('[', end+1, next)
end = next
assert end > 0
mask = what.rfind('&', start, end)
assert mask > 0 and end - mask <= 13
fixed = update_fts(what[start:mask+1] + str(f_sizes[table]-1) + ']')
updates.append((start, end, fixed))
i = end # additional function table uses were done by recursion
# apply updates
if len(updates) == 0: return what
parts = []
so_far = 0
for i in range(len(updates)):
start, end, fixed = updates[i]
parts.append(what[so_far:start])
parts.append(fixed)
so_far = end+1
parts.append(what[so_far:])
return ''.join(parts)
main.funcs_js = update_fts(main.funcs_js)
main.extra_funcs_js = update_fts(main.extra_funcs_js)
# global initializers
if self.global_inits:
my_global_inits = map(lambda init: replacements[init] if init in replacements else init, self.global_inits)
all_global_inits = map(lambda init: '{ func: function() { %s() } }' % init, main.global_inits + my_global_inits)
all_global_inits_js = '/* global initializers */ __ATINIT__.push(' + ','.join(all_global_inits) + ');'
if main.global_inits:
target = main.global_inits_js
else:
target = '// === Body ===\n'
all_global_inits_js = target + all_global_inits_js
main.pre_js = main.pre_js.replace(target, all_global_inits_js)
# exports
def rep_exp(export):
key, value = export.split(':')
if key in replacements:
repped = replacements[key]
return repped + ': ' + repped
return export
my_exports = map(rep_exp, self.exports)
exports = main.exports.union(my_exports)
main.exports_js = 'return {' + ','.join(list(exports)) + '};\n})\n'
# post
def rep_def(deff):
key = deff.split(' ')[1]
if key in replacements:
rep = replacements[key]
return 'var %s = Module["%s"] = asm["%s"];\n' % (rep, rep, rep)
return deff
my_module_defs = map(rep_def, self.module_defs)
new_module_defs = set(my_module_defs).difference(main.module_defs)
if len(new_module_defs) > 0:
position = main.post_js.find('Runtime.') # Runtime is the start of the hardcoded ones
main.post_js = main.post_js[:position] + ''.join(list(new_module_defs)) + '\n' + main.post_js[position:]
CONSTRUCTOR_CLOSURE_SUPPRESSIONS = '/** @suppress {undefinedVars, duplicate} @this{Object} */'
class Dummy:
def __init__(self, init):
for k, v in init.items():
self.__dict__[k] = v
def getExtendedAttribute(self, name): # noqa: U100
return None
input_file = sys.argv[1]
output_base = sys.argv[2]
shared.try_delete(output_base + '.cpp')
shared.try_delete(output_base + '.js')
p = WebIDL.Parser()
p.parse(r'''
interface VoidPtr {
};
''' + utils.read_file(input_file))
data = p.finish()
interfaces = {}
implements = {}
enums = {}
for thing in data:
if isinstance(thing, WebIDL.IDLInterface):
def test_binaryen(self):
import tools.ports.binaryen as binaryen
tag_file = Cache.get_path('binaryen_tag_' + binaryen.TAG + '.txt')
assert not os.environ.get('BINARYEN') # must not have binaryen env var set
# test in 2 modes - with BINARYEN_ROOT in the config file, set to '', and without it entirely
for binaryen_root_in_config in [1, 0]:
print('binaryen_root_in_config:', binaryen_root_in_config)
def prep():
restore_and_set_up()
print('clearing ports...')
print(self.do([PYTHON, EMCC, '--clear-ports']))
wipe()
self.do([PYTHON, EMCC]) # first run stage
try_delete(tag_file)
# if BINARYEN_ROOT is set, we don't build the port. Check we do build it if not
if binaryen_root_in_config:
config = open(CONFIG_FILE).read()
assert '''BINARYEN_ROOT = os.path.expanduser(os.getenv('BINARYEN', ''))''' in config, config # setup created it to be ''
print('created config:')
print(config)
restore_and_set_up()
config = open(CONFIG_FILE).read()
config = config.replace('BINARYEN_ROOT', '''BINARYEN_ROOT = os.path.expanduser(os.getenv('BINARYEN', '')) # ''')
else:
restore_and_set_up()
config = open(CONFIG_FILE).read()
config = config.replace('BINARYEN_ROOT', '#')
print('modified config:')
print(config)
open(CONFIG_FILE, 'w').write(config)
print('build using embuilder')
prep()
run_process([PYTHON, EMBUILDER, 'build', 'binaryen'])
assert os.path.exists(tag_file)
run_process([PYTHON, EMCC] + MINIMAL_HELLO_WORLD + ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'])
self.assertContained('hello, world!', run_js('a.out.js'))
print('see we show an error for emmake (we cannot build natively under emmake)')
prep()
try_delete('a.out.js')
out = self.do([PYTHON, path_from_root('emmake.py'), EMCC] + MINIMAL_HELLO_WORLD + ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'])
assert not os.path.exists(tag_file)
assert not os.path.exists('a.out.js')
self.assertContained('For example, for binaryen, do "python embuilder.py build binaryen"', out)
if not binaryen_root_in_config:
print('build on demand')
for side_module in (False, True):
print(side_module)
prep()
assert not os.path.exists(tag_file)
try_delete('a.out.js')
try_delete('a.out.wasm')
cmd = [PYTHON, EMCC]
if not side_module:
cmd += MINIMAL_HELLO_WORLD
else:
# EM_ASM doesn't work in a wasm side module, build a normal program
cmd += [path_from_root('tests', 'hello_world.c'), '-s', 'SIDE_MODULE=1']
cmd += ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"']
run_process(cmd)
assert os.path.exists(tag_file)
assert os.path.exists('a.out.wasm')
if not side_module:
assert os.path.exists('a.out.js')
self.assertContained('hello, world!', run_js('a.out.js'))