def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
self.old_env = os.environ
os.environ = self.env.copy()
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
env_init = self.env.copy()
# Note that we need to pass in all the flags here because some build
# systems (like zlib) if they see a CFLAGS it will override all their
# default flags, including optimizations.
env_init['CFLAGS'] = ' '.join(LLVM_FEATURE_FLAGS + [OPTIMIZATIONS] + self.extra_args)
emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=env_init)
final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
EMCC, filename,
OPTIMIZATIONS,
'-s', 'INITIAL_MEMORY=256MB',
'-s', 'FILESYSTEM=0',
'--closure', '1',
'-s', 'MINIMAL_RUNTIME=1',
'-s', 'BENCHMARK=%d' % (1 if IGNORE_COMPILATION and not has_output_parser else 0),
'-o', final
] + shared_args + emcc_args + LLVM_FEATURE_FLAGS + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd]
if PROFILING:
cmd += ['--profiling-funcs']
self.cmd = cmd
run_process(cmd, env=self.env)
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
self.parent = parent
if lib_builder:
env = {
'CC': self.cc,
'CXX': self.cxx,
'CXXFLAGS': "-Wno-c++11-narrowing"
}
env.update(clang_native.get_clang_native_env())
native_args = native_args + lib_builder(
self.name, native=True, env_init=env)
if not native_exec:
compiler = self.cxx if filename.endswith('cpp') else self.cc
cmd = [
compiler, '-fno-math-errno', filename, '-o',
filename + '.native'
] + self.args + shared_args + native_args + clang_native.get_clang_native_args(
)
# print(cmd)
run_process(cmd, env=clang_native.get_clang_native_env())
else:
shutil.copyfile(native_exec, filename + '.native')
shutil.copymode(native_exec, filename + '.native')
final = os.path.dirname(
filename) + os.path.sep + self.name + '_' + os.path.basename(
filename) + '.native'
shutil.move(filename + '.native', final)
self.filename = final
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
self.old_env = os.environ
os.environ = self.env.copy()
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
emcc_args = emcc_args + lib_builder(
'js_' + llvm_root, native=False, env_init=self.env.copy())
final = os.path.dirname(filename) + os.path.sep + self.name + (
'_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
PYTHON, EMCC, filename, OPTIMIZATIONS, '-s', 'TOTAL_MEMORY=256MB',
'-s', 'FILESYSTEM=0', '--closure', '1', '-s', 'MINIMAL_RUNTIME=0',
'-s',
'BENCHMARK=%d' %
(1 if IGNORE_COMPILATION and not has_output_parser else 0), '-o',
final
] + shared_args + emcc_args + LLVM_FEATURE_FLAGS + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [
arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd
]
if PROFILING:
cmd += ['--profiling-funcs']
self.cmd = cmd
run_process(cmd, env=self.env)
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
# wasm2c doesn't want minimal runtime which the normal emscripten
# benchmarker defaults to, as we don't have any JS anyhow
emcc_args = emcc_args + [
'-s', 'STANDALONE_WASM', '-s', 'MINIMAL_RUNTIME=0', '-s', 'WASM2C'
]
global LLVM_FEATURE_FLAGS
old_flags = LLVM_FEATURE_FLAGS
try:
# wasm2c does not support anything beyond MVP
LLVM_FEATURE_FLAGS = []
super(EmscriptenWasm2CBenchmarker,
self).build(parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder,
has_output_parser)
finally:
LLVM_FEATURE_FLAGS = old_flags
# move the JS away so there is no chance we run it by mistake
shutil.move(self.filename, self.filename + '.old.js')
base = self.filename[:-3]
c = base + '.wasm.c'
native = base + '.exe'
run_process([
'clang', c, '-o', native, OPTIMIZATIONS, '-lm',
'-DWASM_RT_MAX_CALL_STACK_DEPTH=8000'
]) # for havlak
self.filename = native
def try_js(args=[]):
shared.try_delete(filename + '.js')
js_args = [shared.EMCC, fullname, '-o', filename + '.js'
] + [opts] + llvm_opts + CSMITH_CFLAGS + args + ['-w']
if TEST_BINARYEN:
if random.random() < 0.5:
js_args += ['-g']
if random.random() < 0.5:
# pick random passes
BINARYEN_EXTRA_PASSES = [
"code-pushing",
"duplicate-function-elimination",
"dce",
"remove-unused-brs",
"remove-unused-names",
"local-cse",
"optimize-instructions",
"post-emscripten",
"precompute",
"simplify-locals",
"simplify-locals-nostructure",
"vacuum",
"coalesce-locals",
"reorder-locals",
"merge-blocks",
"remove-unused-module-elements",
"memory-packing",
]
passes = []
while 1:
passes.append(random.choice(BINARYEN_EXTRA_PASSES))
if random.random() < 0.1:
break
js_args += [
'-s', 'BINARYEN_EXTRA_PASSES="' + ','.join(passes) + '"'
]
if random.random() < 0.5:
js_args += ['-s', 'ALLOW_MEMORY_GROWTH=1']
if random.random(
) < 0.5 and 'ALLOW_MEMORY_GROWTH=1' not in js_args and 'BINARYEN=1' not in js_args:
js_args += ['-s', 'MAIN_MODULE=1']
if random.random() < 0.25:
js_args += ['-s', 'INLINING_LIMIT=1'
] # inline nothing, for more call interaction
if random.random() < 0.5:
js_args += ["--memory-init-file", "0", "-s", "MEM_INIT_METHOD=2"]
if random.random() < 0.5:
js_args += ['-s', 'ASSERTIONS=1']
print('(compile)', ' '.join(js_args))
short_args = [shared.EMCC, fail_output_name] + js_args[5:]
escaped_short_args = map(lambda x: ("'" + x + "'")
if '"' in x else x, short_args)
open(fullname,
'a').write('\n// ' + ' '.join(escaped_short_args) + '\n\n')
try:
shared.run_process(js_args)
assert os.path.exists(filename + '.js')
return js_args
except Exception:
return False
def setUpClass(cls):
super(benchmark, cls).setUpClass()
for benchmarker in benchmarkers:
benchmarker.prepare()
fingerprint = [
'ignoring compilation'
if IGNORE_COMPILATION else 'including compilation',
time.asctime()
]
try:
fingerprint.append('em: ' + run_process(
['git', 'show'], stdout=PIPE).stdout.splitlines()[0])
except Exception:
pass
try:
with runner.chdir(os.path.expanduser('~/Dev/mozilla-central')):
fingerprint.append('sm: ' + [
line for line in run_process(['hg', 'tip'], stdout=PIPE).
stdout.splitlines() if 'changeset' in line
][0])
except Exception:
pass
fingerprint.append('llvm: ' + config.LLVM_ROOT)
print('Running Emscripten benchmarks... [ %s ]' %
' | '.join(fingerprint))
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [
shared.CLANG_CXX, 'g++', 'clang++'
]: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
shared.run_process([
compiler,
shared.path_from_root('tools', 'optimizer',
'parser.cpp'),
shared.path_from_root('tools', 'optimizer',
'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-shared.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-main.cpp'), '-O3',
'-std=c++11', '-fno-exceptions', '-fno-rtti', '-o',
output
] + args,
stdout=log_output,
stderr=log_output)
except Exception as e:
logging.debug(str(e))
continue # perhaps the later compilers will succeed
# success
return output
shared.exit_with_error('Failed to build native optimizer')
def test_emcc_cache_flag(self):
restore_and_set_up()
cache_dir_name = self.in_dir('emscripten_cache')
self.assertFalse(os.path.exists(cache_dir_name))
create_test_file(
'test.c', r'''
#include <stdio.h>
int main() {
printf("hello, world!\n");
return 0;
}
''')
run_process([PYTHON, EMCC, 'test.c', '--cache', cache_dir_name],
stderr=PIPE)
# The cache directory must exist after the build
self.assertTrue(os.path.exists(cache_dir_name))
# The cache directory must contain a built libc'
if self.is_wasm_backend():
self.assertTrue(
os.path.exists(os.path.join(cache_dir_name, 'wasm_o',
'libc.a')))
else:
self.assertTrue(
os.path.exists(os.path.join(cache_dir_name, 'asmjs',
'libc.bc')))
def test_emconfig(self):
restore_and_set_up()
fd, custom_config_filename = tempfile.mkstemp(
prefix='.emscripten_config_')
orig_config = open(CONFIG_FILE, 'r').read()
# Move the ~/.emscripten to a custom location.
with os.fdopen(fd, "w") as f:
f.write(orig_config)
# Make a syntax error in the original config file so that attempting to access it would fail.
open(CONFIG_FILE, 'w').write('asdfasdfasdfasdf\n\'\'\'' + orig_config)
temp_dir = tempfile.mkdtemp(prefix='emscripten_temp_')
with chdir(temp_dir):
run_process([EMCC, '--em-config', custom_config_filename] +
MINIMAL_HELLO_WORLD + ['-O2'])
result = run_js('a.out.js')
self.assertContained('hello, world!', result)
# Clean up created temp files.
os.remove(custom_config_filename)
if Settings.WASM_BACKEND:
os.remove(custom_config_filename + "_sanity_wasm")
else:
os.remove(custom_config_filename + "_sanity")
shutil.rmtree(temp_dir)
def __enter__(self):
# assuming this is only used for WebSocket tests at the moment, validate that
# the ws module is installed
child = run_process(NODE_JS + ['-e', 'require("ws");'], check=False)
global node_ws_module_installed
# Attempt to automatically install ws module for Node.js.
if child.returncode != 0 and not node_ws_module_installed:
node_ws_module_installed = True
run_process(
[NPM, 'install',
path_from_root('tests', 'sockets', 'ws')],
cwd=os.path.dirname(EMCC))
# Did installation succeed?
child = run_process(NODE_JS + ['-e', 'require("ws");'],
check=False)
assert child.returncode == 0, 'ws module for Node.js not installed, and automatic installation failed! Please run \'npm install\' from %s' % shared.__rootpath__
# compile the server
proc = run_process([
PYTHON, EMCC,
path_from_root('tests', self.filename), '-o', 'server.js',
'-DSOCKK=%d' % self.listen_port
] + self.args)
print('Socket server build: out:', proc.stdout or '', '/ err:',
proc.stderr or '')
process = Popen(NODE_JS + ['server.js'])
self.processes.append(process)
def setUpClass(self):
super(benchmark, self).setUpClass()
fingerprint = [
'ignoring compilation'
if IGNORE_COMPILATION else 'including compilation',
time.asctime()
]
try:
fingerprint.append('em: ' + run_process(
['git', 'show'], stdout=PIPE).stdout.splitlines()[0])
except:
pass
try:
with chdir(os.path.expanduser('~/Dev/mozilla-central')):
fingerprint.append('sm: ' + [
line for line in run_process(['hg', 'tip'], stdout=PIPE).
stdout.splitlines() if 'changeset' in line
][0])
except:
pass
fingerprint.append('llvm: ' + LLVM_ROOT)
print('Running Emscripten benchmarks... [ %s ]' %
' | '.join(fingerprint))
assert (os.path.exists(CLOSURE_COMPILER))
Building.COMPILER = CLANG
Building.COMPILER_TEST_OPTS = [OPTIMIZATIONS]
def run():
if shared.Settings.WASM_BACKEND:
# The wasm backend does suffer from the same probllem as fastcomp so doesn't
# need the filename hashing.
cmd = [shared.LLVM_AR] + sys.argv[1:]
return shared.run_process(cmd, stdin=sys.stdin, check=False).returncode
try:
args = substitute_response_files(sys.argv)
except IOError as e:
shared.exit_with_error(e)
newargs = [shared.LLVM_AR] + args[1:]
tmpdir = None
response_filename = None
# The 3 argmuent form of ar doesn't involve other files. For example
# 'ar x libfoo.a'.
if len(newargs) > 3:
tmpdir = tempfile.mkdtemp(prefix='emar-')
cmd = newargs[1]
if 'r' in cmd or 'q' in cmd:
# We are adding files to the archive.
# Normally the output file is then arg 2, except in the case were the
# a or b modifiers are used in which case its arg 3.
if 'a' in cmd or 'b' in cmd:
out_arg_index = 3
else:
out_arg_index = 2
# Add a hash to colliding basename, to make them unique.
for j in range(out_arg_index + 1, len(newargs)):
orig_name = newargs[j]
full_name = os.path.abspath(orig_name)
basename = os.path.basename(full_name)
h = hashlib.md5(full_name.encode('utf-8')).hexdigest()[:8]
parts = basename.split('.')
parts[0] += '_' + h
newname = '.'.join(parts)
full_newname = os.path.join(tmpdir, newname)
shutil.copyfile(orig_name, full_newname)
newargs[j] = full_newname
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
response_filename = create_response_file(
newargs[3:], shared.get_emscripten_temp_dir())
newargs = newargs[:3] + ['@' + response_filename]
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
rtn = shared.run_process(newargs, stdin=sys.stdin, check=False).returncode
if tmpdir:
shutil.rmtree(tmpdir)
shared.try_delete(response_filename)
return rtn
def test_embuilder_wasm_backend(self):
if not Settings.WASM_BACKEND:
self.skipTest('wasm backend only')
restore_and_set_up()
# the --lto flag makes us build wasm-bc
self.do([EMCC, '--clear-cache'])
run_process([EMBUILDER, 'build', 'libemmalloc'])
self.assertExists(os.path.join(shared.CACHE, 'wasm'))
self.do([EMCC, '--clear-cache'])
run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
self.assertExists(os.path.join(shared.CACHE, 'wasm-lto'))
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
cheerp_args = [
'-fno-math-errno',
]
cheerp_args += self.args
self.parent = parent
if lib_builder:
# build as "native" (so no emcc env stuff), but with all the cheerp stuff
# set in the env
cheerp_args = cheerp_args + lib_builder(
self.name,
native=True,
env_init={
'CC': CHEERP_BIN + 'clang',
'CXX': CHEERP_BIN + 'clang++',
'AR': CHEERP_BIN + '../libexec/cheerp-unknown-none-ar',
'LD': CHEERP_BIN + 'clang',
'NM': CHEERP_BIN + 'llvm-nm',
'LDSHARED': CHEERP_BIN + 'clang',
'RANLIB':
CHEERP_BIN + '../libexec/cheerp-unknown-none-ranlib',
'CXXFLAGS': "-Wno-c++11-narrowing",
'CHEERP_PREFIX': CHEERP_BIN + '../',
})
if PROFILING:
cheerp_args += ['-cheerp-pretty-code'
] # get function names, like emcc --profiling
final = os.path.dirname(filename) + os.path.sep + self.name + (
'_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
dirs_to_delete = []
cheerp_args += ['-cheerp-preexecute']
try:
# print(cheerp_args)
if filename.endswith('.c'):
compiler = CHEERP_BIN + '/clang'
else:
compiler = CHEERP_BIN + '/clang++'
cmd = [compiler] + cheerp_args + [
'-cheerp-linear-heap-size=256',
'-cheerp-secondary-output-file=' +
final.replace('.js', '.wasm'), filename, '-o', final
] + shared_args
# print(' '.join(cmd))
run_process(cmd, stdout=PIPE, stderr=PIPE)
self.filename = final
if self.binaryen_opts:
run_binaryen_opts(final.replace('.js', '.wasm'),
self.binaryen_opts)
finally:
for dir_ in dirs_to_delete:
try_delete(dir_)
def test_embuilder_wasm_backend(self):
if not Settings.WASM_BACKEND:
self.skipTest('wasm backend only')
restore_and_set_up()
root_cache = os.path.expanduser('~/.emscripten_cache')
# the --lto flag makes us build wasm-bc
self.do([EMCC, '--clear-cache'])
run_process([EMBUILDER, 'build', 'libemmalloc'])
self.assertExists(os.path.join(root_cache, 'wasm-obj'))
self.do([EMCC, '--clear-cache'])
run_process([EMBUILDER, 'build', 'libemmalloc', '--lto'])
self.assertExists(os.path.join(root_cache, 'wasm-bc'))
def test_posix_proxy_sockets(self):
# Build the websocket bridge server
run_process(['cmake', path_from_root('tools', 'websocket_to_posix_proxy')])
run_process(['cmake', '--build', '.'])
if os.name == 'nt': # This is not quite exact, instead of "isWindows()" this should be "If CMake defaults to building with Visual Studio", but there is no good check for that, so assume Windows==VS.
proxy_server = os.path.join(self.get_dir(), 'Debug', 'websocket_to_posix_proxy.exe')
else:
proxy_server = os.path.join(self.get_dir(), 'websocket_to_posix_proxy')
with BackgroundServerProcess([proxy_server, '8080']):
with PythonTcpEchoServerProcess('7777'):
# Build and run the TCP echo client program with Emscripten
self.btest(path_from_root('tests', 'websocket', 'tcp_echo_client.cpp'), expected='101', args=['-lwebsocket', '-s', 'PROXY_POSIX_SOCKETS=1', '-s', 'USE_PTHREADS=1', '-s', 'PROXY_TO_PTHREAD=1'])
def test_enet(self):
# this is also a good test of raw usage of emconfigure and emmake
shared.try_delete('enet')
shutil.copytree(path_from_root('tests', 'enet'), 'enet')
with chdir('enet'):
run_process([path_from_root('emconfigure'), './configure'])
run_process([path_from_root('emmake'), 'make'])
enet = [self.in_dir('enet', '.libs', 'libenet.a'), '-I' + path_from_root('tests', 'enet', 'include')]
for harness in [
CompiledServerHarness(os.path.join('sockets', 'test_enet_server.c'), enet, 49210)
]:
with harness:
self.btest(os.path.join('sockets', 'test_enet_client.c'), expected='0', args=enet + ['-DSOCKK=%d' % harness.listen_port])
def test_enet(self):
# this is also a good test of raw usage of emconfigure and emmake
shared.try_delete(self.in_dir('enet'))
shutil.copytree(path_from_root('tests', 'enet'), self.in_dir('enet'))
with chdir(self.in_dir('enet')):
run_process([PYTHON, path_from_root('emconfigure'), './configure'])
run_process([PYTHON, path_from_root('emmake'), 'make'])
enet = [self.in_dir('enet', '.libs', 'libenet.a'), '-I' + path_from_root('tests', 'enet', 'include')]
for harness in [
CompiledServerHarness(os.path.join('sockets', 'test_enet_server.c'), enet, 49210)
]:
with harness:
self.btest(os.path.join('sockets', 'test_enet_client.c'), expected='0', args=enet + ['-DSOCKK=%d' % harness.listen_port])
def __enter__(self):
# compile the server
# NOTE empty filename support is a hack to support
# the current test_enet
if self.filename:
cmd = [
CLANG_CC,
test_file(self.filename), '-o', 'server',
'-DSOCKK=%d' % self.target_port
] + clang_native.get_clang_native_args() + self.args
print(cmd)
run_process(cmd, env=clang_native.get_clang_native_env())
process = Popen([os.path.abspath('server')])
self.processes.append(process)
import websockify
# start the websocket proxy
print('running websockify on %d, forward to tcp %d' %
(self.listen_port, self.target_port),
file=sys.stderr)
wsp = websockify.WebSocketProxy(verbose=True,
listen_port=self.listen_port,
target_host="127.0.0.1",
target_port=self.target_port,
run_once=True)
self.websockify = multiprocessing.Process(target=wsp.start_server)
self.websockify.start()
self.processes.append(self.websockify)
# Make sure both the actual server and the websocket proxy are running
for _ in range(10):
try:
if self.do_server_check:
server_sock = socket.create_connection(
('localhost', self.target_port), timeout=1)
server_sock.close()
proxy_sock = socket.create_connection(
('localhost', self.listen_port), timeout=1)
proxy_sock.close()
break
except IOError:
time.sleep(1)
else:
clean_processes(self.processes)
raise Exception(
'[Websockify failed to start up in a timely manner]')
print('[Websockify on process %s]' % str(self.processes[-2:]))
return self
def __enter__(self):
# assuming this is only used for WebSocket tests at the moment, validate that
# the ws module is installed
global npm_checked
if not npm_checked:
child = run_process(NODE_JS + ['-e', 'require("ws");'], check=False)
assert child.returncode == 0, '"ws" node module not found. you may need to run npm install'
npm_checked = True
# compile the server
proc = run_process([EMCC, path_from_root('tests', self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args)
print('Socket server build: out:', proc.stdout or '', '/ err:', proc.stderr or '')
process = Popen(NODE_JS + ['server.js'])
self.processes.append(process)
def test_sdl2_mixer_mp3(self):
shutil.copyfile(path_from_root('tests', 'sounds', 'pudinha.mp3'),
os.path.join(self.get_dir(), 'sound.mp3'))
open(os.path.join(self.get_dir(), 'sdl2_mixer_mp3.c'), 'w').write(
self.with_report_result(
open(path_from_root('tests', 'sdl2_mixer_mp3.c')).read()))
run_process([
EMCC, '-O2', '--minify', '0',
os.path.join(self.get_dir(), 'sdl2_mixer_mp3.c'), '-s',
'USE_SDL=2', '-s', 'USE_SDL_MIXER=2', '-s',
'SDL2_MIXER_FORMATS=["mp3"]', '-s', 'INITIAL_MEMORY=33554432',
'--preload-file', 'sound.mp3', '-o', 'page.html'
])
self.run_browser('page.html', '', '/report_result?1')
def validate_asmjs_jsfile(filename, muteOutput):
cmd = shared.SPIDERMONKEY_ENGINE + ['-c', filename]
if not shared.SPIDERMONKEY_ENGINE or cmd[0] == 'js-not-found' or len(
cmd[0].strip()) == 0:
print(
'Could not find SpiderMonkey engine! Please set its location to SPIDERMONKEY_ENGINE in your '
+ shared.hint_config_file_location() + ' configuration file!',
file=sys.stderr)
return False
try:
process = shared.run_process(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE)
except Exception as e:
print('Executing command ' + str(cmd) +
' failed due to an exception: ' + str(e) + '!',
file=sys.stderr)
return False
stdout = process.stdout
stderr = process.stderr
if not muteOutput:
if len(stdout.strip()):
print(stdout.strip())
if len(stderr.strip()):
# Pretty-print the output not to contain a spurious warning.
warning_re = re.compile(
re.escape('warning: successfully compiled asm.js'),
re.IGNORECASE)
stderr = warning_re.sub(' successfully compiled asm.js', stderr)
print(stderr.strip(), file=sys.stderr)
if 'successfully compiled asm.js' in stderr.lower():
return True
else:
return False
def run_on_chunk(command):
try:
if JS_OPTIMIZER in command: # XXX hackish
index = command.index(JS_OPTIMIZER)
filename = command[index + 1]
else:
filename = command[1]
if os.environ.get('EMCC_SAVE_OPT_TEMP') and os.environ.get('EMCC_SAVE_OPT_TEMP') != '0':
saved = 'save_' + os.path.basename(filename)
while os.path.exists(saved): saved = 'input' + str(int(saved.replace('input', '').replace('.txt', ''))+1) + '.txt'
print('running js optimizer command', ' '.join([c if c != filename else saved for c in command]), file=sys.stderr)
shutil.copyfile(filename, os.path.join(shared.get_emscripten_temp_dir(), saved))
if shared.EM_BUILD_VERBOSE >= 3: print('run_on_chunk: ' + str(command), file=sys.stderr)
proc = shared.run_process(command, stdout=subprocess.PIPE)
output = proc.stdout
assert proc.returncode == 0, 'Error in optimizer (return code ' + str(proc.returncode) + '): ' + output
assert len(output) and not output.startswith('Assertion failed'), 'Error in optimizer: ' + output
filename = temp_files.get(os.path.basename(filename) + '.jo.js').name
f = open(filename, 'w')
f.write(output)
f.close()
if DEBUG and not shared.WINDOWS: print('.', file=sys.stderr) # Skip debug progress indicator on Windows, since it doesn't buffer well with multiple threads printing to console.
return filename
except KeyboardInterrupt:
# avoid throwing keyboard interrupts from a child process
raise Exception()
def test_em_config_env_var(self):
# emcc should be configurable directly from EM_CONFIG without any config file
restore_and_set_up()
create_test_file('main.cpp', '''
#include <stdio.h>
int main() {
printf("hello from emcc with no config file\\n");
return 0;
}
''')
wipe()
with env_modify({'EM_CONFIG': get_basic_config()}):
run_process([EMCC, 'main.cpp', '-o', 'a.out.js'])
self.assertContained('hello from emcc with no config file', run_js('a.out.js'))
def build(self, parent, filename, args, shared_args, emcc_args,
native_args, native_exec, lib_builder, has_output_parser):
self.parent = parent
if lib_builder:
native_args += lib_builder(self.name,
native=True,
env_init={
'CC': self.cc,
'CXX': self.cxx
})
if not native_exec:
compiler = self.cxx if filename.endswith('cpp') else self.cc
cmd = [
compiler, '-fno-math-errno', filename, '-o',
filename + '.native'
] + self.args + shared_args + native_args + shared.get_clang_native_args(
)
proc = run_process(cmd,
stdout=PIPE,
stderr=parent.stderr_redirect,
env=shared.get_clang_native_env())
if proc.returncode != 0:
print("Building native executable with command failed",
' '.join(cmd),
file=sys.stderr)
print("Output: " + str(proc.stdout) + '\n' + str(proc.stderr))
else:
shutil.copyfile(native_exec, filename + '.native')
shutil.copymode(native_exec, filename + '.native')
final = os.path.dirname(
filename) + os.path.sep + self.name + '_' + os.path.basename(
filename) + '.native'
shutil.move(filename + '.native', final)
self.filename = final
def generate_symbol_file(symbol_file, lib_file):
"""Regenerate the contents of a given symbol file."""
output = shared.run_process([shared.LLVM_NM, '-g', lib_file, '-defined-only'],
stdout=shared.PIPE).stdout
new_symbols = filter_and_sort(output)
with open(symbol_file, 'w') as f:
f.write(new_symbols)
def handle_symbol_file(args, symbol_file):
"""Regenerate the contents of a given symbol file."""
if args.filter_in_place:
output = open(symbol_file).read()
else:
basename = os.path.splitext(os.path.basename(symbol_file))[0]
if not basename.startswith('lib'):
basename = 'lib' + basename
if basename.endswith('_asmjs'):
basename = basename.rsplit('_asmjs', 1)[0]
basename = basename.replace('cxx', 'c++')
cache_dir = cache.Cache().dirname
pattern = os.path.join(cache_dir, basename + '.*')
libs = glob.glob(pattern)
if basename == 'libgl':
# For libgl we generate the symbol list based on a superset of all
# library variants.
pattern = os.path.join(cache_dir, basename + '-*.*')
libs += glob.glob(pattern)
if not libs:
print(cache_dir)
print("%s: Unable to find library to generate symbols from" %
symbol_file)
return
print('Generating %s based on syms from: %s' % (basename, libs))
output = ''
for lib in libs:
output += shared.run_process([shared.LLVM_NM, '-g', lib],
stdout=shared.PIPE).stdout
new_symbols = filter_and_sort(output)
with open(symbol_file, 'w') as f:
f.write(new_symbols)
def run_on_chunk(command):
try:
if JS_OPTIMIZER in command: # XXX hackish
index = command.index(JS_OPTIMIZER)
filename = command[index + 1]
else:
filename = command[1]
if os.environ.get('EMCC_SAVE_OPT_TEMP') and os.environ.get('EMCC_SAVE_OPT_TEMP') != '0':
saved = 'save_' + os.path.basename(filename)
while os.path.exists(saved): saved = 'input' + str(int(saved.replace('input', '').replace('.txt', ''))+1) + '.txt'
print('running js optimizer command', ' '.join([c if c != filename else saved for c in command]), file=sys.stderr)
shutil.copyfile(filename, os.path.join(shared.get_emscripten_temp_dir(), saved))
if shared.EM_BUILD_VERBOSE >= 3: print('run_on_chunk: ' + str(command), file=sys.stderr)
proc = shared.run_process(command, stdout=subprocess.PIPE)
output = proc.stdout
assert proc.returncode == 0, 'Error in optimizer (return code ' + str(proc.returncode) + '): ' + output
assert len(output) and not output.startswith('Assertion failed'), 'Error in optimizer: ' + output
filename = temp_files.get(os.path.basename(filename) + '.jo.js').name
with open(filename, 'w') as f:
f.write(output)
if DEBUG and not shared.WINDOWS: print('.', file=sys.stderr) # Skip debug progress indicator on Windows, since it doesn't buffer well with multiple threads printing to console.
return filename
except KeyboardInterrupt:
# avoid throwing keyboard interrupts from a child process
raise Exception()
def __enter__(self):
# compile the server
# NOTE empty filename support is a hack to support
# the current test_enet
if self.filename:
proc = run_process([CLANG_CC, path_from_root('tests', self.filename), '-o', 'server', '-DSOCKK=%d' % self.target_port] + shared.get_clang_native_args() + self.args, env=shared.get_clang_native_env(), stdout=PIPE, stderr=PIPE)
print('Socket server build: out:', proc.stdout or '', '/ err:', proc.stderr or '')
process = Popen([os.path.abspath('server')])
self.processes.append(process)
# start the websocket proxy
print('running websockify on %d, forward to tcp %d' % (self.listen_port, self.target_port), file=sys.stderr)
wsp = websockify.WebSocketProxy(verbose=True, listen_port=self.listen_port, target_host="127.0.0.1", target_port=self.target_port, run_once=True)
self.websockify = multiprocessing.Process(target=wsp.start_server)
self.websockify.start()
self.processes.append(self.websockify)
# Make sure both the actual server and the websocket proxy are running
for i in range(10):
try:
if self.do_server_check:
server_sock = socket.create_connection(('localhost', self.target_port), timeout=1)
server_sock.close()
proxy_sock = socket.create_connection(('localhost', self.listen_port), timeout=1)
proxy_sock.close()
break
except:
time.sleep(1)
else:
clean_processes(self.processes)
raise Exception('[Websockify failed to start up in a timely manner]')
print('[Websockify on process %s]' % str(self.processes[-2:]))
def test_emcc_multiprocess_cache_access(self):
restore_and_set_up()
create_test_file('test.c', r'''
#include <stdio.h>
int main() {
printf("hello, world!\n");
return 0;
}
''')
cache_dir_name = self.in_dir('test_cache')
with env_modify({'EM_CACHE': cache_dir_name}):
tasks = []
num_times_libc_was_built = 0
for i in range(3):
p = run_process([EMCC, 'test.c', '-o', '%d.js' % i], stderr=STDOUT, stdout=PIPE)
tasks += [p]
for p in tasks:
print('stdout:\n', p.stdout)
if 'generating system library: libc' in p.stdout:
num_times_libc_was_built += 1
# The cache directory must exist after the build
self.assertTrue(os.path.exists(cache_dir_name))
# The cache directory must contain a built libc
if self.is_wasm_backend():
self.assertTrue(os.path.exists(os.path.join(cache_dir_name, 'wasm-obj', 'libc.a')))
else:
self.assertTrue(os.path.exists(os.path.join(cache_dir_name, 'asmjs', 'libc.bc')))
# Exactly one child process should have triggered libc build!
self.assertEqual(num_times_libc_was_built, 1)
def do(self, command):
print('Running: ' + ' '.join(command))
if type(command) is not list:
command = [command]
if command[0] == EMCC:
command = [PYTHON] + command
return run_process(command, stdout=PIPE, stderr=STDOUT, check=False).stdout
def do(self, command):
print('Running: ' + ' '.join(command))
if type(command) is not list:
command = [command]
if command[0] == EMCC:
command = [PYTHON] + command
return run_process(command, stdout=PIPE, stderr=STDOUT, check=False).stdout
def run():
try:
args = substitute_response_files(sys.argv)
except IOError as e:
shared.exit_with_error(e)
newargs = [shared.LLVM_AR] + args[1:]
to_delete = []
# The 3 argmuent form of ar doesn't involve other files. For example
# 'ar x libfoo.a'.
if len(newargs) > 3:
cmd = newargs[1]
if 'r' in cmd or 'q' in cmd:
# We are adding files to the archive.
# Normally the output file is then arg 2, except in the case were the
# a or b modifiers are used in which case its arg 3.
if 'a' in cmd or 'b' in cmd:
out_arg_index = 3
else:
out_arg_index = 2
# Add a hash to colliding basename, to make them unique.
for j in range(out_arg_index + 1, len(newargs)):
orig_name = newargs[j]
full_name = os.path.abspath(orig_name)
dirname = os.path.dirname(full_name)
basename = os.path.basename(full_name)
h = hashlib.md5(full_name.encode('utf-8')).hexdigest()[:8]
parts = basename.split('.')
parts[0] += '_' + h
newname = '.'.join(parts)
full_newname = os.path.join(dirname, newname)
try:
shutil.copyfile(orig_name, full_newname)
newargs[j] = full_newname
to_delete.append(full_newname)
except Exception:
# it is ok to fail here, we just don't get hashing
pass
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
response_filename = create_response_file(
newargs[3:], shared.get_emscripten_temp_dir())
to_delete += [response_filename]
newargs = newargs[:3] + ['@' + response_filename]
if shared.DEBUG:
print('emar:', sys.argv, ' ==> ', newargs, file=sys.stderr)
rtn = shared.run_process(newargs, stdin=sys.stdin, check=False).returncode
for d in to_delete:
shared.try_delete(d)
return rtn
def do(self, command, env=None):
print('Running: ' + ' '.join(command))
if type(command) is not list:
command = [command]
return run_process(command,
stdout=PIPE,
stderr=STDOUT,
check=False,
env=env).stdout
def __enter__(self):
# assuming this is only used for WebSocket tests at the moment, validate that
# the ws module is installed
child = run_process(NODE_JS + ['-e', 'require("ws");'], check=False)
global node_ws_module_installed
# Attempt to automatically install ws module for Node.js.
if child.returncode != 0 and not node_ws_module_installed:
node_ws_module_installed = True
run_process([NPM, 'install', path_from_root('tests', 'sockets', 'ws')], cwd=os.path.dirname(EMCC))
# Did installation succeed?
child = run_process(NODE_JS + ['-e', 'require("ws");'], check=False)
assert child.returncode == 0, 'ws module for Node.js not installed, and automatic installation failed! Please run \'npm install\' from %s' % shared.__rootpath__
# compile the server
proc = run_process([PYTHON, EMCC, path_from_root('tests', self.filename), '-o', 'server.js', '-DSOCKK=%d' % self.listen_port] + self.args)
print('Socket server build: out:', proc.stdout or '', '/ err:', proc.stderr or '')
process = Popen(NODE_JS + ['server.js'])
self.processes.append(process)
def setUpClass(self):
super(benchmark, self).setUpClass()
fingerprint = ['ignoring compilation' if IGNORE_COMPILATION else 'including compilation', time.asctime()]
try:
fingerprint.append('em: ' + run_process(['git', 'show'], stdout=PIPE).stdout.splitlines()[0])
except:
pass
try:
with chdir(os.path.expanduser('~/Dev/mozilla-central')):
fingerprint.append('sm: ' + [line for line in run_process(['hg', 'tip'], stdout=PIPE).stdout.splitlines() if 'changeset' in line][0])
except:
pass
fingerprint.append('llvm: ' + LLVM_ROOT)
print('Running Emscripten benchmarks... [ %s ]' % ' | '.join(fingerprint))
assert(os.path.exists(CLOSURE_COMPILER))
Building.COMPILER = CLANG
Building.COMPILER_TEST_OPTS = [OPTIMIZATIONS]
def test_nostdincxx(self):
restore_and_set_up()
Cache.erase()
for compiler in [EMCC]:
print(compiler)
run_process([PYTHON, EMCC] + MINIMAL_HELLO_WORLD + ['-v']) # run once to ensure binaryen port is all ready
proc = run_process([PYTHON, EMCC] + MINIMAL_HELLO_WORLD + ['-v'], stdout=PIPE, stderr=PIPE)
out = proc.stdout
err = proc.stderr
proc2 = run_process([PYTHON, EMCC] + MINIMAL_HELLO_WORLD + ['-v', '-nostdinc++'], stdout=PIPE, stderr=PIPE)
out2 = proc2.stdout
err2 = proc2.stderr
self.assertIdentical(out, out2)
def focus(e):
assert 'search starts here:' in e, e
assert e.count('End of search list.') == 1, e
return e[e.index('search starts here:'):e.index('End of search list.') + 20]
err = focus(err)
err2 = focus(err2)
assert err == err2, err + '\n\n\n\n' + err2
def handle_static_lib(self, f):
temp_dir = tempfile.mkdtemp('_archive_contents', 'emscripten_temp_')
with chdir(temp_dir):
contents = [x for x in run_process([CHEERP_BIN + 'llvm-ar', 't', f], stdout=PIPE).stdout.splitlines() if len(x)]
shared.warn_if_duplicate_entries(contents, f)
if len(contents) == 0:
print('Archive %s appears to be empty (recommendation: link an .so instead of .a)' % f)
return {
'returncode': 0,
'dir': temp_dir,
'files': []
}
# We are about to ask llvm-ar to extract all the files in the .a archive file, but
# it will silently fail if the directory for the file does not exist, so make all the necessary directories
for content in contents:
dirname = os.path.dirname(content)
if dirname:
shared.safe_ensure_dirs(dirname)
proc = run_process([CHEERP_BIN + 'llvm-ar', 'xo', f], stdout=PIPE, stderr=PIPE)
# if absolute paths, files will appear there. otherwise, in this directory
contents = list(map(os.path.abspath, contents))
nonexisting_contents = [x for x in contents if not os.path.exists(x)]
if len(nonexisting_contents) != 0:
raise Exception('llvm-ar failed to extract file(s) ' + str(nonexisting_contents) + ' from archive file ' + f + '! Error:' + str(proc.stdout) + str(proc.stderr))
return {
'returncode': proc.returncode,
'dir': temp_dir,
'files': contents
}
return {
'returncode': 1,
'dir': None,
'files': []
}
def process_shell(js_engine, shell, equivalentfn_hash_info=None):
suffix = '.eliminatedupes'
with temp_files.get_file(suffix + '.js') as temp_file:
f = open(temp_file, 'w')
f.write(shell)
f.write('\n')
f.write(equivalentfn_hash_info)
f.close()
proc = shared.run_process(js_engine +
[DUPLICATE_FUNCTION_ELIMINATOR, temp_file, '--use-hash-info', '--no-minimize-whitespace'],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
assert len(proc.stdout) > 0
assert len(proc.stderr) == 0
return proc.stdout
def minify_shell(self, shell, minify_whitespace, source_map=False):
# Run through js-optimizer.js to find and minify the global symbols
# We send it the globals, which it parses at the proper time. JS decides how
# to minify all global names, we receive a dictionary back, which is then
# used by the function processors
shell = shell.replace('0.0', '13371337') # avoid uglify doing 0.0 => 0
# Find all globals in the JS functions code
if not self.profiling_funcs:
self.globs = [m.group(1) for m in func_sig.finditer(self.js)]
if len(self.globs) == 0:
self.globs = [m.group(1) for m in func_sig_json.finditer(self.js)]
else:
self.globs = []
with temp_files.get_file('.minifyglobals.js') as temp_file:
f = open(temp_file, 'w')
f.write(shell)
f.write('\n')
f.write('// EXTRA_INFO:' + json.dumps(self.serialize()))
f.close()
output = shared.run_process(self.js_engine +
[JS_OPTIMIZER, temp_file, 'minifyGlobals', 'noPrintMetadata'] +
(['minifyWhitespace'] if minify_whitespace else []) +
(['--debug'] if source_map else []),
stdout=subprocess.PIPE).stdout
assert len(output) and not output.startswith('Assertion failed'), 'Error in js optimizer: ' + output
#print >> sys.stderr, "minified SHELL 3333333333333333", output, "\n44444444444444444444"
code, metadata = output.split('// EXTRA_INFO:')
self.globs = json.loads(metadata)
if self.symbols_file:
mapfile = open(self.symbols_file, 'w')
for key, value in self.globs.items():
mapfile.write(value + ':' + key + '\n')
mapfile.close()
print('wrote symbol map file to', self.symbols_file, file=sys.stderr)
return code.replace('13371337', '0.0')
def run_on_chunk(command):
try:
file_suffix = '.js'
index = command.index(DUPLICATE_FUNCTION_ELIMINATOR)
filename = command[index + 1]
if '--gen-hash-info' in command:
file_suffix = '.json'
if os.environ.get('EMCC_SAVE_OPT_TEMP') and os.environ.get('EMCC_SAVE_OPT_TEMP') != '0':
saved = 'save_' + os.path.basename(filename)
while os.path.exists(saved):
saved = 'input' + str(int(saved.replace('input', '').replace('.txt', '')) + 1) + '.txt'
print('running DFE command', ' '.join([c if c != filename else saved for c in command]), file=sys.stderr)
shutil.copyfile(filename, os.path.join(shared.get_emscripten_temp_dir(), saved))
if shared.EM_BUILD_VERBOSE >= 3:
print('run_on_chunk: ' + str(command), file=sys.stderr)
proc = shared.run_process(command, stdout=subprocess.PIPE)
output = proc.stdout
assert proc.returncode == 0, 'Error in optimizer (return code ' + str(proc.returncode) + '): ' + output
assert len(output) and not output.startswith('Assertion failed'), 'Error in optimizer: ' + output
filename = temp_files.get(os.path.basename(filename) + '.dfjo' + file_suffix).name
f = open(filename, 'w')
f.write(output)
f.close()
if DEBUG and not shared.WINDOWS:
print('.', file=sys.stderr) # Skip debug progress indicator on Windows, since it doesn't buffer well with multiple threads printing to console.
return filename
except KeyboardInterrupt:
# avoid throwing keyboard interrupts from a child process
raise Exception()
except (TypeError, ValueError):
formatted_lines = traceback.format_exc().splitlines()
print(">>>>>>>>>>>>>>>>>", file=sys.stderr)
for formatted_line in formatted_lines:
print(formatted_line, file=sys.stderr)
print("<<<<<<<<<<<<<<<<<", file=sys.stderr)
raise
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
self.parent = parent
if lib_builder:
native_args += lib_builder(self.name, native=True, env_init={'CC': self.cc, 'CXX': self.cxx})
if not native_exec:
compiler = self.cxx if filename.endswith('cpp') else self.cc
cmd = [
compiler,
'-fno-math-errno',
filename,
'-o', filename + '.native'
] + self.args + shared_args + native_args + shared.get_clang_native_args()
proc = run_process(cmd, stdout=PIPE, stderr=parent.stderr_redirect, env=shared.get_clang_native_env())
if proc.returncode != 0:
print("Building native executable with command failed", ' '.join(cmd), file=sys.stderr)
print("Output: " + str(proc.stdout) + '\n' + str(proc.stderr))
else:
shutil.copyfile(native_exec, filename + '.native')
shutil.copymode(native_exec, filename + '.native')
final = os.path.dirname(filename) + os.path.sep + self.name + '_' + os.path.basename(filename) + '.native'
shutil.move(filename + '.native', final)
self.filename = final
def validate_asmjs_jsfile(filename, muteOutput):
cmd = shared.SPIDERMONKEY_ENGINE + ['-c', filename]
if not shared.SPIDERMONKEY_ENGINE or cmd[0] == 'js-not-found' or len(cmd[0].strip()) == 0:
print('Could not find SpiderMonkey engine! Please set its location to SPIDERMONKEY_ENGINE in your ' + shared.hint_config_file_location() + ' configuration file!', file=sys.stderr)
return False
try:
process = shared.run_process(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
except Exception as e:
print('Executing command ' + str(cmd) + ' failed due to an exception: ' + str(e) + '!', file=sys.stderr)
return False
stdout = process.stdout
stderr = process.stderr
if not muteOutput:
if len(stdout.strip()) > 0:
print(stdout.strip())
if len(stderr.strip()) > 0:
# Pretty-print the output not to contain a spurious warning.
warning_re = re.compile(re.escape('warning: successfully compiled asm.js'), re.IGNORECASE)
stderr = warning_re.sub(' successfully compiled asm.js', stderr)
print(stderr.strip(), file=sys.stderr)
if 'successfully compiled asm.js' in stderr.lower():
return True
else:
return False
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
self.filename = filename
llvm_root = self.env.get('LLVM') or LLVM_ROOT
if lib_builder:
emcc_args = emcc_args + lib_builder('js_' + llvm_root, native=False, env_init=self.env.copy())
open('hardcode.py', 'w').write('''
def process(filename):
js = open(filename).read()
replaced = js.replace("run();", "run(%s.concat(Module[\\"arguments\\"]));")
assert js != replaced
open(filename, 'w').write(replaced)
import sys
process(sys.argv[1])
''' % str(args[:-1])) # do not hardcode in the last argument, the default arg
final = os.path.dirname(filename) + os.path.sep + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
cmd = [
PYTHON, EMCC, filename,
OPTIMIZATIONS,
'--js-transform', 'python hardcode.py',
'-s', 'TOTAL_MEMORY=256*1024*1024',
'-s', 'FILESYSTEM=0',
# '--profiling',
'--closure', '1',
'-s', 'BENCHMARK=%d' % (1 if IGNORE_COMPILATION and not has_output_parser else 0),
'-o', final
] + shared_args + emcc_args + self.extra_args
if 'FORCE_FILESYSTEM=1' in cmd:
cmd = [arg if arg != 'FILESYSTEM=0' else 'FILESYSTEM=1' for arg in cmd]
output = run_process(cmd, stdout=PIPE, stderr=PIPE, env=self.env).stdout
assert os.path.exists(final), 'Failed to compile file: ' + output + ' (looked for ' + final + ')'
if self.binaryen_opts:
run_binaryen_opts(final[:-3] + '.wasm', self.binaryen_opts)
self.filename = final
def test_emcc(self):
SANITY_FAIL_MESSAGE = 'sanity check failed to run'
# emcc should check sanity if no ${EM_CONFIG}_sanity
restore_and_set_up()
time.sleep(1)
assert not os.path.exists(SANITY_FILE) # restore is just the settings, not the sanity
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
assert os.path.exists(SANITY_FILE) # EMCC should have checked sanity successfully
assert mtime(SANITY_FILE) > mtime(CONFIG_FILE)
assert generate_sanity() == open(SANITY_FILE).read()
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc run again should not sanity check, because the sanity file is newer
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# correct sanity contents mean we need not check
open(SANITY_FILE, 'w').write(generate_sanity())
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
# incorrect sanity contents mean we *must* check
open(SANITY_FILE, 'w').write('wakawaka')
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
# but with EMCC_DEBUG=1 we should check
with env_modify({'EMCC_DEBUG': '1'}):
output = self.check_working(EMCC)
try_delete(CANONICAL_TEMP_DIR)
self.assertContained(SANITY_MESSAGE, output)
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
# also with -v, with or without inputs
output = self.check_working([EMCC, '-v'], SANITY_MESSAGE)
output = self.check_working([EMCC, '-v'] + MINIMAL_HELLO_WORLD + [], SANITY_MESSAGE)
# Make sure the test runner didn't do anything to the setup
output = self.check_working(EMCC)
self.assertNotContained(SANITY_MESSAGE, output)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc should also check sanity if the file is outdated
time.sleep(0.1)
restore_and_set_up()
assert mtime(SANITY_FILE) < mtime(CONFIG_FILE)
output = self.check_working(EMCC)
self.assertContained(SANITY_MESSAGE, output)
assert mtime(SANITY_FILE) >= mtime(CONFIG_FILE)
self.assertNotContained(SANITY_FAIL_MESSAGE, output)
# emcc should be configurable directly from EM_CONFIG without any config file
restore_and_set_up()
config = open(CONFIG_FILE, 'r').read()
open('main.cpp', 'w').write('''
#include <stdio.h>
int main() {
printf("hello from emcc with no config file\\n");
return 0;
}
''')
wipe()
with env_modify({'EM_CONFIG': config}):
run_process([PYTHON, EMCC, 'main.cpp', '-o', 'a.out.js'])
self.assertContained('hello from emcc with no config file', run_js('a.out.js'))
def test_nodejs_sockets_echo(self):
# This test checks that sockets work when the client code is run in Node.js
# Run with ./runner.py sockets.test_nodejs_sockets_echo
if NODE_JS not in JS_ENGINES:
self.skipTest('node is not present')
sockets_include = '-I' + path_from_root('tests', 'sockets')
harnesses = [
(CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=0'], 59162), 0),
(CompiledServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include, '-DTEST_DGRAM=1'], 59164), 1)
]
if not WINDOWS: # TODO: Python pickling bug causes WebsockifyServerHarness to not work on Windows.
harnesses += [(WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include], 59160), 0)]
# Basic test of node client against both a Websockified and compiled echo server.
for harness, datagram in harnesses:
with harness:
run_process([PYTHON, EMCC, path_from_root('tests', 'sockets', 'test_sockets_echo_client.c'), '-o', 'client.js', '-DSOCKK=%d' % harness.listen_port, '-DTEST_DGRAM=%d' % datagram], stdout=PIPE, stderr=PIPE)
out = run_js('client.js', engine=NODE_JS, full_output=True)
self.assertContained('do_msg_read: read 14 bytes', out)
if not WINDOWS: # TODO: Python pickling bug causes WebsockifyServerHarness to not work on Windows.
# Test against a Websockified server with compile time configured WebSocket subprotocol. We use a Websockified
# server because as long as the subprotocol list contains binary it will configure itself to accept binary
# This test also checks that the connect url contains the correct subprotocols.
print("\nTesting compile time WebSocket configuration.\n")
for harness in [
WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include], 59166)
]:
with harness:
run_process([PYTHON, EMCC, path_from_root('tests', 'sockets', 'test_sockets_echo_client.c'), '-o', 'client.js', '-s', 'SOCKET_DEBUG=1', '-s', 'WEBSOCKET_SUBPROTOCOL="base64, binary"', '-DSOCKK=59166'], stdout=PIPE, stderr=PIPE)
out = run_js('client.js', engine=NODE_JS, full_output=True)
self.assertContained('do_msg_read: read 14 bytes', out)
self.assertContained(['connect: ws://127.0.0.1:59166, base64,binary', 'connect: ws://127.0.0.1:59166/, base64,binary'], out)
# Test against a Websockified server with runtime WebSocket configuration. We specify both url and subprotocol.
# In this test we have *deliberately* used the wrong port '-DSOCKK=12345' to configure the echo_client.c, so
# the connection would fail without us specifying a valid WebSocket URL in the configuration.
print("\nTesting runtime WebSocket configuration.\n")
for harness in [
WebsockifyServerHarness(os.path.join('sockets', 'test_sockets_echo_server.c'), [sockets_include], 59168)
]:
with harness:
open(os.path.join(self.get_dir(), 'websocket_pre.js'), 'w').write('''
var Module = {
websocket: {
url: 'ws://localhost:59168/testA/testB',
subprotocol: 'text, base64, binary',
}
};
''')
run_process([PYTHON, EMCC, path_from_root('tests', 'sockets', 'test_sockets_echo_client.c'), '-o', 'client.js', '--pre-js', 'websocket_pre.js', '-s', 'SOCKET_DEBUG=1', '-DSOCKK=12345'], stdout=PIPE, stderr=PIPE)
out = run_js('client.js', engine=NODE_JS, full_output=True)
self.assertContained('do_msg_read: read 14 bytes', out)
self.assertContained('connect: ws://localhost:59168/testA/testB, text,base64,binary', out)
def run(self, args): return run_process([self.filename] + args, stdout=PIPE, stderr=PIPE, check=False).stdout
def run_binaryen_opts(filename, opts):
run_process([
os.path.join(Building.get_binaryen_bin(), 'wasm-opt'),
filename,
'-o', filename
] + opts)
def test_embuilder(self):
restore_and_set_up()
tests = [
([PYTHON, EMBUILDER], ['Emscripten System Builder Tool', 'build libc', 'native_optimizer'], True, []),
([PYTHON, EMBUILDER, 'build', 'waka'], 'ERROR', False, []),
([PYTHON, EMBUILDER, 'build', 'struct_info'], ['building and verifying struct_info', 'success'], True, ['generated_struct_info.json']),
([PYTHON, EMBUILDER, 'build', 'libc'], ['building and verifying libc', 'success'], True, ['libc.bc']),
([PYTHON, EMBUILDER, 'build', 'libc-mt'], ['building and verifying libc-mt', 'success'], True, ['libc-mt.bc']),
([PYTHON, EMBUILDER, 'build', 'libc-extras'], ['building and verifying libc-extras', 'success'], True, ['libc-extras.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc'], ['building and verifying dlmalloc', 'success'], True, ['dlmalloc.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc_debug'], ['building and verifying dlmalloc', 'success'], True, ['dlmalloc_debug.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc_threadsafe'], ['building and verifying dlmalloc_threadsafe', 'success'], True, ['dlmalloc_threadsafe.bc']),
([PYTHON, EMBUILDER, 'build', 'dlmalloc_threadsafe_debug'], ['building and verifying dlmalloc', 'success'], True, ['dlmalloc_threadsafe_debug.bc']),
([PYTHON, EMBUILDER, 'build', 'emmalloc'], ['building and verifying emmalloc', 'success'], True, ['emmalloc.bc']),
([PYTHON, EMBUILDER, 'build', 'emmalloc_debug'], ['building and verifying emmalloc', 'success'], True, ['emmalloc_debug.bc']),
([PYTHON, EMBUILDER, 'build', 'pthreads'], ['building and verifying pthreads', 'success'], True, ['pthreads.bc']),
([PYTHON, EMBUILDER, 'build', 'libcxx'], ['success'], True, ['libcxx.a']),
([PYTHON, EMBUILDER, 'build', 'libcxx_noexcept'], ['success'], True, ['libcxx_noexcept.a']),
([PYTHON, EMBUILDER, 'build', 'libcxxabi'], ['success'], True, ['libcxxabi.bc']),
([PYTHON, EMBUILDER, 'build', 'gl'], ['success'], True, ['gl.bc']),
([PYTHON, EMBUILDER, 'build', 'native_optimizer'], ['success'], True, ['optimizer.2.exe']),
([PYTHON, EMBUILDER, 'build', 'zlib'], ['building and verifying zlib', 'success'], True, ['zlib.bc']),
([PYTHON, EMBUILDER, 'build', 'libpng'], ['building and verifying libpng', 'success'], True, ['libpng.bc']),
([PYTHON, EMBUILDER, 'build', 'bullet'], ['building and verifying bullet', 'success'], True, ['bullet.bc']),
([PYTHON, EMBUILDER, 'build', 'vorbis'], ['building and verifying vorbis', 'success'], True, ['vorbis.bc']),
([PYTHON, EMBUILDER, 'build', 'ogg'], ['building and verifying ogg', 'success'], True, ['ogg.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2'], ['success'], True, ['sdl2.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-gfx'], ['success'], True, ['sdl2-gfx.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-image'], ['success'], True, ['sdl2-image.bc']),
([PYTHON, EMBUILDER, 'build', 'freetype'], ['building and verifying freetype', 'success'], True, ['freetype.bc']),
([PYTHON, EMBUILDER, 'build', 'harfbuzz'], ['building and verifying harfbuzz', 'success'], True, ['harfbuzz.bc']),
([PYTHON, EMBUILDER, 'build', 'icu'], ['building and verifying icu', 'success'], True, ['icu.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-ttf'], ['building and verifying sdl2-ttf', 'success'], True, ['sdl2-ttf.bc']),
([PYTHON, EMBUILDER, 'build', 'sdl2-net'], ['building and verifying sdl2-net', 'success'], True, ['sdl2-net.bc']),
([PYTHON, EMBUILDER, 'build', 'binaryen'], ['building and verifying binaryen', 'success'], True, []),
([PYTHON, EMBUILDER, 'build', 'cocos2d'], ['building and verifying cocos2d', 'success'], True, ['libCocos2d.bc']),
([PYTHON, EMBUILDER, 'build', 'wasm-libc'], ['building and verifying wasm-libc', 'success'], True, ['wasm-libc.bc']),
]
if Settings.WASM_BACKEND:
tests.append(([PYTHON, EMBUILDER, 'build', 'wasm_compiler_rt'], ['building and verifying wasm_compiler_rt', 'success'], True, ['wasm_compiler_rt.a']),)
Cache.erase()
for command, expected, success, result_libs in tests:
print(command)
for lib in result_libs:
if os.path.sep in lib:
dirname = os.path.dirname(Cache.get_path(lib))
print(' erase dir', dirname)
try_delete(dirname)
else:
print(' erase file', lib)
try_delete(Cache.get_path(lib))
proc = run_process(command, stdout=PIPE, stderr=STDOUT, check=False)
out = proc.stdout
if success and proc.returncode != 0:
print(out)
assert (proc.returncode == 0) == success
if not isinstance(expected, list):
expected = [expected]
for ex in expected:
print(' seek', ex)
assert ex in out, out
for lib in result_libs:
print(' verify', lib)
assert os.path.exists(Cache.get_path(lib))
def test_binaryen(self):
import tools.ports.binaryen as binaryen
tag_file = Cache.get_path('binaryen_tag_' + binaryen.TAG + '.txt')
assert not os.environ.get('BINARYEN') # must not have binaryen env var set
# test in 2 modes - with BINARYEN_ROOT in the config file, set to '', and without it entirely
for binaryen_root_in_config in [1, 0]:
print('binaryen_root_in_config:', binaryen_root_in_config)
def prep():
restore_and_set_up()
print('clearing ports...')
print(self.do([PYTHON, EMCC, '--clear-ports']))
wipe()
self.do([PYTHON, EMCC]) # first run stage
try_delete(tag_file)
# if BINARYEN_ROOT is set, we don't build the port. Check we do build it if not
if binaryen_root_in_config:
config = open(CONFIG_FILE).read()
assert '''BINARYEN_ROOT = os.path.expanduser(os.getenv('BINARYEN', ''))''' in config, config # setup created it to be ''
print('created config:')
print(config)
restore_and_set_up()
config = open(CONFIG_FILE).read()
config = config.replace('BINARYEN_ROOT', '''BINARYEN_ROOT = os.path.expanduser(os.getenv('BINARYEN', '')) # ''')
else:
restore_and_set_up()
config = open(CONFIG_FILE).read()
config = config.replace('BINARYEN_ROOT', '#')
print('modified config:')
print(config)
open(CONFIG_FILE, 'w').write(config)
print('build using embuilder')
prep()
run_process([PYTHON, EMBUILDER, 'build', 'binaryen'])
assert os.path.exists(tag_file)
run_process([PYTHON, EMCC] + MINIMAL_HELLO_WORLD + ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'])
self.assertContained('hello, world!', run_js('a.out.js'))
print('see we show an error for emmake (we cannot build natively under emmake)')
prep()
try_delete('a.out.js')
out = self.do([PYTHON, path_from_root('emmake.py'), EMCC] + MINIMAL_HELLO_WORLD + ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"'])
assert not os.path.exists(tag_file)
assert not os.path.exists('a.out.js')
self.assertContained('For example, for binaryen, do "python embuilder.py build binaryen"', out)
if not binaryen_root_in_config:
print('build on demand')
for side_module in (False, True):
print(side_module)
prep()
assert not os.path.exists(tag_file)
try_delete('a.out.js')
try_delete('a.out.wasm')
cmd = [PYTHON, EMCC]
if not side_module:
cmd += MINIMAL_HELLO_WORLD
else:
# EM_ASM doesn't work in a wasm side module, build a normal program
cmd += [path_from_root('tests', 'hello_world.c'), '-s', 'SIDE_MODULE=1']
cmd += ['-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="interpret-binary"']
run_process(cmd)
assert os.path.exists(tag_file)
assert os.path.exists('a.out.wasm')
if not side_module:
assert os.path.exists('a.out.js')
self.assertContained('hello, world!', run_js('a.out.js'))
def build(self, parent, filename, args, shared_args, emcc_args, native_args, native_exec, lib_builder, has_output_parser):
suffix = filename.split('.')[-1]
cheerp_temp = filename + '.cheerp.' + suffix
code = open(filename).read()
if 'int main()' in code:
main_args = ''
else:
main_args = 'argc, (%(const)s char**)argv' % {
'const': 'const' if 'const char *argv' in code else ''
}
open(cheerp_temp, 'w').write('''
%(code)s
void webMain() {
// TODO: how to read from commandline?
volatile int argc = 2;
typedef char** charStarStar;
volatile charStarStar argv;
argv[0] = "./cheerp.exe";
argv[1] = "%(arg)s";
volatile int exit_code = main(%(main_args)s);
}
''' % {
'arg': args[-1],
'code': code,
'main_args': main_args
})
cheerp_args = [
'-target', 'cheerp',
'-cheerp-mode=wasm'
]
cheerp_args += self.args
self.parent = parent
if lib_builder:
# build as "native" (so no emcc env stuff), but with all the cheerp stuff
# set in the env
cheerp_args = cheerp_args + lib_builder(self.name, native=True, env_init={
'CC': CHEERP_BIN + 'clang',
'CXX': CHEERP_BIN + 'clang++',
'AR': CHEERP_BIN + 'llvm-ar',
'LD': CHEERP_BIN + 'clang',
'NM': CHEERP_BIN + 'llvm-nm',
'LDSHARED': CHEERP_BIN + 'clang',
'RANLIB': CHEERP_BIN + 'llvm-ranlib',
'CFLAGS': ' '.join(cheerp_args),
'CXXFLAGS': ' '.join(cheerp_args),
})
# cheerp_args += ['-cheerp-pretty-code'] # get function names, like emcc --profiling
final = os.path.dirname(filename) + os.path.sep + 'cheerp_' + self.name + ('_' if self.name else '') + os.path.basename(filename) + '.js'
final = final.replace('.cpp', '')
try_delete(final)
dirs_to_delete = []
try:
for arg in cheerp_args[:]:
if arg.endswith('.a'):
info = self.handle_static_lib(arg)
cheerp_args += info['files']
dirs_to_delete += [info['dir']]
cheerp_args = [arg for arg in cheerp_args if not arg.endswith('.a')]
# print(cheerp_args)
cmd = [CHEERP_BIN + 'clang++'] + cheerp_args + [
'-cheerp-linear-heap-size=256',
'-cheerp-wasm-loader=' + final,
cheerp_temp,
'-Wno-writable-strings', # for how we set up webMain
'-o', final + '.wasm'
] + shared_args
# print(' '.join(cmd))
run_process(cmd)
self.filename = final
Building.get_binaryen()
if self.binaryen_opts:
run_binaryen_opts(final + '.wasm', self.binaryen_opts)
finally:
for dir_ in dirs_to_delete:
try_delete(dir_)
def test_webrtc(self): # XXX see src/settings.js, this is disabled pending investigation
self.skipTest('WebRTC support is not up to date.')
host_src = 'webrtc_host.c'
peer_src = 'webrtc_peer.c'
host_outfile = 'host.html'
peer_outfile = 'peer.html'
host_filepath = path_from_root('tests', 'sockets', host_src)
temp_host_filepath = os.path.join(self.get_dir(), os.path.basename(host_src))
with open(host_filepath) as f:
host_src = f.read()
with open(temp_host_filepath, 'w') as f:
f.write(self.with_report_result(host_src))
peer_filepath = path_from_root('tests', 'sockets', peer_src)
temp_peer_filepath = os.path.join(self.get_dir(), os.path.basename(peer_src))
with open(peer_filepath) as f:
peer_src = f.read()
with open(temp_peer_filepath, 'w') as f:
f.write(self.with_report_result(peer_src))
open(os.path.join(self.get_dir(), 'host_pre.js'), 'w').write('''
var Module = {
webrtc: {
broker: 'http://localhost:8182',
session: undefined,
onpeer: function(peer, route) {
window.open('http://localhost:8888/peer.html?' + route);
// iframe = document.createElement("IFRAME");
// iframe.setAttribute("src", "http://localhost:8888/peer.html?" + route);
// iframe.style.display = "none";
// document.body.appendChild(iframe);
peer.listen();
},
onconnect: function(peer) {
},
ondisconnect: function(peer) {
},
onerror: function(error) {
console.error(error);
}
},
setStatus: function(text) {
console.log('status: ' + text);
}
};
''')
open(os.path.join(self.get_dir(), 'peer_pre.js'), 'w').write('''
var Module = {
webrtc: {
broker: 'http://localhost:8182',
session: window.location.toString().split('?')[1],
onpeer: function(peer, route) {
peer.connect(Module['webrtc']['session']);
},
onconnect: function(peer) {
},
ondisconnect: function(peer) {
// Calling window.close() from this handler hangs my browser, so run it in the next turn
setTimeout(window.close, 0);
},
onerror: function(error) {
console.error(error);
},
},
setStatus: function(text) {
console.log('status: ' + text);
}
};
''')
run_process([PYTHON, EMCC, temp_host_filepath, '-o', host_outfile] + ['-s', 'GL_TESTING=1', '--pre-js', 'host_pre.js', '-s', 'SOCKET_WEBRTC=1', '-s', 'SOCKET_DEBUG=1'])
run_process([PYTHON, EMCC, temp_peer_filepath, '-o', peer_outfile] + ['-s', 'GL_TESTING=1', '--pre-js', 'peer_pre.js', '-s', 'SOCKET_WEBRTC=1', '-s', 'SOCKET_DEBUG=1'])
# note: you may need to run this manually yourself, if npm is not in the path, or if you need a version that is not in the path
run_process([NPM, 'install', path_from_root('tests', 'sockets', 'p2p')])
broker = Popen(NODE_JS + [path_from_root('tests', 'sockets', 'p2p', 'broker', 'p2p-broker.js')])
expected = '1'
self.run_browser(host_outfile, '.', ['/report_result?' + e for e in expected])
broker.kill()
