def build_libc(lib_filename, files):
o_s = []
prev_cxx = os.environ.get("EMMAKEN_CXX")
if prev_cxx:
os.environ["EMMAKEN_CXX"] = ""
musl_internal_includes = shared.path_from_root("system", "lib", "libc", "musl", "src", "internal")
for src in files:
o = in_temp(os.path.basename(src) + ".o")
execute(
[
shared.PYTHON,
shared.EMCC,
shared.path_from_root("system", "lib", src),
"-o",
o,
"-I",
musl_internal_includes,
]
+ lib_opts,
stdout=stdout,
stderr=stderr,
)
o_s.append(o)
if prev_cxx:
os.environ["EMMAKEN_CXX"] = prev_cxx
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def create_dlmalloc(out_name):
o = in_temp(out_name)
cflags = ['-O2']
if shared.Settings.USE_PTHREADS:
cflags += ['-s', 'USE_PTHREADS=1']
if shared.Settings.EMSCRIPTEN_TRACING:
cflags += ['--tracing']
if shared.Settings.SPLIT_MEMORY:
cflags += ['-DMSPACES', '-DONLY_MSPACES']
if shared.Settings.DEBUG_LEVEL:
cflags += ['-DDLMALLOC_DEBUG']
check_call([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o
] + cflags)
if shared.Settings.SPLIT_MEMORY:
split_malloc_o = in_temp('sm' + out_name)
check_call([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', 'split_malloc.cpp'),
'-o', split_malloc_o, '-O2'
])
lib = in_temp('lib' + out_name)
shared.Building.link([o, split_malloc_o], lib)
shutil.move(lib, o)
return o
def create_wasm_compiler_rt(libname):
srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins')
filenames = ['addtf3.c', 'ashlti3.c', 'ashrti3.c', 'comparetf2.c', 'divtf3.c', 'divti3.c', 'udivmodti4.c',
'extenddftf2.c', 'extendsftf2.c',
'fixdfti.c', 'fixsfti.c', 'fixtfdi.c', 'fixtfsi.c', 'fixtfti.c',
'fixunsdfti.c', 'fixunssfti.c', 'fixunstfdi.c', 'fixunstfsi.c', 'fixunstfti.c',
'floatditf.c', 'floatsitf.c', 'floattidf.c', 'floattisf.c',
'floatunditf.c', 'floatunsitf.c', 'floatuntidf.c', 'floatuntisf.c', 'lshrti3.c',
'modti3.c', 'multf3.c', 'multi3.c', 'subtf3.c', 'udivti3.c', 'umodti3.c', 'ashrdi3.c',
'ashldi3.c', 'fixdfdi.c', 'floatdidf.c', 'lshrdi3.c', 'moddi3.c',
'trunctfdf2.c', 'trunctfsf2.c', 'umoddi3.c', 'fixunsdfdi.c', 'muldi3.c',
'divdi3.c', 'divmoddi4.c', 'udivdi3.c', 'udivmoddi4.c']
files = (os.path.join(srcdir, f) for f in filenames)
o_s = []
commands = []
for src in files:
o = in_temp(os.path.basename(src) + '.o')
# Use clang directly instead of emcc. Since emcc's intermediate format (produced by -S) is LLVM IR, there's no way to
# get emcc to output wasm .s files, which is what we archive in compiler_rt.
commands.append([shared.CLANG_CC, '--target=wasm32', '-S', shared.path_from_root('system', 'lib', src), '-O2', '-o', o] + shared.EMSDK_OPTS)
o_s.append(o)
run_commands(commands)
lib = in_temp(libname)
run_commands([[shared.LLVM_AR, 'cr', '-format=gnu', lib] + o_s])
return lib
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [
shared.CLANG, 'g++', 'clang++'
]: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
out, err = subprocess.Popen(
[
compiler,
shared.path_from_root('tools', 'optimizer',
'parser.cpp'),
shared.path_from_root('tools', 'optimizer',
'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-shared.cpp'),
shared.path_from_root('tools', 'optimizer',
'optimizer-main.cpp'),
'-O3', '-std=c++11', '-fno-exceptions',
'-fno-rtti', '-o', output
] + args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
outs.append(out)
errs.append(err)
except OSError:
if compiler == shared.CLANG:
raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
if os.path.exists(output): return output
raise NativeOptimizerCreationException()
def build_libc(lib_filename, files, lib_opts):
o_s = []
musl_internal_includes = [
'-I',
shared.path_from_root('system', 'lib', 'libc', 'musl', 'src',
'internal'), '-I',
shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch',
'js')
]
commands = []
# Hide several musl warnings that produce a lot of spam to unit test build server logs.
# TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up.
c_opts = [
'-Wno-dangling-else', '-Wno-unknown-pragmas',
'-Wno-shift-op-parentheses', '-Wno-string-plus-int',
'-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses',
'-Wno-visibility'
]
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', src), '-o', o
] + musl_internal_includes + default_opts + c_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def create_wasm_libc(libname):
# in asm.js we just use Math.sin etc., which is good for code size. But wasm doesn't have such builtins, so
# we need to bundle in more code
# we also build in musl versions of things that we have hand-optimized asm.js for
files = ([shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'math', x) for x in ('cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'pow.c', 'powf.c', 'powl.c')] +
[shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'string', x) for x in ('memcpy.c', 'memset.c', 'memmove.c')])
return build_libc(libname, files, ['-O2'])
def create_dlmalloc_split(libname):
dlmalloc_o = in_temp('dl' + libname)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', dlmalloc_o, '-O2', '-DMSPACES', '-DONLY_MSPACES'])
split_malloc_o = in_temp('sm' + libname)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2'])
lib = in_temp(libname)
shared.Building.link([dlmalloc_o, split_malloc_o], lib)
return lib
def create_dlmalloc_split(libname):
dlmalloc_o = in_temp('dl' + libname)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', dlmalloc_o, '-O2', '-DMSPACES', '-DONLY_MSPACES'])
split_malloc_o = in_temp('sm' + libname)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2'])
lib = in_temp(libname)
shared.Building.link([dlmalloc_o, split_malloc_o], lib)
return lib
def musl_internal_includes():
return [
'-I',
shared.path_from_root('system', 'lib', 'libc', 'musl', 'src',
'internal'),
'-I',
shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch',
'js'),
]
def create_load_wasm_worker():
emscripten.logging.debug('building load-wasm-worker')
output = emscripten.Cache.get_path('load-wasm-worker.js')
emscripten.try_delete(output)
check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'),
emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'),
'-o', output] + \
'-O3 -std=c++11 --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0'.split(' '))
assert os.path.exists(output)
open(output, 'a').write(open(emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'load-wasm-worker.js')).read())
return output
def build_libc(lib_filename, files):
o_s = []
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')]
for src in files:
o = in_temp(os.path.basename(src) + '.o')
execute([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + lib_opts, stdout=stdout, stderr=stderr)
o_s.append(o)
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def create_pack_asmjs():
emscripten.logging.debug('building pack-asmjs')
output = emscripten.Cache.get_path('pack-asmjs.js')
emscripten.try_delete(output)
check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.cpp'),
emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'),
emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'),
'-o', output] + \
'-O3 -std=c++11 -DCHECKED_OUTPUT_SIZE --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0 -s INVOKE_RUN=0'.split(' ') + \
['-I' + emscripten.path_from_root('tools', 'optimizer')])
assert os.path.exists(output)
open(output, 'a').write(open(emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.js')).read())
return output
def create_compiler_rt(libname):
srcdir = shared.path_from_root('system', 'lib', 'compiler-rt')
filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c']
files = (os.path.join(srcdir, f) for f in filenames)
o_s = []
commands = []
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o])
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(libname))
return in_temp(libname)
def create_load_wasm_worker():
emscripten.logging.debug('building load-wasm-worker')
output = emscripten.Cache.get_path('load-wasm-worker.js')
emscripten.try_delete(output)
check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'),
emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'),
'-o', output] + \
'-O3 -std=c++11 --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0'.split(' '))
assert os.path.exists(output)
open(output, 'a').write(
open(
emscripten.path_from_root('third_party', 'wasm-polyfill', 'src',
'load-wasm-worker.js')).read())
return output
def build_libc(lib_filename, files, lib_opts):
o_s = []
musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')]
commands = []
# Hide several musl warnings that produce a lot of spam to unit test build server logs.
# TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up.
c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility']
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def create_compiler_rt(libname):
srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins')
filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c']
files = (os.path.join(srcdir, f) for f in filenames)
o_s = []
commands = []
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o])
o_s.append(o)
run_commands(commands)
shared.Building.emar('cr', in_temp(libname), o_s)
return in_temp(libname)
def create_gl():
o = in_temp('gl.o')
check_call([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', 'gl.c'), '-o', o
])
return o
def build_libc(lib_filename, files, lib_opts):
o_s = []
commands = []
# Hide several musl warnings that produce a lot of spam to unit test build server logs.
# TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up.
c_opts = [
'-Wno-return-type', '-Wno-parentheses', '-Wno-ignored-attributes',
'-Wno-shift-count-overflow', '-Wno-shift-negative-value',
'-Wno-dangling-else', '-Wno-unknown-pragmas',
'-Wno-shift-op-parentheses', '-Wno-string-plus-int',
'-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses',
'-Wno-visibility', '-Wno-pointer-sign'
]
if shared.Settings.WASM_BACKEND:
c_opts.append('-Wno-error=absolute-value')
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', src), '-o', o
] + musl_internal_includes() + default_opts + c_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def create_gl():
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
o = in_temp('gl.o')
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o])
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
return o
def create_optimizer():
shared.logging.debug('building native optimizer: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
try:
subprocess.Popen([compiler,
shared.path_from_root('tools', 'optimizer', 'parser.cpp'),
shared.path_from_root('tools', 'optimizer', 'simple_ast.cpp'),
shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'),
'-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output] + args).communicate()
except OSError:
if compiler == shared.CLANG: raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
if os.path.exists(output): return output
raise NativeOptimizerCreationException()
def create_dlmalloc(out_name, clflags):
o = in_temp(out_name)
check_call([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o
] + clflags)
return o
def create_libc(libname):
logging.debug(' building libc for cache')
libc_files = [
]
musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src')
blacklist = set(
['ipc', 'passwd', 'thread', 'signal', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'process', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules
['memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c'] + # individual files
['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files
)
# TODO: consider using more math code from musl, doing so makes box2d faster
for dirpath, dirnames, filenames in os.walk(musl_srcdir):
for f in filenames:
if f.endswith('.c'):
if f in blacklist: continue
dir_parts = os.path.split(dirpath)
cancel = False
for part in dir_parts:
if part in blacklist:
cancel = True
break
if not cancel:
libc_files.append(os.path.join(musl_srcdir, dirpath, f))
args = ['-Os']
if shared.Settings.USE_PTHREADS:
args += ['-s', 'USE_PTHREADS=1']
assert '-mt' in libname
else:
assert '-mt' not in libname
return build_libc(libname, libc_files, args)
def build_libcxx(src_dirname,
lib_filename,
files,
lib_opts,
has_noexcept_version=False):
o_s = []
commands = []
opts = default_opts + lib_opts
if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING:
opts += ['-fno-exceptions']
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
commands.append(
[shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] +
opts)
o_s.append(o)
run_commands(commands)
if lib_filename.endswith('.bc'):
shared.Building.link(o_s, in_temp(lib_filename))
elif lib_filename.endswith('.a'):
shared.Building.emar('cr', in_temp(lib_filename), o_s)
else:
raise Exception('unknown suffix ' + lib_filename)
return in_temp(lib_filename)
def create_gl(libname): # libname is ignored, this is just one .o file
o = in_temp('gl.o')
check_call([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', 'gl.c'), '-o', o
])
return o
def create_libc(libname):
logging.debug(' building libc for cache')
libc_files = [
]
musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src')
blacklist = set(
['ipc', 'passwd', 'thread', 'signal', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'process', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules
['memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c'] + # individual files
['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files
)
# TODO: consider using more math code from musl, doing so makes box2d faster
for dirpath, dirnames, filenames in os.walk(musl_srcdir):
for f in filenames:
if f.endswith('.c'):
if f in blacklist: continue
dir_parts = os.path.split(dirpath)
cancel = False
for part in dir_parts:
if part in blacklist:
cancel = True
break
if not cancel:
libc_files.append(os.path.join(musl_srcdir, dirpath, f))
args = ['-Os']
if shared.Settings.USE_PTHREADS:
args += ['-s', 'USE_PTHREADS=1']
assert '-mt' in libname
else:
assert '-mt' not in libname
return build_libc(libname, libc_files, args)
def create_pack_asmjs():
emscripten.logging.debug('building pack-asmjs')
output = emscripten.Cache.get_path('pack-asmjs.js')
emscripten.try_delete(output)
check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.cpp'),
emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'),
emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'),
'-o', output] + \
'-O3 -std=c++11 -DCHECKED_OUTPUT_SIZE --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0 -s INVOKE_RUN=0'.split(' ') + \
['-I' + emscripten.path_from_root('tools', 'optimizer')])
assert os.path.exists(output)
open(output, 'a').write(
open(
emscripten.path_from_root('third_party', 'wasm-polyfill', 'src',
'pack-asmjs.js')).read())
return output
def create_gl():
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
o = in_temp('gl.o')
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o])
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
return o
def create_gl():
prev_cxx = os.environ.get("EMMAKEN_CXX")
if prev_cxx:
os.environ["EMMAKEN_CXX"] = ""
o = in_temp("gl.o")
execute([shared.PYTHON, shared.EMCC, shared.path_from_root("system", "lib", "gl.c"), "-o", o])
if prev_cxx:
os.environ["EMMAKEN_CXX"] = prev_cxx
return o
def build_libcxx(src_dirname, lib_filename, files):
o_s = []
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
execute([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + lib_opts, stdout=stdout, stderr=stderr)
o_s.append(o)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def build_libcxx(src_dirname, lib_filename, files, lib_opts):
o_s = []
commands = []
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + default_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def build_libc(lib_filename, files):
o_s = []
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
musl_internal_includes = shared.path_from_root('system', 'lib', 'libc',
'musl', 'src',
'internal')
for src in files:
o = in_temp(os.path.basename(src) + '.o')
execute([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', src), '-o', o, '-I',
musl_internal_includes
] + lib_opts,
stdout=stdout,
stderr=stderr)
o_s.append(o)
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def build_libcxx(src_dirname, lib_filename, files, lib_opts):
o_s = []
commands = []
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + default_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def apply_libc(need):
# libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
try:
if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
shared.Settings.CORRECT_SIGNS = 2
if shared.Settings.CORRECT_SIGNS == 2:
shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
# If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
# so all is well anyhow too.
return True
def apply_libc(need):
# libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
try:
if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
shared.Settings.CORRECT_SIGNS = 2
if shared.Settings.CORRECT_SIGNS == 2:
shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
# If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
# so all is well anyhow too.
return True
def create_optimizer():
shared.logging.debug('building native optimizer')
output = shared.Cache.get_path('optimizer.exe')
shared.try_delete(output)
errs = []
for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(' using ' + compiler)
out, err = subprocess.Popen([compiler, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output], stderr=subprocess.PIPE).communicate()
# for profiling/debugging: '-g', '-fno-omit-frame-pointer'
if os.path.exists(output): return output
errs.append(err)
raise Exception('failed to build native optimizer, errors from each attempt: ' + '\n=================\n'.join(errs))
def build_libcxx(src_dirname, lib_filename, files):
o_s = []
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
execute(
[shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] +
lib_opts,
stdout=stdout,
stderr=stderr)
o_s.append(o)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def build_libc(lib_filename, files):
o_s = []
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
musl_internal_includes = [
'-I',
shared.path_from_root('system', 'lib', 'libc', 'musl', 'src',
'internal'), '-I',
shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch',
'js')
]
commands = []
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([
shared.PYTHON, shared.EMCC,
shared.path_from_root('system', 'lib', src), '-o', o
] + musl_internal_includes + lib_opts)
o_s.append(o)
run_commands(commands)
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def create_libcxxabi(libname):
logging.debug('building libcxxabi for cache')
libcxxabi_files = [
'abort_message.cpp', 'cxa_aux_runtime.cpp',
'cxa_default_handlers.cpp', 'cxa_demangle.cpp',
'cxa_exception_storage.cpp', 'cxa_guard.cpp', 'cxa_new_delete.cpp',
'cxa_handlers.cpp', 'exception.cpp', 'stdexcept.cpp',
'typeinfo.cpp', 'private_typeinfo.cpp'
]
return build_libcxx(os.path.join(
'system', 'lib', 'libcxxabi', 'src'), libname, libcxxabi_files, [
'-Oz', '-I' +
shared.path_from_root('system', 'lib', 'libcxxabi', 'include')
])
def create_optimizer():
shared.logging.debug("building native optimizer: " + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
for compiler in [
shared.CLANG,
"g++",
"clang++",
]: # try our clang first, otherwise hope for a system compiler in the path
shared.logging.debug(" using " + compiler)
try:
out, err = subprocess.Popen(
[
compiler,
shared.path_from_root("tools", "optimizer", "parser.cpp"),
shared.path_from_root("tools", "optimizer", "simple_ast.cpp"),
shared.path_from_root("tools", "optimizer", "optimizer.cpp"),
shared.path_from_root("tools", "optimizer", "optimizer-main.cpp"),
"-O3",
"-std=c++11",
"-fno-exceptions",
"-fno-rtti",
"-o",
output,
]
+ args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
).communicate()
outs.append(out)
errs.append(err)
except OSError:
if compiler == shared.CLANG:
raise # otherwise, OSError is likely due to g++ or clang++ not being in the path
if os.path.exists(output):
return output
raise NativeOptimizerCreationException()
def create_libcxx():
logging.debug('building libcxx for cache')
libcxx_files = [
'algorithm.cpp', 'condition_variable.cpp', 'future.cpp',
'iostream.cpp', 'memory.cpp', 'random.cpp', 'stdexcept.cpp',
'system_error.cpp', 'utility.cpp', 'bind.cpp', 'debug.cpp',
'hash.cpp', 'mutex.cpp', 'string.cpp', 'thread.cpp',
'valarray.cpp', 'chrono.cpp', 'exception.cpp', 'ios.cpp',
'locale.cpp', 'regex.cpp', 'strstream.cpp'
]
return build_libcxx(os.path.join(
'system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files, [
'-Oz', '-Wno-warn-absolute-paths', '-I' +
shared.path_from_root('system', 'lib', 'libcxxabi', 'include')
])
def create_pthreads(libname):
# Add pthread files.
pthreads_files = [os.path.join('pthread', 'library_pthread.c')]
pthreads_files += [
shared.path_from_root('system', 'lib', 'libc', 'musl', 'src',
'thread', x) for x in
('pthread_attr_destroy.c', 'pthread_condattr_setpshared.c',
'pthread_mutex_lock.c', 'pthread_spin_destroy.c',
'pthread_attr_get.c', 'pthread_cond_broadcast.c',
'pthread_mutex_setprioceiling.c', 'pthread_spin_init.c',
'pthread_attr_init.c', 'pthread_cond_destroy.c',
'pthread_mutex_timedlock.c', 'pthread_spin_lock.c',
'pthread_attr_setdetachstate.c', 'pthread_cond_init.c',
'pthread_mutex_trylock.c', 'pthread_spin_trylock.c',
'pthread_attr_setguardsize.c', 'pthread_cond_signal.c',
'pthread_mutex_unlock.c', 'pthread_spin_unlock.c',
'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c',
'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c',
'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c',
'sem_getvalue.c', 'pthread_attr_setschedpolicy.c',
'pthread_equal.c', 'pthread_rwlockattr_init.c', 'sem_init.c',
'pthread_attr_setscope.c', 'pthread_getspecific.c',
'pthread_rwlockattr_setpshared.c', 'sem_open.c',
'pthread_attr_setstack.c', 'pthread_key_create.c',
'pthread_rwlock_destroy.c', 'sem_post.c',
'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c',
'pthread_rwlock_init.c', 'sem_timedwait.c',
'pthread_barrierattr_destroy.c', 'pthread_mutexattr_init.c',
'pthread_rwlock_rdlock.c', 'sem_trywait.c',
'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c',
'pthread_rwlock_timedrdlock.c', 'sem_unlink.c',
'pthread_barrierattr_setpshared.c',
'pthread_mutexattr_setpshared.c', 'pthread_rwlock_timedwrlock.c',
'sem_wait.c', 'pthread_barrier_destroy.c',
'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c',
'__timedwait.c', 'pthread_barrier_init.c',
'pthread_mutexattr_settype.c', 'pthread_rwlock_trywrlock.c',
'vmlock.c', 'pthread_barrier_wait.c',
'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c',
'__wait.c', 'pthread_condattr_destroy.c',
'pthread_mutex_destroy.c', 'pthread_rwlock_wrlock.c',
'pthread_condattr_init.c', 'pthread_mutex_getprioceiling.c',
'pthread_setcanceltype.c', 'pthread_condattr_setclock.c',
'pthread_mutex_init.c', 'pthread_setspecific.c')
]
return build_libc(libname, pthreads_files,
['-O2', '-s', 'USE_PTHREADS=1'])
def create_wasm_rt_lib(libname, files):
o_s = []
commands = []
for src in files:
o = in_temp(os.path.basename(src) + '.o')
# Use clang directly instead of emcc. Since emcc's intermediate format (produced by -S) is LLVM IR, there's no way to
# get emcc to output wasm .s files, which is what we archive in compiler_rt.
commands.append([
shared.CLANG_CC, '--target=wasm32', '-mthread-model', 'single',
'-S',
shared.path_from_root('system', 'lib', src), '-O2', '-o', o
] + musl_internal_includes() + shared.EMSDK_OPTS)
o_s.append(o)
run_commands(commands)
lib = in_temp(libname)
run_commands([[shared.LLVM_AR, 'cr', '-format=gnu', lib] + o_s])
return lib
def build_libcxx(src_dirname, lib_filename, files, lib_opts, has_noexcept_version=False):
o_s = []
commands = []
opts = default_opts + lib_opts
if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING:
opts += ['-fno-exceptions']
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + opts)
o_s.append(o)
run_commands(commands)
if lib_filename.endswith('.bc'):
shared.Building.link(o_s, in_temp(lib_filename))
elif lib_filename.endswith('.a'):
shared.Building.emar('cr', in_temp(lib_filename), o_s)
else:
raise Exception('unknown suffix ' + lib_filename)
return in_temp(lib_filename)
def create_libcxx(libname):
logging.debug('building libcxx for cache')
libcxx_files = [
'algorithm.cpp', 'any.cpp', 'bind.cpp', 'chrono.cpp',
'condition_variable.cpp', 'debug.cpp', 'exception.cpp',
'future.cpp', 'hash.cpp', 'ios.cpp', 'iostream.cpp', 'locale.cpp',
'memory.cpp', 'mutex.cpp', 'new.cpp', 'optional.cpp', 'random.cpp',
'regex.cpp', 'shared_mutex.cpp', 'stdexcept.cpp', 'string.cpp',
'strstream.cpp', 'system_error.cpp', 'thread.cpp', 'typeinfo.cpp',
'utility.cpp', 'valarray.cpp'
]
return build_libcxx(
os.path.join('system', 'lib', 'libcxx'),
libname,
libcxx_files, [
'-DLIBCXX_BUILDING_LIBCXXABI=1', '-Oz', '-I' +
shared.path_from_root('system', 'lib', 'libcxxabi', 'include')
],
has_noexcept_version=True)
def create_libcxxabi():
logging.debug('building libcxxabi for cache')
libcxxabi_files = [
'abort_message.cpp',
'cxa_aux_runtime.cpp',
'cxa_default_handlers.cpp',
'cxa_demangle.cpp',
'cxa_exception_storage.cpp',
'cxa_new_delete.cpp',
'cxa_handlers.cpp',
'exception.cpp',
'stdexcept.cpp',
'typeinfo.cpp',
'private_typeinfo.cpp',
os.path.join('..', '..', 'libcxx', 'new.cpp'),
]
return build_libcxx(
os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc',
libcxxabi_files, [
'-Oz', '-Wno-warn-absolute-paths', '-I' +
shared.path_from_root('system', 'lib', 'libcxxabi', 'include')
])
def create_libc(libname):
logging.warn(' building libc for cache')
libc_files = [
]
musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src')
blacklist = set(
['ipc', 'passwd',
'getopt_long.c', 'getopt.c',
'raise.c', 'sigsuspend.c', 'sigtimedwait.c', 'getitimer.c', 'sigpending.c', 'sigqueue.c', 'sigaltstack.c',
'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules
['fexecve.c', 'posix_spawnattr_destroy.c', 'posix_spawnattr_getflags.c', 'posix_spawnattr_getpgroup.c', 'posix_spawnattr_getsigdefault.c', 'posix_spawnattr_getsigmask.c', 'posix_spawnattr_init.c', 'posix_spawnattr_sched.c', 'posix_spawnattr_setflags.c', 'posix_spawnattr_setpgroup.c', 'posix_spawnattr_setsigdefault.c', 'posix_spawnattr_setsigmask.c', 'posix_spawn.c', 'posix_spawn_file_actions_addclose.c', 'posix_spawn_file_actions_adddup2.c', 'posix_spawn_file_actions_addopen.c', 'posix_spawn_file_actions_destroy.c', 'posix_spawn_file_actions_init.c', 'posix_spawnp.c', 'system.c', 'vfork.c',
'memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c', 'waitid.c'] + # individual files
['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files
)
# applied after blacklist
whitelist = set(['wait.c', 'wait3.c', 'wait4.c', 'pthread_sigmask.c'])
# TODO: consider using more math code from musl, doing so makes box2d faster
for dirpath, dirnames, filenames in os.walk(musl_srcdir):
for f in filenames:
if f.endswith('.c'):
if f in blacklist: continue
dir_parts = os.path.split(dirpath)
cancel = False
for part in dir_parts:
if part == 'thread':
cancel = True
break
if part in blacklist and (not f.endswith('pthread_atfork.c')):
cancel = True
break
if f in whitelist or not cancel:
libc_files.append(os.path.join(musl_srcdir, dirpath, f))
args = ['-Os', '-g4']
if shared.Settings.USE_PTHREADS:
args += ['-s', 'USE_PTHREADS=1']
assert '-mt' in libname
else:
assert '-mt' not in libname
return build_libc(libname, libc_files, args)
while 1:
print 'Tried %d, notes: %s' % (tried, notes)
print '1) Generate C'
shared.execute([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# +
#['--max-block-depth', '2', '--max-block-size', '2', '--max-expr-complexity', '2', '--max-funcs', '2'],
stdout=open(filename + '.c', 'w'))
#shutil.copyfile(filename + '.c', 'testcase%d.c' % tried)
print '1) Generate C... %.2f K of C source' % (len(open(filename + '.c').read())/1024.)
tried += 1
print '2) Compile natively'
shared.try_delete(filename)
shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS, stderr=PIPE) # + shared.EMSDK_OPTS
shared.execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS, stderr=PIPE)
shared.execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'], stdout=PIPE, stderr=PIPE)
shutil.move(filename + '.bc.run', filename + '2')
shared.execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS, stderr=PIPE)
print '3) Run natively'
try:
correct1 = shared.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct1 or len(correct1) < 10: raise Exception('segfault')
correct2 = shared.timeout_run(Popen([filename + '2'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct2 or len(correct2) < 10: raise Exception('segfault')
correct3 = shared.timeout_run(Popen([filename + '3'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct3 or len(correct3) < 10: raise Exception('segfault')
if correct1 != correct3: raise Exception('clang opts change result')
except Exception, e:
print 'Failed or infinite looping in native, skipping', e
notes['invalid'] += 1
continue
if seen.get(name): return True
seen[name] = 1
return False
data_files = filter(lambda file_: not was_seen(file_['dstpath']), data_files)
if AV_WORKAROUND:
random.shuffle(data_files)
# Apply plugins
for file_ in data_files:
for plugin in plugins:
plugin(file_)
# Crunch files
if crunch:
shutil.copyfile(shared.path_from_root('tools', 'crunch-worker.js'), 'crunch-worker.js')
ret += '''
var decrunchWorker = new Worker('crunch-worker.js');
var decrunchCallbacks = [];
decrunchWorker.onmessage = function(msg) {
decrunchCallbacks[msg.data.callbackID](msg.data.data);
console.log('decrunched ' + msg.data.filename + ' in ' + msg.data.time + ' ms, ' + msg.data.data.length + ' bytes');
decrunchCallbacks[msg.data.callbackID] = null;
};
function requestDecrunch(filename, data, callback) {
decrunchWorker.postMessage({
filename: filename,
data: new Uint8Array(data),
callbackID: decrunchCallbacks.length
});
decrunchCallbacks.push(callback);
def create_optimizer_cmake():
shared.logging.debug('building native optimizer via CMake: ' + name)
output = shared.Cache.get_path(name)
shared.try_delete(output)
if NATIVE_OPTIMIZER == '1':
cmake_build_type = 'RelWithDebInfo'
elif NATIVE_OPTIMIZER == '2':
cmake_build_type = 'Release'
elif NATIVE_OPTIMIZER == 'g':
cmake_build_type = 'Debug'
build_path = shared.Cache.get_path('optimizer_build_' + cmake_build_type)
shared.try_delete(os.path.join(build_path, 'CMakeCache.txt'))
log_output = None if DEBUG else subprocess.PIPE
if not os.path.exists(build_path):
os.mkdir(build_path)
if WINDOWS:
# Poor man's check for whether or not we should attempt 64 bit build
if os.environ.get('ProgramFiles(x86)'):
cmake_generators = ['Visual Studio 12 Win64', 'Visual Studio 12', 'Visual Studio 11 Win64', 'Visual Studio 11', 'MinGW Makefiles', 'Unix Makefiles']
else:
cmake_generators = ['Visual Studio 12', 'Visual Studio 11', 'MinGW Makefiles', 'Unix Makefiles']
else:
cmake_generators = ['Unix Makefiles']
for cmake_generator in cmake_generators:
proc = subprocess.Popen(['cmake', '-G', cmake_generator, '-DCMAKE_BUILD_TYPE='+cmake_build_type, shared.path_from_root('tools', 'optimizer')], cwd=build_path, stdin=log_output, stdout=log_output, stderr=log_output)
proc.communicate()
make_env = os.environ.copy()
if proc.returncode == 0:
if 'Visual Studio' in cmake_generator:
ret = find_msbuild(os.path.join(build_path, 'asmjs_optimizer.sln'), make_env)
make = [ret[0], '/t:Build', '/p:Configuration='+cmake_build_type, '/nologo', '/verbosity:minimal', 'asmjs_optimizer.sln']
make_env = ret[1]
elif 'MinGW' in cmake_generator:
make = ['mingw32-make']
else:
make = ['make']
proc = subprocess.Popen(make, cwd=build_path, stdin=log_output, stdout=log_output, stderr=log_output, env=make_env)
proc.communicate()
if proc.returncode == 0:
if WINDOWS and 'Visual Studio' in cmake_generator:
shutil.copyfile(os.path.join(build_path, cmake_build_type, 'optimizer.exe'), output)
else:
shutil.copyfile(os.path.join(build_path, 'optimizer'), output)
return output
else:
shared.try_delete(os.path.join(build_path, 'CMakeCache.txt'))
# Proceed to next iteration of the loop to try next possible CMake generator.
raise NativeOptimizerCreationException()
import os, sys
from subprocess import Popen, PIPE, STDOUT
import shared
print 'Building zlib'
emscripten_temp_dir = shared.get_emscripten_temp_dir()
zlib = shared.Building.build_library('zlib', emscripten_temp_dir, emscripten_temp_dir, ['libz.a'], make_args=['libz.a'], copy_project=True, source_dir=shared.path_from_root('tests', 'zlib'))[0]
print 'Building minigzip'
Popen([shared.PYTHON, shared.EMCC, '-O2', shared.path_from_root('tests', 'zlib', 'minigzip.c'), zlib, '-o', shared.path_from_root('tools', 'minigzip.js')]).communicate()
def create_dlmalloc(out_name, clflags):
o = in_temp(out_name)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o] + clflags)
return o
stdout=open(fullname, 'w'))
print '1) Generate source... %.2f K' % (len(open(fullname).read())/1024.)
tried += 1
print '2) Compile natively'
shared.try_delete(filename)
try:
shared.check_execute([COMP, '-m32', opts, fullname, '-o', filename + '1'] + CSMITH_CFLAGS + ['-w']) # + shared.EMSDK_OPTS
except Exception, e:
print 'Failed to compile natively using clang'
notes['invalid'] += 1
continue
shared.check_execute([COMP, '-m32', opts, '-emit-llvm', '-c', fullname, '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS)
shared.check_execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'])
shutil.move(filename + '.bc.run', filename + '2')
shared.check_execute([COMP, fullname, '-o', filename + '3'] + CSMITH_CFLAGS)
print '3) Run natively'
try:
correct1 = shared.jsrun.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct1 or len(correct1) < 10: raise Exception('segfault')
correct2 = shared.jsrun.timeout_run(Popen([filename + '2'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct2 or len(correct2) < 10: raise Exception('segfault')
correct3 = shared.jsrun.timeout_run(Popen([filename + '3'], stdout=PIPE, stderr=PIPE), 3)
if 'Segmentation fault' in correct3 or len(correct3) < 10: raise Exception('segfault')
if correct1 != correct3: raise Exception('clang opts change result')
except Exception, e:
print 'Failed or infinite looping in native, skipping', e
notes['invalid'] += 1
continue
def calculate(temp_files, in_temp, stdout_, stderr_, forced=[]):
global stdout, stderr
stdout = stdout_
stderr = stderr_
# Check if we need to include some libraries that we compile. (We implement libc ourselves in js, but
# compile a malloc implementation and stdlibc++.)
def read_symbols(path, exclude=None):
symbols = map(lambda line: line.strip().split(' ')[1], open(path).readlines())
if exclude:
symbols = filter(lambda symbol: symbol not in exclude, symbols)
return set(symbols)
default_opts = []
# XXX We also need to add libc symbols that use malloc, for example strdup. It's very rare to use just them and not
# a normal malloc symbol (like free, after calling strdup), so we haven't hit this yet, but it is possible.
libc_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libc.symbols'))
libcxx_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxx', 'symbols'), exclude=libc_symbols)
libcxxabi_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxxabi', 'symbols'), exclude=libc_symbols)
gl_symbols = read_symbols(shared.path_from_root('system', 'lib', 'gl.symbols'))
compiler_rt_symbols = read_symbols(shared.path_from_root('system', 'lib', 'compiler-rt.symbols'))
pthreads_symbols = read_symbols(shared.path_from_root('system', 'lib', 'pthreads.symbols'))
# XXX we should disable EMCC_DEBUG when building libs, just like in the relooper
def build_libc(lib_filename, files, lib_opts):
o_s = []
musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')]
commands = []
# Hide several musl warnings that produce a lot of spam to unit test build server logs.
# TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up.
c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility', '-Wno-pointer-sign']
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def build_libcxx(src_dirname, lib_filename, files, lib_opts, has_noexcept_version=False):
o_s = []
commands = []
opts = default_opts + lib_opts
if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING:
opts += ['-fno-exceptions']
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + opts)
o_s.append(o)
run_commands(commands)
if lib_filename.endswith('.bc'):
shared.Building.link(o_s, in_temp(lib_filename))
elif lib_filename.endswith('.a'):
shared.Building.emar('cr', in_temp(lib_filename), o_s)
else:
raise Exception('unknown suffix ' + lib_filename)
return in_temp(lib_filename)
# libc
def create_libc(libname):
logging.debug(' building libc for cache')
libc_files = [
]
musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src')
blacklist = set(
['ipc', 'passwd', 'thread', 'signal', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'process', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules
['memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c'] + # individual files
['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files
)
# TODO: consider using more math code from musl, doing so makes box2d faster
for dirpath, dirnames, filenames in os.walk(musl_srcdir):
for f in filenames:
if f.endswith('.c'):
if f in blacklist: continue
dir_parts = os.path.split(dirpath)
cancel = False
for part in dir_parts:
if part in blacklist:
cancel = True
break
if not cancel:
libc_files.append(os.path.join(musl_srcdir, dirpath, f))
args = ['-Os']
if shared.Settings.USE_PTHREADS:
args += ['-s', 'USE_PTHREADS=1']
assert '-mt' in libname
else:
assert '-mt' not in libname
return build_libc(libname, libc_files, args)
def create_pthreads(libname):
# Add pthread files.
pthreads_files = [os.path.join('pthread', 'library_pthread.c')]
pthreads_files += [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'thread', x) for x in ('pthread_attr_destroy.c', 'pthread_condattr_setpshared.c', 'pthread_mutex_lock.c', 'pthread_spin_destroy.c', 'pthread_attr_get.c', 'pthread_cond_broadcast.c', 'pthread_mutex_setprioceiling.c', 'pthread_spin_init.c', 'pthread_attr_init.c', 'pthread_cond_destroy.c', 'pthread_mutex_timedlock.c', 'pthread_spin_lock.c', 'pthread_attr_setdetachstate.c', 'pthread_cond_init.c', 'pthread_mutex_trylock.c', 'pthread_spin_trylock.c', 'pthread_attr_setguardsize.c', 'pthread_cond_signal.c', 'pthread_mutex_unlock.c', 'pthread_spin_unlock.c', 'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c', 'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c', 'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c', 'sem_getvalue.c', 'pthread_attr_setschedpolicy.c', 'pthread_equal.c', 'pthread_rwlockattr_init.c', 'sem_init.c', 'pthread_attr_setscope.c', 'pthread_getspecific.c', 'pthread_rwlockattr_setpshared.c', 'sem_open.c', 'pthread_attr_setstack.c', 'pthread_key_create.c', 'pthread_rwlock_destroy.c', 'sem_post.c', 'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c', 'pthread_rwlock_init.c', 'sem_timedwait.c', 'pthread_barrierattr_destroy.c', 'pthread_mutexattr_init.c', 'pthread_rwlock_rdlock.c', 'sem_trywait.c', 'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c', 'pthread_rwlock_timedrdlock.c', 'sem_unlink.c', 'pthread_barrierattr_setpshared.c', 'pthread_mutexattr_setpshared.c', 'pthread_rwlock_timedwrlock.c', 'sem_wait.c', 'pthread_barrier_destroy.c', 'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c', '__timedwait.c', 'pthread_barrier_init.c', 'pthread_mutexattr_settype.c', 'pthread_rwlock_trywrlock.c', 'vmlock.c', 'pthread_barrier_wait.c', 'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c', '__wait.c', 'pthread_condattr_destroy.c', 'pthread_mutex_destroy.c', 'pthread_rwlock_wrlock.c', 'pthread_condattr_init.c', 'pthread_mutex_getprioceiling.c', 'pthread_setcanceltype.c', 'pthread_condattr_setclock.c', 'pthread_mutex_init.c', 'pthread_setspecific.c')]
return build_libc(libname, pthreads_files, ['-O2', '-s', 'USE_PTHREADS=1'])
# libcxx
def create_libcxx(libname):
logging.debug('building libcxx for cache')
libcxx_files = [
'algorithm.cpp',
'condition_variable.cpp',
'future.cpp',
'iostream.cpp',
'memory.cpp',
'random.cpp',
'stdexcept.cpp',
'system_error.cpp',
'utility.cpp',
'bind.cpp',
'debug.cpp',
'hash.cpp',
'mutex.cpp',
'string.cpp',
'thread.cpp',
'valarray.cpp',
'chrono.cpp',
'exception.cpp',
'ios.cpp',
'locale.cpp',
'regex.cpp',
'strstream.cpp'
]
return build_libcxx(os.path.join('system', 'lib', 'libcxx'), libname, libcxx_files, ['-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')], has_noexcept_version=True)
# libcxxabi - just for dynamic_cast for now
def create_libcxxabi(libname):
logging.debug('building libcxxabi for cache')
libcxxabi_files = [
'abort_message.cpp',
'cxa_aux_runtime.cpp',
'cxa_default_handlers.cpp',
'cxa_demangle.cpp',
'cxa_exception_storage.cpp',
'cxa_new_delete.cpp',
'cxa_handlers.cpp',
'exception.cpp',
'stdexcept.cpp',
'typeinfo.cpp',
'private_typeinfo.cpp',
os.path.join('..', '..', 'libcxx', 'new.cpp'),
]
return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), libname, libcxxabi_files, ['-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
# gl
def create_gl(libname): # libname is ignored, this is just one .o file
o = in_temp('gl.o')
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o])
return o
def create_compiler_rt(libname):
srcdir = shared.path_from_root('system', 'lib', 'compiler-rt')
filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c']
files = (os.path.join(srcdir, f) for f in filenames)
o_s = []
commands = []
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o])
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(libname))
return in_temp(libname)
def create_dlmalloc(out_name, clflags):
o = in_temp(out_name)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o] + clflags)
return o
def create_dlmalloc_singlethreaded(libname):
return create_dlmalloc(libname, ['-O2'])
def create_dlmalloc_singlethreaded_tracing(libname):
return create_dlmalloc(libname, ['-O2', '--tracing'])
def create_dlmalloc_multithreaded(libname):
return create_dlmalloc(libname, ['-O2', '-s', 'USE_PTHREADS=1'])
def create_dlmalloc_multithreaded_tracing(libname):
return create_dlmalloc(libname, ['-O2', '-s', 'USE_PTHREADS=1', '--tracing'])
def create_dlmalloc_split(libname):
dlmalloc_o = in_temp('dl' + libname)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', dlmalloc_o, '-O2', '-DMSPACES', '-DONLY_MSPACES'])
split_malloc_o = in_temp('sm' + libname)
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2'])
lib = in_temp(libname)
shared.Building.link([dlmalloc_o, split_malloc_o], lib)
return lib
# Setting this in the environment will avoid checking dependencies and make building big projects a little faster
# 1 means include everything; otherwise it can be the name of a lib (libcxx, etc.)
# You can provide 1 to include everything, or a comma-separated list with the ones you want
force = os.environ.get('EMCC_FORCE_STDLIBS')
force_all = force == '1'
force = set((force.split(',') if force else []) + forced)
if force: logging.debug('forcing stdlibs: ' + str(force))
# Setting this will only use the forced libs in EMCC_FORCE_STDLIBS. This avoids spending time checking
# for unresolved symbols in your project files, which can speed up linking, but if you do not have
# the proper list of actually needed libraries, errors can occur. See below for how we must
# export all the symbols in deps_info when using this option.
only_forced = os.environ.get('EMCC_ONLY_FORCED_STDLIBS')
if only_forced:
temp_files = []
# Add in some hacks for js libraries. If a js lib depends on a symbol provided by a C library, it must be
# added to here, because our deps go only one way (each library here is checked, then we check the next
# in order - libcxx, libcxextra, etc. - and then we run the JS compiler and provide extra symbols from
# library*.js files. But we cannot then go back to the C libraries if a new dep was added!
# TODO: Move all __deps from src/library*.js to deps_info.json, and use that single source of info
# both here and in the JS compiler.
deps_info = json.loads(open(shared.path_from_root('src', 'deps_info.json')).read())
added = set()
def add_back_deps(need):
more = False
for ident, deps in deps_info.iteritems():
if ident in need.undefs and not ident in added:
added.add(ident)
more = True
for dep in deps:
need.undefs.add(dep)
shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep)
if more:
add_back_deps(need) # recurse to get deps of deps
# Scan symbols
symbolses = map(lambda temp_file: shared.Building.llvm_nm(temp_file), temp_files)
if len(symbolses) == 0:
class Dummy:
defs = set()
undefs = set()
symbolses.append(Dummy())
# depend on exported functions
for export in shared.Settings.EXPORTED_FUNCTIONS:
if shared.Settings.VERBOSE: logging.debug('adding dependency on export %s' % export)
symbolses[0].undefs.add(export[1:])
for symbols in symbolses:
add_back_deps(symbols)
# If we are only doing forced stdlibs, then we don't know the actual symbols we need,
# and must assume all of deps_info must be exported. Note that this might cause
# warnings on exports that do not exist.
if only_forced:
for key, value in deps_info.iteritems():
for dep in value:
shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep)
all_needed = set()
for symbols in symbolses:
all_needed.update(symbols.undefs)
for symbols in symbolses:
all_needed.difference_update(symbols.defs)
system_libs = [('libcxx', 'a', create_libcxx, libcxx_symbols, ['libcxxabi'], True),
('libcxxabi', 'bc', create_libcxxabi, libcxxabi_symbols, ['libc'], False),
('gl', 'bc', create_gl, gl_symbols, ['libc'], False),
('compiler-rt', 'bc', create_compiler_rt, compiler_rt_symbols, ['libc'], False)]
# malloc dependency is force-added, so when using pthreads, it must be force-added
# as well, since malloc needs to be thread-safe, so it depends on mutexes.
if shared.Settings.USE_PTHREADS:
system_libs += [('libc-mt', 'bc', create_libc, libc_symbols, [], False),
('pthreads', 'bc', create_pthreads, pthreads_symbols, ['libc'], False),
('dlmalloc_threadsafe', 'bc', create_dlmalloc_multithreaded, [], [], False),
('dlmalloc_threadsafe_tracing', 'bc', create_dlmalloc_multithreaded_tracing, [], [], False)]
force.add('pthreads')
if shared.Settings.EMSCRIPTEN_TRACING:
force.add('dlmalloc_threadsafe_tracing')
else:
force.add('dlmalloc_threadsafe')
else:
system_libs += [('libc', 'bc', create_libc, libc_symbols, [], False)]
if shared.Settings.EMSCRIPTEN_TRACING:
system_libs += [('dlmalloc_tracing', 'bc', create_dlmalloc_singlethreaded_tracing, [], [], False)]
force.add('dlmalloc_tracing')
else:
if shared.Settings.SPLIT_MEMORY:
system_libs += [('dlmalloc_split', 'bc', create_dlmalloc_split, [], [], False)]
force.add('dlmalloc_split')
else:
system_libs += [('dlmalloc', 'bc', create_dlmalloc_singlethreaded, [], [], False)]
force.add('dlmalloc')
# Go over libraries to figure out which we must include
def maybe_noexcept(name):
if shared.Settings.DISABLE_EXCEPTION_CATCHING:
name += '_noexcept'
return name
ret = []
has = need = None
for shortname, suffix, create, library_symbols, deps, can_noexcept in system_libs:
force_this = force_all or shortname in force
if can_noexcept: shortname = maybe_noexcept(shortname)
if force_this:
suffix = 'bc' # .a files do not always link in all their parts; don't use them when forced
name = shortname + '.' + suffix
if not force_this:
need = set()
has = set()
for symbols in symbolses:
if shared.Settings.VERBOSE: logging.debug('undefs: ' + str(symbols.undefs))
for library_symbol in library_symbols:
if library_symbol in symbols.undefs:
need.add(library_symbol)
if library_symbol in symbols.defs:
has.add(library_symbol)
for haz in has: # remove symbols that are supplied by another of the inputs
if haz in need:
need.remove(haz)
if shared.Settings.VERBOSE: logging.debug('considering %s: we need %s and have %s' % (name, str(need), str(has)))
if force_this or (len(need) > 0 and not only_forced):
# We need to build and link the library in
logging.debug('including %s' % name)
def do_create():
ret = create(name)
return ret
libfile = shared.Cache.get(name, do_create, extension=suffix)
ret.append(libfile)
force = force.union(deps)
ret.sort(key=lambda x: x.endswith('.a')) # make sure to put .a files at the end.
for actual in ret:
if os.path.basename(actual) == 'libcxxabi.bc':
# libcxxabi and libcxx *static* linking is tricky. e.g. cxa_demangle.cpp disables c++
# exceptions, but since the string methods in the headers are *weakly* linked, then
# we might have exception-supporting versions of them from elsewhere, and if libcxxabi
# is first then it would "win", breaking exception throwing from those string
# header methods. To avoid that, we link libcxxabi last.
ret = filter(lambda f: f != actual, ret) + [actual]
return ret
def create_libcxxabi():
logging.debug('building libcxxabi for cache')
libcxxabi_files = [
'abort_message.cpp',
'cxa_aux_runtime.cpp',
'cxa_default_handlers.cpp',
'cxa_demangle.cpp',
'cxa_exception_storage.cpp',
'cxa_new_delete.cpp',
'cxa_handlers.cpp',
'exception.cpp',
'stdexcept.cpp',
'typeinfo.cpp',
'private_typeinfo.cpp',
os.path.join('..', '..', 'libcxx', 'new.cpp'),
]
return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc', libcxxabi_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
def create_libcxx():
logging.debug('building libcxx for cache')
libcxx_files = [
'algorithm.cpp',
'condition_variable.cpp',
'future.cpp',
'iostream.cpp',
'memory.cpp',
'random.cpp',
'stdexcept.cpp',
'system_error.cpp',
'utility.cpp',
'bind.cpp',
'debug.cpp',
'hash.cpp',
'mutex.cpp',
'string.cpp',
'thread.cpp',
'valarray.cpp',
'chrono.cpp',
'exception.cpp',
'ios.cpp',
'locale.cpp',
'regex.cpp',
'strstream.cpp'
]
return build_libcxx(os.path.join('system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
def calculate(temp_files, in_temp, stdout_, stderr_, forced=[]):
global stdout, stderr
stdout = stdout_
stderr = stderr_
# Check if we need to include some libraries that we compile. (We implement libc ourselves in js, but
# compile a malloc implementation and stdlibc++.)
def read_symbols(path, exclude=None):
symbols = map(lambda line: line.strip().split(' ')[1], open(path).readlines())
if exclude:
symbols = filter(lambda symbol: symbol not in exclude, symbols)
return set(symbols)
default_opts = []
# If we're building tracing, we should build the system libraries that way too.
if shared.Settings.EMSCRIPTEN_TRACING:
default_opts.append('--tracing')
# XXX We also need to add libc symbols that use malloc, for example strdup. It's very rare to use just them and not
# a normal malloc symbol (like free, after calling strdup), so we haven't hit this yet, but it is possible.
libc_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libc.symbols'))
libcextra_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcextra.symbols'))
libcxx_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxx', 'symbols'), exclude=libc_symbols)
libcxxabi_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxxabi', 'symbols'), exclude=libc_symbols)
gl_symbols = read_symbols(shared.path_from_root('system', 'lib', 'gl.symbols'))
# XXX we should disable EMCC_DEBUG when building libs, just like in the relooper
def build_libc(lib_filename, files, lib_opts):
o_s = []
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')]
commands = []
# Hide several musl warnings that produce a lot of spam to unit test build server logs.
# TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up.
c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility']
for src in files:
o = in_temp(os.path.basename(src) + '.o')
commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts)
o_s.append(o)
run_commands(commands)
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
def build_libcxx(src_dirname, lib_filename, files, lib_opts):
o_s = []
commands = []
for src in files:
o = in_temp(src + '.o')
srcfile = shared.path_from_root(src_dirname, src)
commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + default_opts + lib_opts)
o_s.append(o)
run_commands(commands)
shared.Building.link(o_s, in_temp(lib_filename))
return in_temp(lib_filename)
# libc
def create_libc():
logging.debug(' building libc for cache')
libc_files = [
'dlmalloc.c',
]
musl_files = [
['ctype', [
'isdigit.c',
'isspace.c',
'isupper.c',
'isxdigit.c',
'tolower.c',
]],
['internal', [
'intscan.c',
'floatscan.c',
'shgetc.c',
]],
['math', [
'frexp.c',
'frexpf.c',
'frexpl.c',
'scalbn.c',
'scalbnl.c',
]],
['multibyte', [
'wctomb.c',
'wcrtomb.c',
]],
['prng', [
'__rand48_step.c',
'__seed48.c',
'drand48.c',
'lcong48.c',
'lrand48.c',
'mrand48.c',
'rand_r.c',
'rand.c',
'random.c',
'seed48.c',
'srand48.c'
]],
['stdio', [
'__overflow.c',
'__toread.c',
'__towrite.c',
'__uflow.c',
'fwrite.c',
'snprintf.c',
'sprintf.c',
'vfprintf.c',
'vsnprintf.c',
'vsprintf.c',
]],
['stdlib', [
'atof.c',
'atoi.c',
'atol.c',
'strtod.c',
'strtol.c',
]],
['string', [
'memchr.c',
'memcmp.c',
'strcasecmp.c',
'strcmp.c',
'strncasecmp.c',
'strncmp.c',
]]
]
for directory, sources in musl_files:
libc_files += [os.path.join('libc', 'musl', 'src', directory, source) for source in sources]
return build_libc('libc.bc', libc_files, ['-O2'])
def apply_libc(need):
# libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines
try:
if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2')
except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings
shared.Settings.CORRECT_SIGNS = 2
if shared.Settings.CORRECT_SIGNS == 2:
shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]]
# If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected
# so all is well anyhow too.
return True
# libcextra
def create_libcextra():
logging.debug('building libcextra for cache')
musl_files = [
['compat', [
'strlwr.c',
'strtol_l.c',
'strupr.c'
]],
['ctype', [
'isalnum.c',
'isalpha.c',
'isascii.c',
'isblank.c',
'iscntrl.c',
'isgraph.c',
'islower.c',
'isprint.c',
'ispunct.c',
'iswalnum.c',
'iswalpha.c',
'iswblank.c',
'iswcntrl.c',
'iswctype.c',
'iswdigit.c',
'iswgraph.c',
'iswlower.c',
'iswprint.c',
'iswpunct.c',
'iswspace.c',
'iswupper.c',
'iswxdigit.c',
'toascii.c',
'toupper.c',
'towctrans.c',
'wcswidth.c',
'wctrans.c',
'wcwidth.c',
]],
['legacy', [
'err.c',
]],
['locale', [
'iconv.c',
'isalnum_l.c',
'isalpha_l.c',
'isblank_l.c',
'iscntrl_l.c',
'isdigit_l.c',
'isgraph_l.c',
'islower_l.c',
'isprint_l.c',
'ispunct_l.c',
'isspace_l.c',
'isupper_l.c',
'isxdigit_l.c',
'iswalnum_l.c',
'iswalpha_l.c',
'iswblank_l.c',
'iswcntrl_l.c',
'iswctype_l.c',
'iswdigit_l.c',
'iswgraph_l.c',
'iswlower_l.c',
'iswprint_l.c',
'iswpunct_l.c',
'iswspace_l.c',
'iswupper_l.c',
'iswxdigit_l.c',
'strcoll.c',
'strcasecmp_l.c',
'strfmon.c',
'strncasecmp_l.c',
'strxfrm.c',
'tolower_l.c',
'toupper_l.c',
'towctrans_l.c',
'towlower_l.c',
'towupper_l.c',
'wcscoll.c',
'wcscoll_l.c',
'wcsxfrm.c',
'wcsxfrm_l.c',
'wctrans_l.c',
'wctype_l.c',
]],
['math', [
'__cos.c',
'__cosdf.c',
'__sin.c',
'__sindf.c',
'ilogb.c',
'ilogbf.c',
'ilogbl.c',
'j0.c',
'j0f.c',
'j1.c',
'j1f.c',
'jn.c',
'jnf.c',
'ldexp.c',
'ldexpf.c',
'ldexpl.c',
'logb.c',
'logbf.c',
'logbl.c',
'lgamma.c',
'lgamma_r.c',
'lgammaf.c',
'lgammaf_r.c',
'lgammal.c',
'scalbnf.c',
'signgam.c',
'tgamma.c',
'tgammaf.c',
'tgammal.c'
]],
['misc', [
'ffs.c',
'getopt.c',
'getopt_long.c',
]],
['multibyte', [
'btowc.c',
'internal.c',
'mblen.c',
'mbrlen.c',
'mbrtowc.c',
'mbsinit.c',
'mbsnrtowcs.c',
'mbsrtowcs.c',
'mbstowcs.c',
'mbtowc.c',
'wcsnrtombs.c',
'wcsrtombs.c',
'wcstombs.c',
'wctob.c',
]],
['regex', [
'fnmatch.c',
'glob.c',
'regcomp.c',
'regerror.c',
'regexec.c',
'tre-mem.c',
]],
['stdio', [
'__string_read.c',
'asprintf.c',
'fwprintf.c',
'swprintf.c',
'vfwprintf.c',
'vswprintf.c',
'vwprintf.c',
'wprintf.c',
'fputwc.c',
'fputws.c',
'sscanf.c',
'vasprintf.c',
'vfscanf.c',
'vsscanf.c',
]],
['stdlib', [
'atoll.c',
'bsearch.c',
'ecvt.c',
'fcvt.c',
'gcvt.c',
'qsort.c',
'wcstod.c',
'wcstol.c',
]],
['string', [
'bcmp.c',
'bcopy.c',
'bzero.c',
'index.c',
'memccpy.c',
'memmem.c',
'mempcpy.c',
'memrchr.c',
'rindex.c',
'stpcpy.c',
'strcasestr.c',
'strchr.c',
'strchrnul.c',
'strcspn.c',
'strdup.c',
'strlcat.c',
'strlcpy.c',
'strncat.c',
'strndup.c',
'strnlen.c',
'strpbrk.c',
'strrchr.c',
'strsep.c',
'strsignal.c',
'strspn.c',
'strstr.c',
'strtok.c',
'strtok_r.c',
'strverscmp.c',
'wcpcpy.c',
'wcpncpy.c',
'wcscasecmp.c',
'wcscasecmp_l.c',
'wcscat.c',
'wcschr.c',
'wcscmp.c',
'wcscpy.c',
'wcscspn.c',
'wcsdup.c',
'wcslen.c',
'wcsncasecmp.c',
'wcsncasecmp_l.c',
'wcsncat.c',
'wcsncmp.c',
'wcsncpy.c',
'wcsnlen.c',
'wcspbrk.c',
'wcsrchr.c',
'wcsspn.c',
'wcsstr.c',
'wcstok.c',
'wcswcs.c',
'wmemchr.c',
'wmemcmp.c',
'wmemcpy.c',
'wmemmove.c',
'wmemset.c',
]]
]
libcextra_files = []
for directory, sources in musl_files:
libcextra_files += [os.path.join('libc', 'musl', 'src', directory, source) for source in sources]
return build_libc('libcextra.bc', libcextra_files, ['-O2'])
# libcxx
def create_libcxx():
logging.debug('building libcxx for cache')
libcxx_files = [
'algorithm.cpp',
'condition_variable.cpp',
'future.cpp',
'iostream.cpp',
'memory.cpp',
'random.cpp',
'stdexcept.cpp',
'system_error.cpp',
'utility.cpp',
'bind.cpp',
'debug.cpp',
'hash.cpp',
'mutex.cpp',
'string.cpp',
'thread.cpp',
'valarray.cpp',
'chrono.cpp',
'exception.cpp',
'ios.cpp',
'locale.cpp',
'regex.cpp',
'strstream.cpp'
]
return build_libcxx(os.path.join('system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
def apply_libcxx(need):
assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++ with QUANTUM_SIZE == 1'
# libcxx might need corrections, so turn them all on. TODO: check which are actually needed
shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1
#logging.info('using libcxx turns on CORRECT_* options')
return True
# libcxxabi - just for dynamic_cast for now
def create_libcxxabi():
logging.debug('building libcxxabi for cache')
libcxxabi_files = [
'abort_message.cpp',
'cxa_aux_runtime.cpp',
'cxa_default_handlers.cpp',
'cxa_demangle.cpp',
'cxa_exception_storage.cpp',
'cxa_new_delete.cpp',
'cxa_handlers.cpp',
'exception.cpp',
'stdexcept.cpp',
'typeinfo.cpp',
'private_typeinfo.cpp',
os.path.join('..', '..', 'libcxx', 'new.cpp'),
]
return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc', libcxxabi_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
def apply_libcxxabi(need):
assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++abi with QUANTUM_SIZE == 1'
#logging.info('using libcxxabi, this may need CORRECT_* options')
#shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1
return True
# gl
def create_gl():
prev_cxx = os.environ.get('EMMAKEN_CXX')
if prev_cxx: os.environ['EMMAKEN_CXX'] = ''
o = in_temp('gl.o')
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o])
if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx
return o
# Setting this in the environment will avoid checking dependencies and make building big projects a little faster
# 1 means include everything; otherwise it can be the name of a lib (libcxx, etc.)
# You can provide 1 to include everything, or a comma-separated list with the ones you want
force = os.environ.get('EMCC_FORCE_STDLIBS')
force_all = force == '1'
force = set((force.split(',') if force else []) + forced)
if force: logging.debug('forcing stdlibs: ' + str(force))
# Setting this will only use the forced libs in EMCC_FORCE_STDLIBS. This avoids spending time checking
# for unresolved symbols in your project files, which can speed up linking, but if you do not have
# the proper list of actually needed libraries, errors can occur. See below for how we must
# export all the symbols in deps_info when using this option.
only_forced = os.environ.get('EMCC_ONLY_FORCED_STDLIBS')
if only_forced:
temp_files = []
# Add in some hacks for js libraries. If a js lib depends on a symbol provided by a C library, it must be
# added to here, because our deps go only one way (each library here is checked, then we check the next
# in order - libcxx, libcxextra, etc. - and then we run the JS compiler and provide extra symbols from
# library*.js files. But we cannot then go back to the C libraries if a new dep was added!
# TODO: Move all __deps from src/library*.js to deps_info.json, and use that single source of info
# both here and in the JS compiler.
deps_info = json.loads(open(shared.path_from_root('src', 'deps_info.json')).read())
added = set()
def add_back_deps(need):
more = False
for ident, deps in deps_info.iteritems():
if ident in need.undefs and not ident in added:
added.add(ident)
more = True
for dep in deps:
need.undefs.add(dep)
shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep)
if more:
add_back_deps(need) # recurse to get deps of deps
# Scan symbols
symbolses = map(lambda temp_file: shared.Building.llvm_nm(temp_file), temp_files)
if len(symbolses) == 0:
class Dummy:
defs = set()
undefs = set()
symbolses.append(Dummy())
# depend on exported functions
for export in shared.Settings.EXPORTED_FUNCTIONS:
if shared.Settings.VERBOSE: logging.debug('adding dependency on export %s' % export)
symbolses[0].undefs.add(export[1:])
for symbols in symbolses:
add_back_deps(symbols)
# If we are only doing forced stdlibs, then we don't know the actual symbols we need,
# and must assume all of deps_info must be exported. Note that this might cause
# warnings on exports that do not exist.
if only_forced:
for key, value in deps_info.iteritems():
for dep in value:
shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep)
all_needed = set()
for symbols in symbolses:
all_needed.update(symbols.undefs)
for symbols in symbolses:
all_needed.difference_update(symbols.defs)
# Go over libraries to figure out which we must include
ret = []
has = need = None
for name, create, apply_, library_symbols, deps in [('libcxx', create_libcxx, apply_libcxx, libcxx_symbols, ['libcextra', 'libcxxabi']),
('libcextra', create_libcextra, lambda x: True, libcextra_symbols, ['libc']),
('libcxxabi', create_libcxxabi, apply_libcxxabi, libcxxabi_symbols, ['libc']),
('gl', create_gl, lambda x: True, gl_symbols, ['libc']),
('libc', create_libc, apply_libc, libc_symbols, [])]:
force_this = force_all or name in force
if not force_this:
need = set()
has = set()
for symbols in symbolses:
if shared.Settings.VERBOSE: logging.debug('undefs: ' + str(symbols.undefs))
for library_symbol in library_symbols:
if library_symbol in symbols.undefs:
need.add(library_symbol)
if library_symbol in symbols.defs:
has.add(library_symbol)
for haz in has: # remove symbols that are supplied by another of the inputs
if haz in need:
need.remove(haz)
if shared.Settings.VERBOSE: logging.debug('considering %s: we need %s and have %s' % (name, str(need), str(has)))
if force_this or (len(need) > 0 and not only_forced):
if apply_(need):
# We need to build and link the library in
logging.debug('including %s' % name)
libfile = shared.Cache.get(name, create)
ret.append(libfile)
force = force.union(deps)
return ret
def create_gl():
o = in_temp('gl.o')
check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o])
return o
