def build_libc(lib_filename, files): o_s = [] prev_cxx = os.environ.get("EMMAKEN_CXX") if prev_cxx: os.environ["EMMAKEN_CXX"] = "" musl_internal_includes = shared.path_from_root("system", "lib", "libc", "musl", "src", "internal") for src in files: o = in_temp(os.path.basename(src) + ".o") execute( [ shared.PYTHON, shared.EMCC, shared.path_from_root("system", "lib", src), "-o", o, "-I", musl_internal_includes, ] + lib_opts, stdout=stdout, stderr=stderr, ) o_s.append(o) if prev_cxx: os.environ["EMMAKEN_CXX"] = prev_cxx shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def create_dlmalloc(out_name): o = in_temp(out_name) cflags = ['-O2'] if shared.Settings.USE_PTHREADS: cflags += ['-s', 'USE_PTHREADS=1'] if shared.Settings.EMSCRIPTEN_TRACING: cflags += ['--tracing'] if shared.Settings.SPLIT_MEMORY: cflags += ['-DMSPACES', '-DONLY_MSPACES'] if shared.Settings.DEBUG_LEVEL: cflags += ['-DDLMALLOC_DEBUG'] check_call([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o ] + cflags) if shared.Settings.SPLIT_MEMORY: split_malloc_o = in_temp('sm' + out_name) check_call([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2' ]) lib = in_temp('lib' + out_name) shared.Building.link([o, split_malloc_o], lib) shutil.move(lib, o) return o
def create_wasm_compiler_rt(libname): srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins') filenames = ['addtf3.c', 'ashlti3.c', 'ashrti3.c', 'comparetf2.c', 'divtf3.c', 'divti3.c', 'udivmodti4.c', 'extenddftf2.c', 'extendsftf2.c', 'fixdfti.c', 'fixsfti.c', 'fixtfdi.c', 'fixtfsi.c', 'fixtfti.c', 'fixunsdfti.c', 'fixunssfti.c', 'fixunstfdi.c', 'fixunstfsi.c', 'fixunstfti.c', 'floatditf.c', 'floatsitf.c', 'floattidf.c', 'floattisf.c', 'floatunditf.c', 'floatunsitf.c', 'floatuntidf.c', 'floatuntisf.c', 'lshrti3.c', 'modti3.c', 'multf3.c', 'multi3.c', 'subtf3.c', 'udivti3.c', 'umodti3.c', 'ashrdi3.c', 'ashldi3.c', 'fixdfdi.c', 'floatdidf.c', 'lshrdi3.c', 'moddi3.c', 'trunctfdf2.c', 'trunctfsf2.c', 'umoddi3.c', 'fixunsdfdi.c', 'muldi3.c', 'divdi3.c', 'divmoddi4.c', 'udivdi3.c', 'udivmoddi4.c'] files = (os.path.join(srcdir, f) for f in filenames) o_s = [] commands = [] for src in files: o = in_temp(os.path.basename(src) + '.o') # Use clang directly instead of emcc. Since emcc's intermediate format (produced by -S) is LLVM IR, there's no way to # get emcc to output wasm .s files, which is what we archive in compiler_rt. commands.append([shared.CLANG_CC, '--target=wasm32', '-S', shared.path_from_root('system', 'lib', src), '-O2', '-o', o] + shared.EMSDK_OPTS) o_s.append(o) run_commands(commands) lib = in_temp(libname) run_commands([[shared.LLVM_AR, 'cr', '-format=gnu', lib] + o_s]) return lib
def create_optimizer(): shared.logging.debug('building native optimizer: ' + name) output = shared.Cache.get_path(name) shared.try_delete(output) for compiler in [ shared.CLANG, 'g++', 'clang++' ]: # try our clang first, otherwise hope for a system compiler in the path shared.logging.debug(' using ' + compiler) try: out, err = subprocess.Popen( [ compiler, shared.path_from_root('tools', 'optimizer', 'parser.cpp'), shared.path_from_root('tools', 'optimizer', 'simple_ast.cpp'), shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), shared.path_from_root('tools', 'optimizer', 'optimizer-shared.cpp'), shared.path_from_root('tools', 'optimizer', 'optimizer-main.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output ] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate() outs.append(out) errs.append(err) except OSError: if compiler == shared.CLANG: raise # otherwise, OSError is likely due to g++ or clang++ not being in the path if os.path.exists(output): return output raise NativeOptimizerCreationException()
def build_libc(lib_filename, files, lib_opts): o_s = [] musl_internal_includes = [ '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js') ] commands = [] # Hide several musl warnings that produce a lot of spam to unit test build server logs. # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up. c_opts = [ '-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility' ] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o ] + musl_internal_includes + default_opts + c_opts + lib_opts) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def create_wasm_libc(libname): # in asm.js we just use Math.sin etc., which is good for code size. But wasm doesn't have such builtins, so # we need to bundle in more code # we also build in musl versions of things that we have hand-optimized asm.js for files = ([shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'math', x) for x in ('cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'pow.c', 'powf.c', 'powl.c')] + [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'string', x) for x in ('memcpy.c', 'memset.c', 'memmove.c')]) return build_libc(libname, files, ['-O2'])
def create_dlmalloc_split(libname): dlmalloc_o = in_temp('dl' + libname) check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', dlmalloc_o, '-O2', '-DMSPACES', '-DONLY_MSPACES']) split_malloc_o = in_temp('sm' + libname) check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2']) lib = in_temp(libname) shared.Building.link([dlmalloc_o, split_malloc_o], lib) return lib
def musl_internal_includes(): return [ '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js'), ]
def create_load_wasm_worker(): emscripten.logging.debug('building load-wasm-worker') output = emscripten.Cache.get_path('load-wasm-worker.js') emscripten.try_delete(output) check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'), emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'), '-o', output] + \ '-O3 -std=c++11 --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0'.split(' ')) assert os.path.exists(output) open(output, 'a').write(open(emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'load-wasm-worker.js')).read()) return output
def build_libc(lib_filename, files): o_s = [] prev_cxx = os.environ.get('EMMAKEN_CXX') if prev_cxx: os.environ['EMMAKEN_CXX'] = '' musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')] for src in files: o = in_temp(os.path.basename(src) + '.o') execute([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + lib_opts, stdout=stdout, stderr=stderr) o_s.append(o) if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def create_pack_asmjs(): emscripten.logging.debug('building pack-asmjs') output = emscripten.Cache.get_path('pack-asmjs.js') emscripten.try_delete(output) check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.cpp'), emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'), emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'), '-o', output] + \ '-O3 -std=c++11 -DCHECKED_OUTPUT_SIZE --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0 -s INVOKE_RUN=0'.split(' ') + \ ['-I' + emscripten.path_from_root('tools', 'optimizer')]) assert os.path.exists(output) open(output, 'a').write(open(emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.js')).read()) return output
def create_compiler_rt(libname): srcdir = shared.path_from_root('system', 'lib', 'compiler-rt') filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c'] files = (os.path.join(srcdir, f) for f in filenames) o_s = [] commands = [] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o]) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(libname)) return in_temp(libname)
def create_load_wasm_worker(): emscripten.logging.debug('building load-wasm-worker') output = emscripten.Cache.get_path('load-wasm-worker.js') emscripten.try_delete(output) check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'), emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'), '-o', output] + \ '-O3 -std=c++11 --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0'.split(' ')) assert os.path.exists(output) open(output, 'a').write( open( emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'load-wasm-worker.js')).read()) return output
def build_libc(lib_filename, files, lib_opts): o_s = [] musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')] commands = [] # Hide several musl warnings that produce a lot of spam to unit test build server logs. # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up. c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility'] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def create_compiler_rt(libname): srcdir = shared.path_from_root('system', 'lib', 'compiler-rt', 'lib', 'builtins') filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c'] files = (os.path.join(srcdir, f) for f in filenames) o_s = [] commands = [] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o]) o_s.append(o) run_commands(commands) shared.Building.emar('cr', in_temp(libname), o_s) return in_temp(libname)
def create_gl(): o = in_temp('gl.o') check_call([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o ]) return o
def build_libc(lib_filename, files, lib_opts): o_s = [] commands = [] # Hide several musl warnings that produce a lot of spam to unit test build server logs. # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up. c_opts = [ '-Wno-return-type', '-Wno-parentheses', '-Wno-ignored-attributes', '-Wno-shift-count-overflow', '-Wno-shift-negative-value', '-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility', '-Wno-pointer-sign' ] if shared.Settings.WASM_BACKEND: c_opts.append('-Wno-error=absolute-value') for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o ] + musl_internal_includes() + default_opts + c_opts + lib_opts) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def create_gl(): prev_cxx = os.environ.get('EMMAKEN_CXX') if prev_cxx: os.environ['EMMAKEN_CXX'] = '' o = in_temp('gl.o') check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o]) if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx return o
def create_optimizer(): shared.logging.debug('building native optimizer: ' + name) output = shared.Cache.get_path(name) shared.try_delete(output) for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path shared.logging.debug(' using ' + compiler) try: subprocess.Popen([compiler, shared.path_from_root('tools', 'optimizer', 'parser.cpp'), shared.path_from_root('tools', 'optimizer', 'simple_ast.cpp'), shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output] + args).communicate() except OSError: if compiler == shared.CLANG: raise # otherwise, OSError is likely due to g++ or clang++ not being in the path if os.path.exists(output): return output raise NativeOptimizerCreationException()
def create_dlmalloc(out_name, clflags): o = in_temp(out_name) check_call([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o ] + clflags) return o
def create_libc(libname): logging.debug(' building libc for cache') libc_files = [ ] musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src') blacklist = set( ['ipc', 'passwd', 'thread', 'signal', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'process', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules ['memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c'] + # individual files ['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files ) # TODO: consider using more math code from musl, doing so makes box2d faster for dirpath, dirnames, filenames in os.walk(musl_srcdir): for f in filenames: if f.endswith('.c'): if f in blacklist: continue dir_parts = os.path.split(dirpath) cancel = False for part in dir_parts: if part in blacklist: cancel = True break if not cancel: libc_files.append(os.path.join(musl_srcdir, dirpath, f)) args = ['-Os'] if shared.Settings.USE_PTHREADS: args += ['-s', 'USE_PTHREADS=1'] assert '-mt' in libname else: assert '-mt' not in libname return build_libc(libname, libc_files, args)
def build_libcxx(src_dirname, lib_filename, files, lib_opts, has_noexcept_version=False): o_s = [] commands = [] opts = default_opts + lib_opts if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING: opts += ['-fno-exceptions'] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) commands.append( [shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + opts) o_s.append(o) run_commands(commands) if lib_filename.endswith('.bc'): shared.Building.link(o_s, in_temp(lib_filename)) elif lib_filename.endswith('.a'): shared.Building.emar('cr', in_temp(lib_filename), o_s) else: raise Exception('unknown suffix ' + lib_filename) return in_temp(lib_filename)
def create_gl(libname): # libname is ignored, this is just one .o file o = in_temp('gl.o') check_call([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o ]) return o
def create_pack_asmjs(): emscripten.logging.debug('building pack-asmjs') output = emscripten.Cache.get_path('pack-asmjs.js') emscripten.try_delete(output) check_call([PYTHON, emscripten.EMCC, emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.cpp'), emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'unpack.cpp'), emscripten.path_from_root('tools', 'optimizer', 'parser.cpp'), '-o', output] + \ '-O3 -std=c++11 -DCHECKED_OUTPUT_SIZE --memory-init-file 0 --llvm-lto 1 -s TOTAL_MEMORY=67108864 -s WASM=0 -s INVOKE_RUN=0'.split(' ') + \ ['-I' + emscripten.path_from_root('tools', 'optimizer')]) assert os.path.exists(output) open(output, 'a').write( open( emscripten.path_from_root('third_party', 'wasm-polyfill', 'src', 'pack-asmjs.js')).read()) return output
def create_gl(): prev_cxx = os.environ.get("EMMAKEN_CXX") if prev_cxx: os.environ["EMMAKEN_CXX"] = "" o = in_temp("gl.o") execute([shared.PYTHON, shared.EMCC, shared.path_from_root("system", "lib", "gl.c"), "-o", o]) if prev_cxx: os.environ["EMMAKEN_CXX"] = prev_cxx return o
def build_libcxx(src_dirname, lib_filename, files): o_s = [] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) execute([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + lib_opts, stdout=stdout, stderr=stderr) o_s.append(o) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def build_libcxx(src_dirname, lib_filename, files, lib_opts): o_s = [] commands = [] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + default_opts + lib_opts) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def build_libc(lib_filename, files): o_s = [] prev_cxx = os.environ.get('EMMAKEN_CXX') if prev_cxx: os.environ['EMMAKEN_CXX'] = '' musl_internal_includes = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal') for src in files: o = in_temp(os.path.basename(src) + '.o') execute([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o, '-I', musl_internal_includes ] + lib_opts, stdout=stdout, stderr=stderr) o_s.append(o) if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def apply_libc(need): # libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines try: if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2') except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings shared.Settings.CORRECT_SIGNS = 2 if shared.Settings.CORRECT_SIGNS == 2: shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]] # If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected # so all is well anyhow too. return True
def create_optimizer(): shared.logging.debug('building native optimizer') output = shared.Cache.get_path('optimizer.exe') shared.try_delete(output) errs = [] for compiler in [shared.CLANG, 'g++', 'clang++']: # try our clang first, otherwise hope for a system compiler in the path shared.logging.debug(' using ' + compiler) out, err = subprocess.Popen([compiler, shared.path_from_root('tools', 'optimizer', 'optimizer.cpp'), '-O3', '-std=c++11', '-fno-exceptions', '-fno-rtti', '-o', output], stderr=subprocess.PIPE).communicate() # for profiling/debugging: '-g', '-fno-omit-frame-pointer' if os.path.exists(output): return output errs.append(err) raise Exception('failed to build native optimizer, errors from each attempt: ' + '\n=================\n'.join(errs))
def build_libcxx(src_dirname, lib_filename, files): o_s = [] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) execute( [shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + lib_opts, stdout=stdout, stderr=stderr) o_s.append(o) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def build_libc(lib_filename, files): o_s = [] prev_cxx = os.environ.get('EMMAKEN_CXX') if prev_cxx: os.environ['EMMAKEN_CXX'] = '' musl_internal_includes = [ '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js') ] commands = [] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([ shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o ] + musl_internal_includes + lib_opts) o_s.append(o) run_commands(commands) if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename)
def create_libcxxabi(libname): logging.debug('building libcxxabi for cache') libcxxabi_files = [ 'abort_message.cpp', 'cxa_aux_runtime.cpp', 'cxa_default_handlers.cpp', 'cxa_demangle.cpp', 'cxa_exception_storage.cpp', 'cxa_guard.cpp', 'cxa_new_delete.cpp', 'cxa_handlers.cpp', 'exception.cpp', 'stdexcept.cpp', 'typeinfo.cpp', 'private_typeinfo.cpp' ] return build_libcxx(os.path.join( 'system', 'lib', 'libcxxabi', 'src'), libname, libcxxabi_files, [ '-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include') ])
def create_optimizer(): shared.logging.debug("building native optimizer: " + name) output = shared.Cache.get_path(name) shared.try_delete(output) for compiler in [ shared.CLANG, "g++", "clang++", ]: # try our clang first, otherwise hope for a system compiler in the path shared.logging.debug(" using " + compiler) try: out, err = subprocess.Popen( [ compiler, shared.path_from_root("tools", "optimizer", "parser.cpp"), shared.path_from_root("tools", "optimizer", "simple_ast.cpp"), shared.path_from_root("tools", "optimizer", "optimizer.cpp"), shared.path_from_root("tools", "optimizer", "optimizer-main.cpp"), "-O3", "-std=c++11", "-fno-exceptions", "-fno-rtti", "-o", output, ] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ).communicate() outs.append(out) errs.append(err) except OSError: if compiler == shared.CLANG: raise # otherwise, OSError is likely due to g++ or clang++ not being in the path if os.path.exists(output): return output raise NativeOptimizerCreationException()
def create_libcxx(): logging.debug('building libcxx for cache') libcxx_files = [ 'algorithm.cpp', 'condition_variable.cpp', 'future.cpp', 'iostream.cpp', 'memory.cpp', 'random.cpp', 'stdexcept.cpp', 'system_error.cpp', 'utility.cpp', 'bind.cpp', 'debug.cpp', 'hash.cpp', 'mutex.cpp', 'string.cpp', 'thread.cpp', 'valarray.cpp', 'chrono.cpp', 'exception.cpp', 'ios.cpp', 'locale.cpp', 'regex.cpp', 'strstream.cpp' ] return build_libcxx(os.path.join( 'system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files, [ '-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include') ])
def create_pthreads(libname): # Add pthread files. pthreads_files = [os.path.join('pthread', 'library_pthread.c')] pthreads_files += [ shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'thread', x) for x in ('pthread_attr_destroy.c', 'pthread_condattr_setpshared.c', 'pthread_mutex_lock.c', 'pthread_spin_destroy.c', 'pthread_attr_get.c', 'pthread_cond_broadcast.c', 'pthread_mutex_setprioceiling.c', 'pthread_spin_init.c', 'pthread_attr_init.c', 'pthread_cond_destroy.c', 'pthread_mutex_timedlock.c', 'pthread_spin_lock.c', 'pthread_attr_setdetachstate.c', 'pthread_cond_init.c', 'pthread_mutex_trylock.c', 'pthread_spin_trylock.c', 'pthread_attr_setguardsize.c', 'pthread_cond_signal.c', 'pthread_mutex_unlock.c', 'pthread_spin_unlock.c', 'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c', 'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c', 'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c', 'sem_getvalue.c', 'pthread_attr_setschedpolicy.c', 'pthread_equal.c', 'pthread_rwlockattr_init.c', 'sem_init.c', 'pthread_attr_setscope.c', 'pthread_getspecific.c', 'pthread_rwlockattr_setpshared.c', 'sem_open.c', 'pthread_attr_setstack.c', 'pthread_key_create.c', 'pthread_rwlock_destroy.c', 'sem_post.c', 'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c', 'pthread_rwlock_init.c', 'sem_timedwait.c', 'pthread_barrierattr_destroy.c', 'pthread_mutexattr_init.c', 'pthread_rwlock_rdlock.c', 'sem_trywait.c', 'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c', 'pthread_rwlock_timedrdlock.c', 'sem_unlink.c', 'pthread_barrierattr_setpshared.c', 'pthread_mutexattr_setpshared.c', 'pthread_rwlock_timedwrlock.c', 'sem_wait.c', 'pthread_barrier_destroy.c', 'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c', '__timedwait.c', 'pthread_barrier_init.c', 'pthread_mutexattr_settype.c', 'pthread_rwlock_trywrlock.c', 'vmlock.c', 'pthread_barrier_wait.c', 'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c', '__wait.c', 'pthread_condattr_destroy.c', 'pthread_mutex_destroy.c', 'pthread_rwlock_wrlock.c', 'pthread_condattr_init.c', 'pthread_mutex_getprioceiling.c', 'pthread_setcanceltype.c', 'pthread_condattr_setclock.c', 'pthread_mutex_init.c', 'pthread_setspecific.c') ] return build_libc(libname, pthreads_files, ['-O2', '-s', 'USE_PTHREADS=1'])
def create_wasm_rt_lib(libname, files): o_s = [] commands = [] for src in files: o = in_temp(os.path.basename(src) + '.o') # Use clang directly instead of emcc. Since emcc's intermediate format (produced by -S) is LLVM IR, there's no way to # get emcc to output wasm .s files, which is what we archive in compiler_rt. commands.append([ shared.CLANG_CC, '--target=wasm32', '-mthread-model', 'single', '-S', shared.path_from_root('system', 'lib', src), '-O2', '-o', o ] + musl_internal_includes() + shared.EMSDK_OPTS) o_s.append(o) run_commands(commands) lib = in_temp(libname) run_commands([[shared.LLVM_AR, 'cr', '-format=gnu', lib] + o_s]) return lib
def build_libcxx(src_dirname, lib_filename, files, lib_opts, has_noexcept_version=False): o_s = [] commands = [] opts = default_opts + lib_opts if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING: opts += ['-fno-exceptions'] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + opts) o_s.append(o) run_commands(commands) if lib_filename.endswith('.bc'): shared.Building.link(o_s, in_temp(lib_filename)) elif lib_filename.endswith('.a'): shared.Building.emar('cr', in_temp(lib_filename), o_s) else: raise Exception('unknown suffix ' + lib_filename) return in_temp(lib_filename)
def create_libcxx(libname): logging.debug('building libcxx for cache') libcxx_files = [ 'algorithm.cpp', 'any.cpp', 'bind.cpp', 'chrono.cpp', 'condition_variable.cpp', 'debug.cpp', 'exception.cpp', 'future.cpp', 'hash.cpp', 'ios.cpp', 'iostream.cpp', 'locale.cpp', 'memory.cpp', 'mutex.cpp', 'new.cpp', 'optional.cpp', 'random.cpp', 'regex.cpp', 'shared_mutex.cpp', 'stdexcept.cpp', 'string.cpp', 'strstream.cpp', 'system_error.cpp', 'thread.cpp', 'typeinfo.cpp', 'utility.cpp', 'valarray.cpp' ] return build_libcxx( os.path.join('system', 'lib', 'libcxx'), libname, libcxx_files, [ '-DLIBCXX_BUILDING_LIBCXXABI=1', '-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include') ], has_noexcept_version=True)
def create_libcxxabi(): logging.debug('building libcxxabi for cache') libcxxabi_files = [ 'abort_message.cpp', 'cxa_aux_runtime.cpp', 'cxa_default_handlers.cpp', 'cxa_demangle.cpp', 'cxa_exception_storage.cpp', 'cxa_new_delete.cpp', 'cxa_handlers.cpp', 'exception.cpp', 'stdexcept.cpp', 'typeinfo.cpp', 'private_typeinfo.cpp', os.path.join('..', '..', 'libcxx', 'new.cpp'), ] return build_libcxx( os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc', libcxxabi_files, [ '-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include') ])
def create_libc(libname): logging.warn(' building libc for cache') libc_files = [ ] musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src') blacklist = set( ['ipc', 'passwd', 'getopt_long.c', 'getopt.c', 'raise.c', 'sigsuspend.c', 'sigtimedwait.c', 'getitimer.c', 'sigpending.c', 'sigqueue.c', 'sigaltstack.c', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules ['fexecve.c', 'posix_spawnattr_destroy.c', 'posix_spawnattr_getflags.c', 'posix_spawnattr_getpgroup.c', 'posix_spawnattr_getsigdefault.c', 'posix_spawnattr_getsigmask.c', 'posix_spawnattr_init.c', 'posix_spawnattr_sched.c', 'posix_spawnattr_setflags.c', 'posix_spawnattr_setpgroup.c', 'posix_spawnattr_setsigdefault.c', 'posix_spawnattr_setsigmask.c', 'posix_spawn.c', 'posix_spawn_file_actions_addclose.c', 'posix_spawn_file_actions_adddup2.c', 'posix_spawn_file_actions_addopen.c', 'posix_spawn_file_actions_destroy.c', 'posix_spawn_file_actions_init.c', 'posix_spawnp.c', 'system.c', 'vfork.c', 'memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c', 'waitid.c'] + # individual files ['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files ) # applied after blacklist whitelist = set(['wait.c', 'wait3.c', 'wait4.c', 'pthread_sigmask.c']) # TODO: consider using more math code from musl, doing so makes box2d faster for dirpath, dirnames, filenames in os.walk(musl_srcdir): for f in filenames: if f.endswith('.c'): if f in blacklist: continue dir_parts = os.path.split(dirpath) cancel = False for part in dir_parts: if part == 'thread': cancel = True break if part in blacklist and (not f.endswith('pthread_atfork.c')): cancel = True break if f in whitelist or not cancel: libc_files.append(os.path.join(musl_srcdir, dirpath, f)) args = ['-Os', '-g4'] if shared.Settings.USE_PTHREADS: args += ['-s', 'USE_PTHREADS=1'] assert '-mt' in libname else: assert '-mt' not in libname return build_libc(libname, libc_files, args)
while 1: print 'Tried %d, notes: %s' % (tried, notes) print '1) Generate C' shared.execute([CSMITH, '--no-volatiles', '--no-math64', '--no-packed-struct'],# + #['--max-block-depth', '2', '--max-block-size', '2', '--max-expr-complexity', '2', '--max-funcs', '2'], stdout=open(filename + '.c', 'w')) #shutil.copyfile(filename + '.c', 'testcase%d.c' % tried) print '1) Generate C... %.2f K of C source' % (len(open(filename + '.c').read())/1024.) tried += 1 print '2) Compile natively' shared.try_delete(filename) shared.execute([shared.CLANG_CC, '-O2', filename + '.c', '-o', filename + '1'] + CSMITH_CFLAGS, stderr=PIPE) # + shared.EMSDK_OPTS shared.execute([shared.CLANG_CC, '-O2', '-emit-llvm', '-c', '-Xclang', '-triple=i386-pc-linux-gnu', filename + '.c', '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS, stderr=PIPE) shared.execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc'], stdout=PIPE, stderr=PIPE) shutil.move(filename + '.bc.run', filename + '2') shared.execute([shared.CLANG_CC, filename + '.c', '-o', filename + '3'] + CSMITH_CFLAGS, stderr=PIPE) print '3) Run natively' try: correct1 = shared.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3) if 'Segmentation fault' in correct1 or len(correct1) < 10: raise Exception('segfault') correct2 = shared.timeout_run(Popen([filename + '2'], stdout=PIPE, stderr=PIPE), 3) if 'Segmentation fault' in correct2 or len(correct2) < 10: raise Exception('segfault') correct3 = shared.timeout_run(Popen([filename + '3'], stdout=PIPE, stderr=PIPE), 3) if 'Segmentation fault' in correct3 or len(correct3) < 10: raise Exception('segfault') if correct1 != correct3: raise Exception('clang opts change result') except Exception, e: print 'Failed or infinite looping in native, skipping', e notes['invalid'] += 1 continue
if seen.get(name): return True seen[name] = 1 return False data_files = filter(lambda file_: not was_seen(file_['dstpath']), data_files) if AV_WORKAROUND: random.shuffle(data_files) # Apply plugins for file_ in data_files: for plugin in plugins: plugin(file_) # Crunch files if crunch: shutil.copyfile(shared.path_from_root('tools', 'crunch-worker.js'), 'crunch-worker.js') ret += ''' var decrunchWorker = new Worker('crunch-worker.js'); var decrunchCallbacks = []; decrunchWorker.onmessage = function(msg) { decrunchCallbacks[msg.data.callbackID](msg.data.data); console.log('decrunched ' + msg.data.filename + ' in ' + msg.data.time + ' ms, ' + msg.data.data.length + ' bytes'); decrunchCallbacks[msg.data.callbackID] = null; }; function requestDecrunch(filename, data, callback) { decrunchWorker.postMessage({ filename: filename, data: new Uint8Array(data), callbackID: decrunchCallbacks.length }); decrunchCallbacks.push(callback);
def create_optimizer_cmake(): shared.logging.debug('building native optimizer via CMake: ' + name) output = shared.Cache.get_path(name) shared.try_delete(output) if NATIVE_OPTIMIZER == '1': cmake_build_type = 'RelWithDebInfo' elif NATIVE_OPTIMIZER == '2': cmake_build_type = 'Release' elif NATIVE_OPTIMIZER == 'g': cmake_build_type = 'Debug' build_path = shared.Cache.get_path('optimizer_build_' + cmake_build_type) shared.try_delete(os.path.join(build_path, 'CMakeCache.txt')) log_output = None if DEBUG else subprocess.PIPE if not os.path.exists(build_path): os.mkdir(build_path) if WINDOWS: # Poor man's check for whether or not we should attempt 64 bit build if os.environ.get('ProgramFiles(x86)'): cmake_generators = ['Visual Studio 12 Win64', 'Visual Studio 12', 'Visual Studio 11 Win64', 'Visual Studio 11', 'MinGW Makefiles', 'Unix Makefiles'] else: cmake_generators = ['Visual Studio 12', 'Visual Studio 11', 'MinGW Makefiles', 'Unix Makefiles'] else: cmake_generators = ['Unix Makefiles'] for cmake_generator in cmake_generators: proc = subprocess.Popen(['cmake', '-G', cmake_generator, '-DCMAKE_BUILD_TYPE='+cmake_build_type, shared.path_from_root('tools', 'optimizer')], cwd=build_path, stdin=log_output, stdout=log_output, stderr=log_output) proc.communicate() make_env = os.environ.copy() if proc.returncode == 0: if 'Visual Studio' in cmake_generator: ret = find_msbuild(os.path.join(build_path, 'asmjs_optimizer.sln'), make_env) make = [ret[0], '/t:Build', '/p:Configuration='+cmake_build_type, '/nologo', '/verbosity:minimal', 'asmjs_optimizer.sln'] make_env = ret[1] elif 'MinGW' in cmake_generator: make = ['mingw32-make'] else: make = ['make'] proc = subprocess.Popen(make, cwd=build_path, stdin=log_output, stdout=log_output, stderr=log_output, env=make_env) proc.communicate() if proc.returncode == 0: if WINDOWS and 'Visual Studio' in cmake_generator: shutil.copyfile(os.path.join(build_path, cmake_build_type, 'optimizer.exe'), output) else: shutil.copyfile(os.path.join(build_path, 'optimizer'), output) return output else: shared.try_delete(os.path.join(build_path, 'CMakeCache.txt')) # Proceed to next iteration of the loop to try next possible CMake generator. raise NativeOptimizerCreationException()
import os, sys from subprocess import Popen, PIPE, STDOUT import shared print 'Building zlib' emscripten_temp_dir = shared.get_emscripten_temp_dir() zlib = shared.Building.build_library('zlib', emscripten_temp_dir, emscripten_temp_dir, ['libz.a'], make_args=['libz.a'], copy_project=True, source_dir=shared.path_from_root('tests', 'zlib'))[0] print 'Building minigzip' Popen([shared.PYTHON, shared.EMCC, '-O2', shared.path_from_root('tests', 'zlib', 'minigzip.c'), zlib, '-o', shared.path_from_root('tools', 'minigzip.js')]).communicate()
def create_dlmalloc(out_name, clflags): o = in_temp(out_name) check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o] + clflags) return o
stdout=open(fullname, 'w')) print '1) Generate source... %.2f K' % (len(open(fullname).read())/1024.) tried += 1 print '2) Compile natively' shared.try_delete(filename) try: shared.check_execute([COMP, '-m32', opts, fullname, '-o', filename + '1'] + CSMITH_CFLAGS + ['-w']) # + shared.EMSDK_OPTS except Exception, e: print 'Failed to compile natively using clang' notes['invalid'] += 1 continue shared.check_execute([COMP, '-m32', opts, '-emit-llvm', '-c', fullname, '-o', filename + '.bc'] + CSMITH_CFLAGS + shared.EMSDK_OPTS) shared.check_execute([shared.path_from_root('tools', 'nativize_llvm.py'), filename + '.bc']) shutil.move(filename + '.bc.run', filename + '2') shared.check_execute([COMP, fullname, '-o', filename + '3'] + CSMITH_CFLAGS) print '3) Run natively' try: correct1 = shared.jsrun.timeout_run(Popen([filename + '1'], stdout=PIPE, stderr=PIPE), 3) if 'Segmentation fault' in correct1 or len(correct1) < 10: raise Exception('segfault') correct2 = shared.jsrun.timeout_run(Popen([filename + '2'], stdout=PIPE, stderr=PIPE), 3) if 'Segmentation fault' in correct2 or len(correct2) < 10: raise Exception('segfault') correct3 = shared.jsrun.timeout_run(Popen([filename + '3'], stdout=PIPE, stderr=PIPE), 3) if 'Segmentation fault' in correct3 or len(correct3) < 10: raise Exception('segfault') if correct1 != correct3: raise Exception('clang opts change result') except Exception, e: print 'Failed or infinite looping in native, skipping', e notes['invalid'] += 1 continue
def calculate(temp_files, in_temp, stdout_, stderr_, forced=[]): global stdout, stderr stdout = stdout_ stderr = stderr_ # Check if we need to include some libraries that we compile. (We implement libc ourselves in js, but # compile a malloc implementation and stdlibc++.) def read_symbols(path, exclude=None): symbols = map(lambda line: line.strip().split(' ')[1], open(path).readlines()) if exclude: symbols = filter(lambda symbol: symbol not in exclude, symbols) return set(symbols) default_opts = [] # XXX We also need to add libc symbols that use malloc, for example strdup. It's very rare to use just them and not # a normal malloc symbol (like free, after calling strdup), so we haven't hit this yet, but it is possible. libc_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libc.symbols')) libcxx_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxx', 'symbols'), exclude=libc_symbols) libcxxabi_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxxabi', 'symbols'), exclude=libc_symbols) gl_symbols = read_symbols(shared.path_from_root('system', 'lib', 'gl.symbols')) compiler_rt_symbols = read_symbols(shared.path_from_root('system', 'lib', 'compiler-rt.symbols')) pthreads_symbols = read_symbols(shared.path_from_root('system', 'lib', 'pthreads.symbols')) # XXX we should disable EMCC_DEBUG when building libs, just like in the relooper def build_libc(lib_filename, files, lib_opts): o_s = [] musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')] commands = [] # Hide several musl warnings that produce a lot of spam to unit test build server logs. # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up. c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility', '-Wno-pointer-sign'] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename) def build_libcxx(src_dirname, lib_filename, files, lib_opts, has_noexcept_version=False): o_s = [] commands = [] opts = default_opts + lib_opts if has_noexcept_version and shared.Settings.DISABLE_EXCEPTION_CATCHING: opts += ['-fno-exceptions'] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + opts) o_s.append(o) run_commands(commands) if lib_filename.endswith('.bc'): shared.Building.link(o_s, in_temp(lib_filename)) elif lib_filename.endswith('.a'): shared.Building.emar('cr', in_temp(lib_filename), o_s) else: raise Exception('unknown suffix ' + lib_filename) return in_temp(lib_filename) # libc def create_libc(libname): logging.debug(' building libc for cache') libc_files = [ ] musl_srcdir = shared.path_from_root('system', 'lib', 'libc', 'musl', 'src') blacklist = set( ['ipc', 'passwd', 'thread', 'signal', 'sched', 'ipc', 'time', 'linux', 'aio', 'exit', 'legacy', 'mq', 'process', 'search', 'setjmp', 'env', 'ldso', 'conf'] + # musl modules ['memcpy.c', 'memset.c', 'memmove.c', 'getaddrinfo.c', 'getnameinfo.c', 'inet_addr.c', 'res_query.c', 'gai_strerror.c', 'proto.c', 'gethostbyaddr.c', 'gethostbyaddr_r.c', 'gethostbyname.c', 'gethostbyname2_r.c', 'gethostbyname_r.c', 'gethostbyname2.c', 'usleep.c', 'alarm.c', 'syscall.c'] + # individual files ['abs.c', 'cos.c', 'cosf.c', 'cosl.c', 'sin.c', 'sinf.c', 'sinl.c', 'tan.c', 'tanf.c', 'tanl.c', 'acos.c', 'acosf.c', 'acosl.c', 'asin.c', 'asinf.c', 'asinl.c', 'atan.c', 'atanf.c', 'atanl.c', 'atan2.c', 'atan2f.c', 'atan2l.c', 'exp.c', 'expf.c', 'expl.c', 'log.c', 'logf.c', 'logl.c', 'sqrt.c', 'sqrtf.c', 'sqrtl.c', 'fabs.c', 'fabsf.c', 'fabsl.c', 'ceil.c', 'ceilf.c', 'ceill.c', 'floor.c', 'floorf.c', 'floorl.c', 'pow.c', 'powf.c', 'powl.c', 'round.c', 'roundf.c'] # individual math files ) # TODO: consider using more math code from musl, doing so makes box2d faster for dirpath, dirnames, filenames in os.walk(musl_srcdir): for f in filenames: if f.endswith('.c'): if f in blacklist: continue dir_parts = os.path.split(dirpath) cancel = False for part in dir_parts: if part in blacklist: cancel = True break if not cancel: libc_files.append(os.path.join(musl_srcdir, dirpath, f)) args = ['-Os'] if shared.Settings.USE_PTHREADS: args += ['-s', 'USE_PTHREADS=1'] assert '-mt' in libname else: assert '-mt' not in libname return build_libc(libname, libc_files, args) def create_pthreads(libname): # Add pthread files. pthreads_files = [os.path.join('pthread', 'library_pthread.c')] pthreads_files += [shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'thread', x) for x in ('pthread_attr_destroy.c', 'pthread_condattr_setpshared.c', 'pthread_mutex_lock.c', 'pthread_spin_destroy.c', 'pthread_attr_get.c', 'pthread_cond_broadcast.c', 'pthread_mutex_setprioceiling.c', 'pthread_spin_init.c', 'pthread_attr_init.c', 'pthread_cond_destroy.c', 'pthread_mutex_timedlock.c', 'pthread_spin_lock.c', 'pthread_attr_setdetachstate.c', 'pthread_cond_init.c', 'pthread_mutex_trylock.c', 'pthread_spin_trylock.c', 'pthread_attr_setguardsize.c', 'pthread_cond_signal.c', 'pthread_mutex_unlock.c', 'pthread_spin_unlock.c', 'pthread_attr_setinheritsched.c', 'pthread_cond_timedwait.c', 'pthread_once.c', 'sem_destroy.c', 'pthread_attr_setschedparam.c', 'pthread_cond_wait.c', 'pthread_rwlockattr_destroy.c', 'sem_getvalue.c', 'pthread_attr_setschedpolicy.c', 'pthread_equal.c', 'pthread_rwlockattr_init.c', 'sem_init.c', 'pthread_attr_setscope.c', 'pthread_getspecific.c', 'pthread_rwlockattr_setpshared.c', 'sem_open.c', 'pthread_attr_setstack.c', 'pthread_key_create.c', 'pthread_rwlock_destroy.c', 'sem_post.c', 'pthread_attr_setstacksize.c', 'pthread_mutexattr_destroy.c', 'pthread_rwlock_init.c', 'sem_timedwait.c', 'pthread_barrierattr_destroy.c', 'pthread_mutexattr_init.c', 'pthread_rwlock_rdlock.c', 'sem_trywait.c', 'pthread_barrierattr_init.c', 'pthread_mutexattr_setprotocol.c', 'pthread_rwlock_timedrdlock.c', 'sem_unlink.c', 'pthread_barrierattr_setpshared.c', 'pthread_mutexattr_setpshared.c', 'pthread_rwlock_timedwrlock.c', 'sem_wait.c', 'pthread_barrier_destroy.c', 'pthread_mutexattr_setrobust.c', 'pthread_rwlock_tryrdlock.c', '__timedwait.c', 'pthread_barrier_init.c', 'pthread_mutexattr_settype.c', 'pthread_rwlock_trywrlock.c', 'vmlock.c', 'pthread_barrier_wait.c', 'pthread_mutex_consistent.c', 'pthread_rwlock_unlock.c', '__wait.c', 'pthread_condattr_destroy.c', 'pthread_mutex_destroy.c', 'pthread_rwlock_wrlock.c', 'pthread_condattr_init.c', 'pthread_mutex_getprioceiling.c', 'pthread_setcanceltype.c', 'pthread_condattr_setclock.c', 'pthread_mutex_init.c', 'pthread_setspecific.c')] return build_libc(libname, pthreads_files, ['-O2', '-s', 'USE_PTHREADS=1']) # libcxx def create_libcxx(libname): logging.debug('building libcxx for cache') libcxx_files = [ 'algorithm.cpp', 'condition_variable.cpp', 'future.cpp', 'iostream.cpp', 'memory.cpp', 'random.cpp', 'stdexcept.cpp', 'system_error.cpp', 'utility.cpp', 'bind.cpp', 'debug.cpp', 'hash.cpp', 'mutex.cpp', 'string.cpp', 'thread.cpp', 'valarray.cpp', 'chrono.cpp', 'exception.cpp', 'ios.cpp', 'locale.cpp', 'regex.cpp', 'strstream.cpp' ] return build_libcxx(os.path.join('system', 'lib', 'libcxx'), libname, libcxx_files, ['-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')], has_noexcept_version=True) # libcxxabi - just for dynamic_cast for now def create_libcxxabi(libname): logging.debug('building libcxxabi for cache') libcxxabi_files = [ 'abort_message.cpp', 'cxa_aux_runtime.cpp', 'cxa_default_handlers.cpp', 'cxa_demangle.cpp', 'cxa_exception_storage.cpp', 'cxa_new_delete.cpp', 'cxa_handlers.cpp', 'exception.cpp', 'stdexcept.cpp', 'typeinfo.cpp', 'private_typeinfo.cpp', os.path.join('..', '..', 'libcxx', 'new.cpp'), ] return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), libname, libcxxabi_files, ['-Oz', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')]) # gl def create_gl(libname): # libname is ignored, this is just one .o file o = in_temp('gl.o') check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o]) return o def create_compiler_rt(libname): srcdir = shared.path_from_root('system', 'lib', 'compiler-rt') filenames = ['divdc3.c', 'divsc3.c', 'muldc3.c', 'mulsc3.c'] files = (os.path.join(srcdir, f) for f in filenames) o_s = [] commands = [] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-O2', '-o', o]) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(libname)) return in_temp(libname) def create_dlmalloc(out_name, clflags): o = in_temp(out_name) check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', o] + clflags) return o def create_dlmalloc_singlethreaded(libname): return create_dlmalloc(libname, ['-O2']) def create_dlmalloc_singlethreaded_tracing(libname): return create_dlmalloc(libname, ['-O2', '--tracing']) def create_dlmalloc_multithreaded(libname): return create_dlmalloc(libname, ['-O2', '-s', 'USE_PTHREADS=1']) def create_dlmalloc_multithreaded_tracing(libname): return create_dlmalloc(libname, ['-O2', '-s', 'USE_PTHREADS=1', '--tracing']) def create_dlmalloc_split(libname): dlmalloc_o = in_temp('dl' + libname) check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'dlmalloc.c'), '-o', dlmalloc_o, '-O2', '-DMSPACES', '-DONLY_MSPACES']) split_malloc_o = in_temp('sm' + libname) check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'split_malloc.cpp'), '-o', split_malloc_o, '-O2']) lib = in_temp(libname) shared.Building.link([dlmalloc_o, split_malloc_o], lib) return lib # Setting this in the environment will avoid checking dependencies and make building big projects a little faster # 1 means include everything; otherwise it can be the name of a lib (libcxx, etc.) # You can provide 1 to include everything, or a comma-separated list with the ones you want force = os.environ.get('EMCC_FORCE_STDLIBS') force_all = force == '1' force = set((force.split(',') if force else []) + forced) if force: logging.debug('forcing stdlibs: ' + str(force)) # Setting this will only use the forced libs in EMCC_FORCE_STDLIBS. This avoids spending time checking # for unresolved symbols in your project files, which can speed up linking, but if you do not have # the proper list of actually needed libraries, errors can occur. See below for how we must # export all the symbols in deps_info when using this option. only_forced = os.environ.get('EMCC_ONLY_FORCED_STDLIBS') if only_forced: temp_files = [] # Add in some hacks for js libraries. If a js lib depends on a symbol provided by a C library, it must be # added to here, because our deps go only one way (each library here is checked, then we check the next # in order - libcxx, libcxextra, etc. - and then we run the JS compiler and provide extra symbols from # library*.js files. But we cannot then go back to the C libraries if a new dep was added! # TODO: Move all __deps from src/library*.js to deps_info.json, and use that single source of info # both here and in the JS compiler. deps_info = json.loads(open(shared.path_from_root('src', 'deps_info.json')).read()) added = set() def add_back_deps(need): more = False for ident, deps in deps_info.iteritems(): if ident in need.undefs and not ident in added: added.add(ident) more = True for dep in deps: need.undefs.add(dep) shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep) if more: add_back_deps(need) # recurse to get deps of deps # Scan symbols symbolses = map(lambda temp_file: shared.Building.llvm_nm(temp_file), temp_files) if len(symbolses) == 0: class Dummy: defs = set() undefs = set() symbolses.append(Dummy()) # depend on exported functions for export in shared.Settings.EXPORTED_FUNCTIONS: if shared.Settings.VERBOSE: logging.debug('adding dependency on export %s' % export) symbolses[0].undefs.add(export[1:]) for symbols in symbolses: add_back_deps(symbols) # If we are only doing forced stdlibs, then we don't know the actual symbols we need, # and must assume all of deps_info must be exported. Note that this might cause # warnings on exports that do not exist. if only_forced: for key, value in deps_info.iteritems(): for dep in value: shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep) all_needed = set() for symbols in symbolses: all_needed.update(symbols.undefs) for symbols in symbolses: all_needed.difference_update(symbols.defs) system_libs = [('libcxx', 'a', create_libcxx, libcxx_symbols, ['libcxxabi'], True), ('libcxxabi', 'bc', create_libcxxabi, libcxxabi_symbols, ['libc'], False), ('gl', 'bc', create_gl, gl_symbols, ['libc'], False), ('compiler-rt', 'bc', create_compiler_rt, compiler_rt_symbols, ['libc'], False)] # malloc dependency is force-added, so when using pthreads, it must be force-added # as well, since malloc needs to be thread-safe, so it depends on mutexes. if shared.Settings.USE_PTHREADS: system_libs += [('libc-mt', 'bc', create_libc, libc_symbols, [], False), ('pthreads', 'bc', create_pthreads, pthreads_symbols, ['libc'], False), ('dlmalloc_threadsafe', 'bc', create_dlmalloc_multithreaded, [], [], False), ('dlmalloc_threadsafe_tracing', 'bc', create_dlmalloc_multithreaded_tracing, [], [], False)] force.add('pthreads') if shared.Settings.EMSCRIPTEN_TRACING: force.add('dlmalloc_threadsafe_tracing') else: force.add('dlmalloc_threadsafe') else: system_libs += [('libc', 'bc', create_libc, libc_symbols, [], False)] if shared.Settings.EMSCRIPTEN_TRACING: system_libs += [('dlmalloc_tracing', 'bc', create_dlmalloc_singlethreaded_tracing, [], [], False)] force.add('dlmalloc_tracing') else: if shared.Settings.SPLIT_MEMORY: system_libs += [('dlmalloc_split', 'bc', create_dlmalloc_split, [], [], False)] force.add('dlmalloc_split') else: system_libs += [('dlmalloc', 'bc', create_dlmalloc_singlethreaded, [], [], False)] force.add('dlmalloc') # Go over libraries to figure out which we must include def maybe_noexcept(name): if shared.Settings.DISABLE_EXCEPTION_CATCHING: name += '_noexcept' return name ret = [] has = need = None for shortname, suffix, create, library_symbols, deps, can_noexcept in system_libs: force_this = force_all or shortname in force if can_noexcept: shortname = maybe_noexcept(shortname) if force_this: suffix = 'bc' # .a files do not always link in all their parts; don't use them when forced name = shortname + '.' + suffix if not force_this: need = set() has = set() for symbols in symbolses: if shared.Settings.VERBOSE: logging.debug('undefs: ' + str(symbols.undefs)) for library_symbol in library_symbols: if library_symbol in symbols.undefs: need.add(library_symbol) if library_symbol in symbols.defs: has.add(library_symbol) for haz in has: # remove symbols that are supplied by another of the inputs if haz in need: need.remove(haz) if shared.Settings.VERBOSE: logging.debug('considering %s: we need %s and have %s' % (name, str(need), str(has))) if force_this or (len(need) > 0 and not only_forced): # We need to build and link the library in logging.debug('including %s' % name) def do_create(): ret = create(name) return ret libfile = shared.Cache.get(name, do_create, extension=suffix) ret.append(libfile) force = force.union(deps) ret.sort(key=lambda x: x.endswith('.a')) # make sure to put .a files at the end. for actual in ret: if os.path.basename(actual) == 'libcxxabi.bc': # libcxxabi and libcxx *static* linking is tricky. e.g. cxa_demangle.cpp disables c++ # exceptions, but since the string methods in the headers are *weakly* linked, then # we might have exception-supporting versions of them from elsewhere, and if libcxxabi # is first then it would "win", breaking exception throwing from those string # header methods. To avoid that, we link libcxxabi last. ret = filter(lambda f: f != actual, ret) + [actual] return ret
def create_libcxxabi(): logging.debug('building libcxxabi for cache') libcxxabi_files = [ 'abort_message.cpp', 'cxa_aux_runtime.cpp', 'cxa_default_handlers.cpp', 'cxa_demangle.cpp', 'cxa_exception_storage.cpp', 'cxa_new_delete.cpp', 'cxa_handlers.cpp', 'exception.cpp', 'stdexcept.cpp', 'typeinfo.cpp', 'private_typeinfo.cpp', os.path.join('..', '..', 'libcxx', 'new.cpp'), ] return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc', libcxxabi_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
def create_libcxx(): logging.debug('building libcxx for cache') libcxx_files = [ 'algorithm.cpp', 'condition_variable.cpp', 'future.cpp', 'iostream.cpp', 'memory.cpp', 'random.cpp', 'stdexcept.cpp', 'system_error.cpp', 'utility.cpp', 'bind.cpp', 'debug.cpp', 'hash.cpp', 'mutex.cpp', 'string.cpp', 'thread.cpp', 'valarray.cpp', 'chrono.cpp', 'exception.cpp', 'ios.cpp', 'locale.cpp', 'regex.cpp', 'strstream.cpp' ] return build_libcxx(os.path.join('system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')])
def calculate(temp_files, in_temp, stdout_, stderr_, forced=[]): global stdout, stderr stdout = stdout_ stderr = stderr_ # Check if we need to include some libraries that we compile. (We implement libc ourselves in js, but # compile a malloc implementation and stdlibc++.) def read_symbols(path, exclude=None): symbols = map(lambda line: line.strip().split(' ')[1], open(path).readlines()) if exclude: symbols = filter(lambda symbol: symbol not in exclude, symbols) return set(symbols) default_opts = [] # If we're building tracing, we should build the system libraries that way too. if shared.Settings.EMSCRIPTEN_TRACING: default_opts.append('--tracing') # XXX We also need to add libc symbols that use malloc, for example strdup. It's very rare to use just them and not # a normal malloc symbol (like free, after calling strdup), so we haven't hit this yet, but it is possible. libc_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libc.symbols')) libcextra_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcextra.symbols')) libcxx_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxx', 'symbols'), exclude=libc_symbols) libcxxabi_symbols = read_symbols(shared.path_from_root('system', 'lib', 'libcxxabi', 'symbols'), exclude=libc_symbols) gl_symbols = read_symbols(shared.path_from_root('system', 'lib', 'gl.symbols')) # XXX we should disable EMCC_DEBUG when building libs, just like in the relooper def build_libc(lib_filename, files, lib_opts): o_s = [] prev_cxx = os.environ.get('EMMAKEN_CXX') if prev_cxx: os.environ['EMMAKEN_CXX'] = '' musl_internal_includes = ['-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'src', 'internal'), '-I', shared.path_from_root('system', 'lib', 'libc', 'musl', 'arch', 'js')] commands = [] # Hide several musl warnings that produce a lot of spam to unit test build server logs. # TODO: When updating musl the next time, feel free to recheck which of their warnings might have been fixed, and which ones of these could be cleaned up. c_opts = ['-Wno-dangling-else', '-Wno-unknown-pragmas', '-Wno-shift-op-parentheses', '-Wno-string-plus-int', '-Wno-logical-op-parentheses', '-Wno-bitwise-op-parentheses', '-Wno-visibility'] for src in files: o = in_temp(os.path.basename(src) + '.o') commands.append([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', src), '-o', o] + musl_internal_includes + default_opts + c_opts + lib_opts) o_s.append(o) run_commands(commands) if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename) def build_libcxx(src_dirname, lib_filename, files, lib_opts): o_s = [] commands = [] for src in files: o = in_temp(src + '.o') srcfile = shared.path_from_root(src_dirname, src) commands.append([shared.PYTHON, shared.EMXX, srcfile, '-o', o, '-std=c++11'] + default_opts + lib_opts) o_s.append(o) run_commands(commands) shared.Building.link(o_s, in_temp(lib_filename)) return in_temp(lib_filename) # libc def create_libc(): logging.debug(' building libc for cache') libc_files = [ 'dlmalloc.c', ] musl_files = [ ['ctype', [ 'isdigit.c', 'isspace.c', 'isupper.c', 'isxdigit.c', 'tolower.c', ]], ['internal', [ 'intscan.c', 'floatscan.c', 'shgetc.c', ]], ['math', [ 'frexp.c', 'frexpf.c', 'frexpl.c', 'scalbn.c', 'scalbnl.c', ]], ['multibyte', [ 'wctomb.c', 'wcrtomb.c', ]], ['prng', [ '__rand48_step.c', '__seed48.c', 'drand48.c', 'lcong48.c', 'lrand48.c', 'mrand48.c', 'rand_r.c', 'rand.c', 'random.c', 'seed48.c', 'srand48.c' ]], ['stdio', [ '__overflow.c', '__toread.c', '__towrite.c', '__uflow.c', 'fwrite.c', 'snprintf.c', 'sprintf.c', 'vfprintf.c', 'vsnprintf.c', 'vsprintf.c', ]], ['stdlib', [ 'atof.c', 'atoi.c', 'atol.c', 'strtod.c', 'strtol.c', ]], ['string', [ 'memchr.c', 'memcmp.c', 'strcasecmp.c', 'strcmp.c', 'strncasecmp.c', 'strncmp.c', ]] ] for directory, sources in musl_files: libc_files += [os.path.join('libc', 'musl', 'src', directory, source) for source in sources] return build_libc('libc.bc', libc_files, ['-O2']) def apply_libc(need): # libc needs some sign correction. # If we are in mode 0, switch to 2. We will add our lines try: if shared.Settings.CORRECT_SIGNS == 0: raise Exception('we need to change to 2') except: # we fail if equal to 0 - so we need to switch to 2 - or if CORRECT_SIGNS is not even in Settings shared.Settings.CORRECT_SIGNS = 2 if shared.Settings.CORRECT_SIGNS == 2: shared.Settings.CORRECT_SIGNS_LINES = [shared.path_from_root('src', 'dlmalloc.c') + ':' + str(i+4) for i in [4816, 4191, 4246, 4199, 4205, 4235, 4227]] # If we are in mode 1, we are correcting everything anyhow. If we are in mode 3, we will be corrected # so all is well anyhow too. return True # libcextra def create_libcextra(): logging.debug('building libcextra for cache') musl_files = [ ['compat', [ 'strlwr.c', 'strtol_l.c', 'strupr.c' ]], ['ctype', [ 'isalnum.c', 'isalpha.c', 'isascii.c', 'isblank.c', 'iscntrl.c', 'isgraph.c', 'islower.c', 'isprint.c', 'ispunct.c', 'iswalnum.c', 'iswalpha.c', 'iswblank.c', 'iswcntrl.c', 'iswctype.c', 'iswdigit.c', 'iswgraph.c', 'iswlower.c', 'iswprint.c', 'iswpunct.c', 'iswspace.c', 'iswupper.c', 'iswxdigit.c', 'toascii.c', 'toupper.c', 'towctrans.c', 'wcswidth.c', 'wctrans.c', 'wcwidth.c', ]], ['legacy', [ 'err.c', ]], ['locale', [ 'iconv.c', 'isalnum_l.c', 'isalpha_l.c', 'isblank_l.c', 'iscntrl_l.c', 'isdigit_l.c', 'isgraph_l.c', 'islower_l.c', 'isprint_l.c', 'ispunct_l.c', 'isspace_l.c', 'isupper_l.c', 'isxdigit_l.c', 'iswalnum_l.c', 'iswalpha_l.c', 'iswblank_l.c', 'iswcntrl_l.c', 'iswctype_l.c', 'iswdigit_l.c', 'iswgraph_l.c', 'iswlower_l.c', 'iswprint_l.c', 'iswpunct_l.c', 'iswspace_l.c', 'iswupper_l.c', 'iswxdigit_l.c', 'strcoll.c', 'strcasecmp_l.c', 'strfmon.c', 'strncasecmp_l.c', 'strxfrm.c', 'tolower_l.c', 'toupper_l.c', 'towctrans_l.c', 'towlower_l.c', 'towupper_l.c', 'wcscoll.c', 'wcscoll_l.c', 'wcsxfrm.c', 'wcsxfrm_l.c', 'wctrans_l.c', 'wctype_l.c', ]], ['math', [ '__cos.c', '__cosdf.c', '__sin.c', '__sindf.c', 'ilogb.c', 'ilogbf.c', 'ilogbl.c', 'j0.c', 'j0f.c', 'j1.c', 'j1f.c', 'jn.c', 'jnf.c', 'ldexp.c', 'ldexpf.c', 'ldexpl.c', 'logb.c', 'logbf.c', 'logbl.c', 'lgamma.c', 'lgamma_r.c', 'lgammaf.c', 'lgammaf_r.c', 'lgammal.c', 'scalbnf.c', 'signgam.c', 'tgamma.c', 'tgammaf.c', 'tgammal.c' ]], ['misc', [ 'ffs.c', 'getopt.c', 'getopt_long.c', ]], ['multibyte', [ 'btowc.c', 'internal.c', 'mblen.c', 'mbrlen.c', 'mbrtowc.c', 'mbsinit.c', 'mbsnrtowcs.c', 'mbsrtowcs.c', 'mbstowcs.c', 'mbtowc.c', 'wcsnrtombs.c', 'wcsrtombs.c', 'wcstombs.c', 'wctob.c', ]], ['regex', [ 'fnmatch.c', 'glob.c', 'regcomp.c', 'regerror.c', 'regexec.c', 'tre-mem.c', ]], ['stdio', [ '__string_read.c', 'asprintf.c', 'fwprintf.c', 'swprintf.c', 'vfwprintf.c', 'vswprintf.c', 'vwprintf.c', 'wprintf.c', 'fputwc.c', 'fputws.c', 'sscanf.c', 'vasprintf.c', 'vfscanf.c', 'vsscanf.c', ]], ['stdlib', [ 'atoll.c', 'bsearch.c', 'ecvt.c', 'fcvt.c', 'gcvt.c', 'qsort.c', 'wcstod.c', 'wcstol.c', ]], ['string', [ 'bcmp.c', 'bcopy.c', 'bzero.c', 'index.c', 'memccpy.c', 'memmem.c', 'mempcpy.c', 'memrchr.c', 'rindex.c', 'stpcpy.c', 'strcasestr.c', 'strchr.c', 'strchrnul.c', 'strcspn.c', 'strdup.c', 'strlcat.c', 'strlcpy.c', 'strncat.c', 'strndup.c', 'strnlen.c', 'strpbrk.c', 'strrchr.c', 'strsep.c', 'strsignal.c', 'strspn.c', 'strstr.c', 'strtok.c', 'strtok_r.c', 'strverscmp.c', 'wcpcpy.c', 'wcpncpy.c', 'wcscasecmp.c', 'wcscasecmp_l.c', 'wcscat.c', 'wcschr.c', 'wcscmp.c', 'wcscpy.c', 'wcscspn.c', 'wcsdup.c', 'wcslen.c', 'wcsncasecmp.c', 'wcsncasecmp_l.c', 'wcsncat.c', 'wcsncmp.c', 'wcsncpy.c', 'wcsnlen.c', 'wcspbrk.c', 'wcsrchr.c', 'wcsspn.c', 'wcsstr.c', 'wcstok.c', 'wcswcs.c', 'wmemchr.c', 'wmemcmp.c', 'wmemcpy.c', 'wmemmove.c', 'wmemset.c', ]] ] libcextra_files = [] for directory, sources in musl_files: libcextra_files += [os.path.join('libc', 'musl', 'src', directory, source) for source in sources] return build_libc('libcextra.bc', libcextra_files, ['-O2']) # libcxx def create_libcxx(): logging.debug('building libcxx for cache') libcxx_files = [ 'algorithm.cpp', 'condition_variable.cpp', 'future.cpp', 'iostream.cpp', 'memory.cpp', 'random.cpp', 'stdexcept.cpp', 'system_error.cpp', 'utility.cpp', 'bind.cpp', 'debug.cpp', 'hash.cpp', 'mutex.cpp', 'string.cpp', 'thread.cpp', 'valarray.cpp', 'chrono.cpp', 'exception.cpp', 'ios.cpp', 'locale.cpp', 'regex.cpp', 'strstream.cpp' ] return build_libcxx(os.path.join('system', 'lib', 'libcxx'), 'libcxx.bc', libcxx_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')]) def apply_libcxx(need): assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++ with QUANTUM_SIZE == 1' # libcxx might need corrections, so turn them all on. TODO: check which are actually needed shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1 #logging.info('using libcxx turns on CORRECT_* options') return True # libcxxabi - just for dynamic_cast for now def create_libcxxabi(): logging.debug('building libcxxabi for cache') libcxxabi_files = [ 'abort_message.cpp', 'cxa_aux_runtime.cpp', 'cxa_default_handlers.cpp', 'cxa_demangle.cpp', 'cxa_exception_storage.cpp', 'cxa_new_delete.cpp', 'cxa_handlers.cpp', 'exception.cpp', 'stdexcept.cpp', 'typeinfo.cpp', 'private_typeinfo.cpp', os.path.join('..', '..', 'libcxx', 'new.cpp'), ] return build_libcxx(os.path.join('system', 'lib', 'libcxxabi', 'src'), 'libcxxabi.bc', libcxxabi_files, ['-Oz', '-Wno-warn-absolute-paths', '-I' + shared.path_from_root('system', 'lib', 'libcxxabi', 'include')]) def apply_libcxxabi(need): assert shared.Settings.QUANTUM_SIZE == 4, 'We do not support libc++abi with QUANTUM_SIZE == 1' #logging.info('using libcxxabi, this may need CORRECT_* options') #shared.Settings.CORRECT_SIGNS = shared.Settings.CORRECT_OVERFLOWS = shared.Settings.CORRECT_ROUNDINGS = 1 return True # gl def create_gl(): prev_cxx = os.environ.get('EMMAKEN_CXX') if prev_cxx: os.environ['EMMAKEN_CXX'] = '' o = in_temp('gl.o') check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o]) if prev_cxx: os.environ['EMMAKEN_CXX'] = prev_cxx return o # Setting this in the environment will avoid checking dependencies and make building big projects a little faster # 1 means include everything; otherwise it can be the name of a lib (libcxx, etc.) # You can provide 1 to include everything, or a comma-separated list with the ones you want force = os.environ.get('EMCC_FORCE_STDLIBS') force_all = force == '1' force = set((force.split(',') if force else []) + forced) if force: logging.debug('forcing stdlibs: ' + str(force)) # Setting this will only use the forced libs in EMCC_FORCE_STDLIBS. This avoids spending time checking # for unresolved symbols in your project files, which can speed up linking, but if you do not have # the proper list of actually needed libraries, errors can occur. See below for how we must # export all the symbols in deps_info when using this option. only_forced = os.environ.get('EMCC_ONLY_FORCED_STDLIBS') if only_forced: temp_files = [] # Add in some hacks for js libraries. If a js lib depends on a symbol provided by a C library, it must be # added to here, because our deps go only one way (each library here is checked, then we check the next # in order - libcxx, libcxextra, etc. - and then we run the JS compiler and provide extra symbols from # library*.js files. But we cannot then go back to the C libraries if a new dep was added! # TODO: Move all __deps from src/library*.js to deps_info.json, and use that single source of info # both here and in the JS compiler. deps_info = json.loads(open(shared.path_from_root('src', 'deps_info.json')).read()) added = set() def add_back_deps(need): more = False for ident, deps in deps_info.iteritems(): if ident in need.undefs and not ident in added: added.add(ident) more = True for dep in deps: need.undefs.add(dep) shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep) if more: add_back_deps(need) # recurse to get deps of deps # Scan symbols symbolses = map(lambda temp_file: shared.Building.llvm_nm(temp_file), temp_files) if len(symbolses) == 0: class Dummy: defs = set() undefs = set() symbolses.append(Dummy()) # depend on exported functions for export in shared.Settings.EXPORTED_FUNCTIONS: if shared.Settings.VERBOSE: logging.debug('adding dependency on export %s' % export) symbolses[0].undefs.add(export[1:]) for symbols in symbolses: add_back_deps(symbols) # If we are only doing forced stdlibs, then we don't know the actual symbols we need, # and must assume all of deps_info must be exported. Note that this might cause # warnings on exports that do not exist. if only_forced: for key, value in deps_info.iteritems(): for dep in value: shared.Settings.EXPORTED_FUNCTIONS.append('_' + dep) all_needed = set() for symbols in symbolses: all_needed.update(symbols.undefs) for symbols in symbolses: all_needed.difference_update(symbols.defs) # Go over libraries to figure out which we must include ret = [] has = need = None for name, create, apply_, library_symbols, deps in [('libcxx', create_libcxx, apply_libcxx, libcxx_symbols, ['libcextra', 'libcxxabi']), ('libcextra', create_libcextra, lambda x: True, libcextra_symbols, ['libc']), ('libcxxabi', create_libcxxabi, apply_libcxxabi, libcxxabi_symbols, ['libc']), ('gl', create_gl, lambda x: True, gl_symbols, ['libc']), ('libc', create_libc, apply_libc, libc_symbols, [])]: force_this = force_all or name in force if not force_this: need = set() has = set() for symbols in symbolses: if shared.Settings.VERBOSE: logging.debug('undefs: ' + str(symbols.undefs)) for library_symbol in library_symbols: if library_symbol in symbols.undefs: need.add(library_symbol) if library_symbol in symbols.defs: has.add(library_symbol) for haz in has: # remove symbols that are supplied by another of the inputs if haz in need: need.remove(haz) if shared.Settings.VERBOSE: logging.debug('considering %s: we need %s and have %s' % (name, str(need), str(has))) if force_this or (len(need) > 0 and not only_forced): if apply_(need): # We need to build and link the library in logging.debug('including %s' % name) libfile = shared.Cache.get(name, create) ret.append(libfile) force = force.union(deps) return ret
def create_gl(): o = in_temp('gl.o') check_call([shared.PYTHON, shared.EMCC, shared.path_from_root('system', 'lib', 'gl.c'), '-o', o]) return o