def CompilerRT():
# TODO(sbc): Figure out how to do this step as part of the llvm build.
# I suspect that this can be done using the llvm/runtimes directory but
# have yet to make it actually work this way.
buildbot.Step('compiler-rt')
# TODO(sbc): Remove this.
# The compiler-rt doesn't currently rebuild libraries when a new -DCMAKE_AR
# value is specified.
if os.path.isdir(COMPILER_RT_OUT_DIR):
Remove(COMPILER_RT_OUT_DIR)
Mkdir(COMPILER_RT_OUT_DIR)
command = [
CMAKE_BIN, '-G', CMAKE_GENERATOR,
WindowsFSEscape(os.path.join(COMPILER_RT_SRC_DIR, 'lib', 'builtins')),
'-DCMAKE_BUILD_TYPE=RelWithDebInfo', '-DCMAKE_TOOLCHAIN_FILE=' +
WindowsFSEscape(os.path.join(HOST_DIR, 'wavix_toolchain.cmake')),
'-DCMAKE_EXPORT_COMPILE_COMMANDS=YES', '-DCMAKE_C_COMPILER_WORKS=YES',
'-DCMAKE_CXX_COMPILER_WORKS=YES', '-DCOMPILER_RT_BAREMETAL_BUILD=On',
'-DCOMPILER_RT_BUILD_XRAY=OFF', '-DCOMPILER_RT_INCLUDE_TESTS=OFF',
'-DCOMPILER_RT_ENABLE_IOS=OFF', '-DCOMPILER_RT_DEFAULT_TARGET_ONLY=On',
'-DLLVM_CONFIG_PATH=' +
WindowsFSEscape(os.path.join(HOST_DIR, 'bin', 'llvm-config')),
'-DCOMPILER_RT_OS_DIR=wavix', '-DCMAKE_INSTALL_PREFIX=' +
WindowsFSEscape(os.path.join(HOST_DIR, 'lib', 'clang', '9.0.0'))
]
proc.check_call(command, cwd=COMPILER_RT_OUT_DIR)
proc.check_call([NINJA_BIN, 'install'], cwd=COMPILER_RT_OUT_DIR)
def HostLLVM():
buildbot.Step('host-llvm')
Mkdir(HOST_LLVM_OUT_DIR)
build_dylib = 'ON' if not IsWindows() else 'OFF'
command = [
CMAKE_BIN,
'-G',
CMAKE_GENERATOR,
LLVM_SRC_DIR,
'-DCMAKE_EXPORT_COMPILE_COMMANDS=YES',
'-DLLVM_EXTERNAL_CLANG_SOURCE_DIR=' + CLANG_SRC_DIR,
'-DLLVM_EXTERNAL_LLD_SOURCE_DIR=' + LLD_SRC_DIR,
'-DLLVM_TOOL_CLANG_BUILD=ON',
'-DLLVM_TOOL_LLD_BUILD=ON',
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DLLVM_BUILD_TESTS=OFF',
'-DCMAKE_BUILD_TYPE=Release', #'-DCMAKE_BUILD_TYPE=RelWithDebInfo',
'-DCMAKE_INSTALL_PREFIX=' + HOST_DIR,
'-DCLANG_INCLUDE_TESTS=OFF',
'-DCLANG_INCLUDE_DOCS=OFF',
'-DLLVM_INCLUDE_EXAMPLES=OFF',
'-DLLVM_INCLUDE_DOCS=OFF',
'-DLLVM_INCLUDE_GO_TESTS=OFF',
'-DLLVM_INCLUDE_TESTS=OFF',
'-DLLVM_BUILD_LLVM_DYLIB=%s' % build_dylib,
'-DLLVM_LINK_LLVM_DYLIB=%s' % build_dylib,
# Our mac bot's toolchain's ld64 is too old for trunk libLTO.
'-DLLVM_TOOL_LTO_BUILD=OFF',
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
'-DLLVM_TARGETS_TO_BUILD=X86'
]
proc.check_call(command, cwd=HOST_LLVM_OUT_DIR)
proc.check_call([NINJA_BIN, 'install'], cwd=HOST_LLVM_OUT_DIR)
def LibCXXABI():
buildbot.Step('libcxxabi')
# TODO(sbc): Remove this.
# The compiler-rt doesn't currently rebuild libraries when a new -DCMAKE_AR
# value is specified.
if os.path.isdir(LIBCXXABI_OUT_DIR):
Remove(LIBCXXABI_OUT_DIR)
Mkdir(LIBCXXABI_OUT_DIR)
command = [
CMAKE_BIN, '-G', CMAKE_GENERATOR, LIBCXXABI_SRC_DIR,
'-DCMAKE_BUILD_TYPE=RelWithDebInfo', '-DCMAKE_TOOLCHAIN_FILE=' +
WindowsFSEscape(os.path.join(HOST_DIR, 'wavix_toolchain.cmake')),
'-DCMAKE_EXPORT_COMPILE_COMMANDS=YES', '-DCMAKE_C_COMPILER_WORKS=YES',
'-DCMAKE_CXX_COMPILER_WORKS=YES', '-DLIBCXXABI_LIBCXX_PATH=' +
LIBCXX_SRC_DIR, '-DLIBCXXABI_LIBCXX_INCLUDES=' +
os.path.join(LIBCXX_SRC_DIR, 'include'),
'-DLIBCXXABI_ENABLE_STATIC=ON', '-DLIBCXXABI_ENABLE_SHARED=OFF',
'-DLIBCXXABI_ENABLE_THREADS=ON', '-DLIBCXXABI_SYSROOT=' + SYSROOT_DIR,
'-DLIBCXXABI_USE_COMPILER_RT=ON', '-DLIBCXXABI_ENABLE_PIC=OFF',
'-DCMAKE_INSTALL_PREFIX=' + WindowsFSEscape(SYSROOT_DIR)
]
proc.check_call(command, cwd=LIBCXXABI_OUT_DIR)
proc.check_call([NINJA_BIN, 'install'], cwd=LIBCXXABI_OUT_DIR)
CopyLibraryToSysroot(SYSROOT_DIR,
os.path.join(LIBCXXABI_SRC_DIR, 'libc++abi.imports'))
def DebianPackage():
is_linux = sys.platform.startswith('linux')
if not (is_linux and IsBuildbot()):
return
buildbot.Step('Debian package')
top_dir = os.path.dirname(SCRIPT_DIR)
try:
if BUILDBOT_BUILDNUMBER:
message = ('Automatic build %s produced on http://wasm-stat.us' %
BUILDBOT_BUILDNUMBER)
version = '0.1.' + BUILDBOT_BUILDNUMBER
proc.check_call(['dch', '-D', 'unstable', '-v', version, message],
cwd=top_dir)
proc.check_call(['debuild', '--no-lintian', '-i', '-us', '-uc', '-b'],
cwd=top_dir)
if BUILDBOT_BUILDNUMBER:
Git(['checkout', 'debian/changelog'], cwd=top_dir)
debfile = os.path.join(os.path.dirname(top_dir),
'wasm-toolchain_0.1_amd64.deb')
UploadFile(debfile, os.path.basename(debfile))
except proc.CalledProcessError:
# Note the failure but allow the build to continue.
buildbot.Fail()
return
def Musl():
buildbot.Step('musl')
Mkdir(MUSL_OUT_DIR)
path = os.environ['PATH']
try:
# Build musl directly to wasm object files in an ar library
proc.check_call([
os.path.join(MUSL_SRC_DIR, 'libc.py'), '--clang_dir', HOST_BIN,
'--out',
os.path.join(MUSL_OUT_DIR, 'libc.a'), '--musl', MUSL_SRC_DIR
])
CopyLibraryToSysroot(SYSROOT_DIR, os.path.join(MUSL_OUT_DIR, 'libc.a'))
CopyLibraryToSysroot(SYSROOT_DIR, os.path.join(MUSL_OUT_DIR, 'crt1.o'))
CopyLibraryToSysroot(
SYSROOT_DIR,
os.path.join(MUSL_SRC_DIR, 'arch', 'wasm32', 'libc.imports'))
CopyTree(os.path.join(MUSL_SRC_DIR, 'include'),
os.path.join(SYSROOT_DIR, 'include'))
CopyTree(os.path.join(MUSL_SRC_DIR, 'arch', 'generic'),
os.path.join(SYSROOT_DIR, 'include'))
CopyTree(os.path.join(MUSL_SRC_DIR, 'arch', 'wasm32'),
os.path.join(SYSROOT_DIR, 'include'))
CopyTree(os.path.join(MUSL_OUT_DIR, 'obj', 'include'),
os.path.join(SYSROOT_DIR, 'include'))
except proc.CalledProcessError:
# Note the failure but allow the build to continue.
buildbot.Fail()
finally:
os.environ['PATH'] = path
def Emscripten():
buildbot.Step('emscripten')
# Remove cached library builds (e.g. libc, libc++) to force them to be
# rebuilt in the step below.
Remove(os.path.expanduser(os.path.join('~', '.emscripten_cache')))
em_config = os.path.join(INSTALL_DIR, 'emscripten_config')
emscripten_dir = os.path.join(INSTALL_DIR, 'bin', 'emscripten')
Remove(emscripten_dir)
shutil.copytree(EMSCRIPTEN_SRC_DIR,
emscripten_dir,
symlinks=True,
# Ignore the big git blob so it doesn't get archived.
ignore=shutil.ignore_patterns('.git'))
shutil.copy2(os.path.join(SCRIPT_DIR, 'emscripten_config_vanilla'),
em_config + '_vanilla')
shutil.copy2(os.path.join(SCRIPT_DIR, 'emscripten_config'), em_config)
try:
# Build a C++ file with each active emscripten config. This causes system
# libs to be built and cached (so we don't have that happen when building
# tests in parallel). Do it with full debug output.
# This depends on binaryen already being built and installed into the
# archive/install dir.
os.environ['EMCC_DEBUG'] = '2'
for config in [em_config, em_config + '_vanilla']:
os.environ['EM_CONFIG'] = config
proc.check_call([
os.path.join(emscripten_dir, 'em++'),
os.path.join(EMSCRIPTEN_SRC_DIR, 'tests', 'hello_world.cpp'),
'-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"'])
del os.environ['EMCC_DEBUG']
except proc.CalledProcessError:
# Don't make it fatal yet.
buildbot.Fail(True)
def Emscripten():
buildbot.Step('emscripten')
em_config = os.path.join(INSTALL_DIR, 'emscripten_config')
emscripten_dir = os.path.join(INSTALL_DIR, 'bin', 'emscripten')
os.environ['EM_CONFIG'] = em_config
Remove(emscripten_dir)
shutil.copytree(EMSCRIPTEN_SRC_DIR,
emscripten_dir,
symlinks=True,
# Ignore the big git blob so it doesn't get archived.
ignore=shutil.ignore_patterns('.git'))
shutil.copy2(os.path.join(SCRIPT_DIR, 'emscripten_config'),
os.path.join(em_config))
try:
proc.check_call([
os.path.join(emscripten_dir, 'emcc'),
os.path.join(EMSCRIPTEN_SRC_DIR, 'tests', 'hello_world.cpp')])
# This test depends on binaryen already being built and installed into the
# archive/install dir. Arguably we shouldn't do that here.
proc.check_call([
os.path.join(emscripten_dir, 'emcc'),
os.path.join(EMSCRIPTEN_SRC_DIR, 'tests', 'hello_world.cpp'),
'-O2', '-s', 'BINARYEN=1'])
except proc.CalledProcessError:
# Don't make it fatal yet.
buildbot.Fail(True)
def GitConfigRebaseMaster(cwd):
"""Avoid generating a non-linear history in the clone
The upstream repository is in Subversion. Use `git pull --rebase` instead of
git pull: llvm.org/docs/GettingStarted.html#git-mirror
"""
proc.check_call(['git', 'config', 'branch.master.rebase', 'true'], cwd=cwd)
def Spec():
buildbot.Step('spec')
# Spec builds in-tree. Always clobber and run the tests.
proc.check_call(['make', 'clean'], cwd=ML_DIR)
proc.check_call(['make', 'all'], cwd=ML_DIR)
wasm = os.path.join(ML_DIR, 'wasm.opt')
CopyBinaryToArchive(wasm)
def CopyDlls(dir, configuration):
"""Copy MSVS Runtime dlls into a build directory"""
proc.check_call([VS_TOOLCHAIN, 'copy_dlls', dir, configuration, 'x64'])
# LLD needs also concrt140.dll, which the Chromium copy_dlls doesn't include.
for dll in glob.glob(os.path.join(GetRuntimeDir(), 'concrt140*.dll')):
print 'Copying %s to %s' % (dll, dir)
shutil.copy2(dll, dir)
def GetRuntimeDir():
# Get the chromium-packaged toolchain directory info in a JSON file
proc.check_call([VS_TOOLCHAIN, 'get_toolchain_dir'])
with open(WIN_TOOLCHAIN_JSON) as f:
paths = json.load(f)
# Extract the 64-bit runtime path
return [path for path in paths['runtime_dirs'] if path.endswith('64')][0]
def Spec():
buildbot.Step('spec')
# Spec builds in-tree. Always clobber and run the tests.
proc.check_call(['make', 'clean'], cwd=ML_DIR)
proc.check_call(['make', 'all'], cwd=ML_DIR)
wasm = os.path.join(ML_DIR, 'wasm.opt')
CopyBinaryToArchive(wasm)
def Upload(local, remote):
"""Upload file to Cloud Storage."""
if not IsUploadingBot():
return
remote = GetCloudStoragePath() + remote
proc.check_call(['gsutil.py', 'cp', local, 'gs://' + remote])
return CLOUD_STORAGE_BASE_URL + remote
def SyncToSameSvnRev(primary, secondary):
"""Use primary's SVN rev to figure out which rev secondary goes to."""
primary_svn_rev = CurrentSvnRev(primary)
print 'SVN REV for %s: %d' % (primary, primary_svn_rev)
print 'Finding prior %s rev' % secondary
prior_rev = FindPriorSvnRev(secondary, primary_svn_rev)
print 'Checking out %s rev: %s' % (secondary, prior_rev)
proc.check_call(['git', 'checkout', prior_rev], cwd=secondary)
def Upload(local, remote):
"""Upload file to Cloud Storage."""
if not os.environ.get('BUILDBOT_BUILDERNAME'):
return
remote = CLOUD_STORAGE_PATH + remote
proc.check_call(
['gsutil.py', 'cp', '-a', 'public-read', local, 'gs://' + remote])
return CLOUD_STORAGE_BASE_URL + remote
def SyncToSameSvnRev(primary, secondary):
"""Use primary's SVN rev to figure out which rev secondary goes to."""
primary_svn_rev = CurrentSvnRev(primary)
print 'SVN REV for %s: %d' % (primary, primary_svn_rev)
print 'Finding prior %s rev' % secondary
prior_rev = FindPriorSvnRev(secondary, primary_svn_rev)
print 'Checking out %s rev: %s' % (secondary, prior_rev)
proc.check_call(['git', 'checkout', prior_rev], cwd=secondary)
def GitConfigRebaseMaster(cwd):
"""Avoid generating a non-linear history in the clone
The upstream repository is in Subversion. Use `git pull --rebase` instead of
git pull: llvm.org/docs/GettingStarted.html#git-mirror
"""
proc.check_call(
['git', 'config', 'branch.master.rebase', 'true'], cwd=cwd)
def Upload(local, remote):
"""Upload file to Cloud Storage."""
if not os.environ.get('BUILDBOT_BUILDERNAME'):
return
remote = CLOUD_STORAGE_PATH + remote
proc.check_call(
['gsutil.py', 'cp', '-a', 'public-read', local, 'gs://' + remote])
return CLOUD_STORAGE_BASE_URL + remote
def CopyDlls(dir, configuration):
"""Copy MSVS Runtime dlls into a build directory"""
file_util.Mkdir(dir)
proc.check_call(VSToolchainPy() + ['copy_dlls', dir, configuration, 'x64'])
# LLD needs also concrt140.dll, which the Chromium copy_dlls doesn't
# include.
for dll in glob.glob(os.path.join(GetRuntimeDir(), 'concrt140*.dll')):
print('Copying %s to %s' % (dll, dir))
shutil.copy2(dll, dir)
def GitUpdateRemote(src_dir, git_repo, remote_name):
try:
proc.check_call(['git', 'remote', 'set-url', remote_name, git_repo],
cwd=src_dir)
except proc.CalledProcessError:
# If proc.check_call fails it throws an exception. 'git remote set-url'
# fails when the remote doesn't exist, so we should try to add it.
proc.check_call(['git', 'remote', 'add', remote_name, git_repo],
cwd=src_dir)
def Copy(copy_from, copy_to):
"""Copy from one Cloud Storage file to another."""
if not os.environ.get('BUILDBOT_BUILDERNAME'):
return
copy_from = CLOUD_STORAGE_PATH + copy_from
copy_to = CLOUD_STORAGE_PATH + copy_to
proc.check_call(
['gsutil.py', 'cp', '-a', 'public-read',
'gs://' + copy_from, 'gs://' + copy_to])
return CLOUD_STORAGE_BASE_URL + copy_to
def V8():
buildbot.Step('V8')
proc.check_call(['ninja', '-C', V8_OUT_DIR, 'd8', 'unittests'],
cwd=V8_SRC_DIR)
proc.check_call(['tools/run-tests.py', 'unittests', '--no-presubmit',
'--shell-dir', V8_OUT_DIR],
cwd=V8_SRC_DIR)
to_archive = ['d8', 'natives_blob.bin', 'snapshot_blob.bin']
for a in to_archive:
CopyBinaryToArchive(os.path.join(V8_OUT_DIR, a))
def ExecuteEmscriptenTestSuite(name, config, outdir, warn_only):
buildbot.Step('Execute emscripten testsuite (%s)' % name)
Mkdir(outdir)
try:
proc.check_call(
[sys.executable,
os.path.join(INSTALL_DIR, 'emscripten', 'tests', 'runner.py'),
'binaryen2', '--em-config', config],
cwd=outdir)
except proc.CalledProcessError:
buildbot.Fail(warn_only)
def Copy(copy_from, copy_to):
"""Copy from one Cloud Storage file to another."""
if not os.environ.get('BUILDBOT_BUILDERNAME'):
return
copy_from = CLOUD_STORAGE_PATH + copy_from
copy_to = CLOUD_STORAGE_PATH + copy_to
proc.check_call([
'gsutil.py', 'cp', '-a', 'public-read', 'gs://' + copy_from,
'gs://' + copy_to
])
return CLOUD_STORAGE_BASE_URL + copy_to
def HostWAVM():
buildbot.Step('HostWAVM')
Mkdir(HOST_WAVM_OUT_DIR)
command = [CMAKE_BIN, '-G', CMAKE_GENERATOR, WAVM_SRC_DIR,
'-DCMAKE_BUILD_TYPE=RelWithDebInfo',
'-DCMAKE_EXPORT_COMPILE_COMMANDS=YES',
'-DCMAKE_INSTALL_PREFIX=' + HOST_DIR,
'-DLLVM_DIR=' + os.path.join(HOST_DIR, 'lib/cmake/llvm') ]
proc.check_call(command, cwd=HOST_WAVM_OUT_DIR)
proc.check_call([NINJA_BIN, 'install'], cwd=HOST_WAVM_OUT_DIR)
def ExecuteEmscriptenTestSuite(name, outdir):
buildbot.Step('Execute emscripten testsuite (emwasm)')
Mkdir(EMSCRIPTEN_TEST_OUT_DIR)
try:
proc.check_call(
[sys.executable,
os.path.join(INSTALL_BIN, 'emscripten', 'tests', 'runner.py'),
'binaryen2', '--em-config', EMSCRIPTEN_CONFIG_WASM],
cwd=outdir)
except proc.CalledProcessError:
buildbot.Fail(True)
def Sexpr():
buildbot.Step('Sexpr')
Mkdir(SEXPR_OUT_DIR),
proc.check_call([PREBUILT_CMAKE_BIN, '-G', 'Ninja', SEXPR_SRC_DIR,
'-DCMAKE_C_COMPILER=%s' % CC,
'-DCMAKE_CXX_COMPILER=%s' % CXX,
'-DBUILD_TESTS=OFF'],
cwd=SEXPR_OUT_DIR)
proc.check_call(['ninja'], cwd=SEXPR_OUT_DIR)
sexpr = os.path.join(SEXPR_OUT_DIR, 'sexpr-wasm')
CopyBinaryToArchive(sexpr)
def Sexpr():
buildbot.Step('Sexpr')
# sexpr-wasm builds in its own in-tree out/ folder. The build is fast, so
# always clobber.
proc.check_call(['make', 'clean'], cwd=SEXPR_SRC_DIR)
proc.check_call(['make',
'CC=%s' % CC,
'CXX=%s' % CXX],
cwd=SEXPR_SRC_DIR)
sexpr = os.path.join(SEXPR_OUT_DIR, 'sexpr-wasm')
CopyBinaryToArchive(sexpr)
def ExecuteEmscriptenTestSuite(name, outdir):
buildbot.Step('Execute emscripten testsuite (emwasm)')
Mkdir(EMSCRIPTEN_TEST_OUT_DIR)
try:
proc.check_call(
[sys.executable,
os.path.join(INSTALL_BIN, 'emscripten', 'tests', 'runner.py'),
'binaryen2', '--em-config', EMSCRIPTEN_CONFIG_WASM],
cwd=outdir)
except proc.CalledProcessError:
buildbot.Fail(True)
def Sexpr():
buildbot.Step('Sexpr')
Mkdir(SEXPR_OUT_DIR),
proc.check_call(['cmake', '-G', 'Ninja', SEXPR_SRC_DIR,
'-DCMAKE_C_COMPILER=%s' % CC,
'-DCMAKE_CXX_COMPILER=%s' % CXX,
'-DBUILD_TESTS=OFF'],
cwd=SEXPR_OUT_DIR)
proc.check_call(['ninja'], cwd=SEXPR_OUT_DIR)
sexpr = os.path.join(SEXPR_OUT_DIR, 'sexpr-wasm')
CopyBinaryToArchive(sexpr)
def SyncLLVMClang():
llvm_rev = BUILDBOT_REVISION if SCHEDULER == 'llvm' else 'origin/master'
clang_rev = BUILDBOT_REVISION if SCHEDULER == 'clang' else 'origin/master'
proc.check_call(['git', 'checkout', llvm_rev], cwd=LLVM_SRC_DIR)
proc.check_call(['git', 'checkout', clang_rev], cwd=CLANG_SRC_DIR)
# If LLVM didn't trigger the new build then sync LLVM to the corresponding
# clang revision, even if clang may not have triggered the build: usually
# LLVM provides APIs which clang uses, which means that most synchronized
# commits touch LLVM before clang. This should reduce the chance of breakage.
primary = LLVM_SRC_DIR if SCHEDULER == 'llvm' else CLANG_SRC_DIR
secondary = LLVM_SRC_DIR if primary == CLANG_SRC_DIR else CLANG_SRC_DIR
SyncToSameSvnRev(primary, secondary)
def SyncLLVMClang(): llvm_rev = BUILDBOT_REVISION if SCHEDULER == 'llvm' else 'origin/master' clang_rev = BUILDBOT_REVISION if SCHEDULER == 'clang' else 'origin/master' proc.check_call(['git', 'checkout', llvm_rev], cwd=LLVM_SRC_DIR) proc.check_call(['git', 'checkout', clang_rev], cwd=CLANG_SRC_DIR) # If LLVM didn't trigger the new build then sync LLVM to the corresponding # clang revision, even if clang may not have triggered the build: usually # LLVM provides APIs which clang uses, which means that most synchronized # commits touch LLVM before clang. This should reduce the chance of breakage. primary = LLVM_SRC_DIR if SCHEDULER == 'llvm' else CLANG_SRC_DIR secondary = LLVM_SRC_DIR if primary == CLANG_SRC_DIR else CLANG_SRC_DIR SyncToSameSvnRev(primary, secondary)
def SyncOCaml():
if os.path.isdir(OCAML_DIR):
print 'OCaml directory already exists'
else:
print 'Downloading OCaml %s from %s' % (OCAML_VERSION, OCAML_URL)
f = urllib2.urlopen(OCAML_URL)
print 'URL: %s' % f.geturl()
print 'Info: %s' % f.info()
with open(OCAML_TAR, 'wb') as out:
out.write(f.read())
proc.check_call(['tar', '-xvf', OCAML_TAR], cwd=WORK_DIR)
assert os.path.isdir(OCAML_DIR), 'Untar should produce %s' % OCAML_DIR
def Emscripten(use_asm=True):
buildbot.Step('emscripten')
# Remove cached library builds (e.g. libc, libc++) to force them to be
# rebuilt in the step below.
Remove(os.path.expanduser(os.path.join('~', '.emscripten_cache')))
emscripten_dir = os.path.join(INSTALL_DIR, 'emscripten')
Remove(emscripten_dir)
print 'Copying directory %s to %s' % (EMSCRIPTEN_SRC_DIR, emscripten_dir)
shutil.copytree(EMSCRIPTEN_SRC_DIR,
emscripten_dir,
symlinks=True,
# Ignore the big git blob so it doesn't get archived.
ignore=shutil.ignore_patterns('.git'))
def WriteEmscriptenConfig(infile, outfile):
with open(infile) as config:
text = config.read().replace('{{WASM_INSTALL}}', INSTALL_DIR)
with open(outfile, 'w') as config:
config.write(text)
configs = [('emwasm', EMSCRIPTEN_CONFIG_WASM)]
if use_asm:
# build with asm2wasm first to match the ordering of the test steps
configs.insert(0, ('asm2wasm', EMSCRIPTEN_CONFIG_ASMJS))
for config_name, config in configs:
buildbot.Step('emscripten (%s)' % config_name)
print 'Config file: ', config
src_config = os.path.join(SCRIPT_DIR, os.path.basename(config))
WriteEmscriptenConfig(src_config, config)
try:
# Build a C++ file with each active emscripten config. This causes system
# libs to be built and cached (so we don't have that happen when building
# tests in parallel). Do it with full debug output.
# This depends on binaryen already being built and installed into the
# archive/install dir.
os.environ['EMCC_DEBUG'] = '2'
os.environ['EM_CONFIG'] = config
proc.check_call([
os.path.join(emscripten_dir, 'em++'),
os.path.join(EMSCRIPTEN_SRC_DIR, 'tests', 'hello_libcxx.cpp'),
'-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"'])
except proc.CalledProcessError:
# Note the failure but allow the build to continue.
buildbot.Fail()
finally:
del os.environ['EMCC_DEBUG']
del os.environ['EM_CONFIG']
wrapper = os.path.join(SCRIPT_DIR, 'emcc_wrapper.sh')
shutil.copy2(wrapper, os.path.join(INSTALL_BIN, 'emcc'))
shutil.copy2(wrapper, os.path.join(INSTALL_BIN, 'em++'))
def V8():
buildbot.Step('V8')
proc.check_call(['ninja', '-C', V8_OUT_DIR, 'd8', 'unittests'],
cwd=V8_SRC_DIR)
proc.check_call([
'tools/run-tests.py', 'unittests', '--no-presubmit', '--shell-dir',
V8_OUT_DIR
],
cwd=V8_SRC_DIR)
to_archive = ['d8', 'natives_blob.bin', 'snapshot_blob.bin']
for a in to_archive:
CopyBinaryToArchive(os.path.join(V8_OUT_DIR, a))
def SyncOCaml():
if os.path.isdir(OCAML_DIR):
print 'OCaml directory already exists'
else:
print 'Downloading OCaml %s from %s' % (OCAML_VERSION, OCAML_URL)
f = urllib2.urlopen(OCAML_URL)
print 'URL: %s' % f.geturl()
print 'Info: %s' % f.info()
with open(OCAML_TAR, 'wb') as out:
out.write(f.read())
proc.check_call(['tar', '-xvf', OCAML_TAR], cwd=WORK_DIR)
assert os.path.isdir(OCAML_DIR), 'Untar should produce %s' % OCAML_DIR
def ChromiumFetchSync(name, work_dir, git_repo, checkout='origin/master'):
"""Some Chromium projects want to use gclient for clone and dependencies."""
if os.path.isdir(work_dir):
print '%s directory already exists' % name
else:
# Create Chromium repositories one deeper, separating .gclient files.
parent = os.path.split(work_dir)[0]
Mkdir(parent)
proc.check_call(['gclient', 'config', git_repo], cwd=parent)
proc.check_call(['git', 'clone', git_repo], cwd=parent)
proc.check_call(['git', 'fetch'], cwd=work_dir)
proc.check_call(['git', 'checkout', checkout], cwd=work_dir)
proc.check_call(['gclient', 'sync'], cwd=work_dir)
return (name, work_dir)
def ChromiumFetchSync(name, work_dir, git_repo, checkout='origin/master'):
"""Some Chromium projects want to use gclient for clone and dependencies."""
if os.path.isdir(work_dir):
print '%s directory already exists' % name
else:
# Create Chromium repositories one deeper, separating .gclient files.
parent = os.path.split(work_dir)[0]
Mkdir(parent)
proc.check_call(['gclient', 'config', git_repo], cwd=parent)
proc.check_call(['git', 'clone', git_repo], cwd=parent)
proc.check_call(['git', 'fetch'], cwd=work_dir)
proc.check_call(['git', 'checkout', checkout], cwd=work_dir)
proc.check_call(['gclient', 'sync'], cwd=work_dir)
return (name, work_dir)
def AddGithubRemote(cwd):
"""When using the cloned repository for development, it's useful to have a
remote to github because origin points at a cache which is read-only."""
remote_url = GitRemoteUrl(cwd, WATERFALL_REMOTE)
if WASM_GIT_BASE not in remote_url:
print '%s not a github mirror' % cwd
return
if HasRemote(cwd, GITHUB_REMOTE):
print '%s has %s as its "%s" remote' % (
cwd, GitRemoteUrl(cwd, GITHUB_REMOTE), GITHUB_REMOTE)
return
remote = GITHUB_SSH + '/'.join(remote_url.split('/')[-2:])
print '%s has no github remote, adding %s' % (cwd, remote)
proc.check_call(['git', 'remote', 'add', GITHUB_REMOTE, remote], cwd=cwd)
def Remove(path):
"""Remove file or directory if it exists, do nothing otherwise."""
if not os.path.exists(path):
return
print 'Removing %s' % path
if not os.path.isdir(path):
os.remove(path)
return
if sys.platform == 'win32':
# shutil.rmtree() may not work in Windows if a directory contains read-only
# files.
proc.check_call('rmdir /S /Q "%s"' % path, shell=True)
else:
shutil.rmtree(path)
def WAVM():
buildbot.Step('WAVM')
Mkdir(WAVM_OUT_DIR)
command = [CMAKE_BIN, '-G', CMAKE_GENERATOR, WAVM_SRC_DIR,
'-DCMAKE_BUILD_TYPE=RelWithDebInfo',
'-DCMAKE_EXPORT_COMPILE_COMMANDS=YES',
'-DCMAKE_INSTALL_PREFIX=' + SYSROOT_DIR,
'-DCMAKE_TOOLCHAIN_FILE=' +
WindowsFSEscape(os.path.join(HOST_DIR, 'wavix_toolchain.cmake')),
'-DWAVM_ENABLE_RUNTIME=OFF',
'-DWAVM_ENABLE_STATIC_LINKING=ON',
'-DCMAKE_BUILD_WITH_INSTALL_RPATH=ON' ]
proc.check_call(command, cwd=WAVM_OUT_DIR)
proc.check_call([NINJA_BIN, 'install'], cwd=WAVM_OUT_DIR)
def SyncPrebuiltClang(name, src_dir, git_repo):
tools_clang = os.path.join(src_dir, 'tools', 'clang')
if os.path.isdir(tools_clang):
print 'Prebuilt Chromium Clang directory already exists'
else:
print 'Cloning Prebuilt Chromium Clang directory'
Mkdir(src_dir)
Mkdir(os.path.join(src_dir, 'tools'))
Git(['clone', git_repo, tools_clang])
Git(['fetch'], cwd=tools_clang)
proc.check_call(
[sys.executable, os.path.join(tools_clang, 'scripts', 'update.py')])
assert os.path.isfile(CC), 'Expect clang at %s' % CC
assert os.path.isfile(CXX), 'Expect clang++ at %s' % CXX
return ('chromium-clang', tools_clang)
def AddGithubRemote(cwd):
"""When using the cloned repository for development, it's useful to have a
remote to github because origin points at a cache which is read-only."""
origin_url = GitRemoteUrl(cwd, 'origin')
if WASM_GIT_BASE not in origin_url:
print '%s not a github mirror' % cwd
return
if HasRemote(cwd, GITHUB_REMOTE):
print '%s has %s as its "%s" remote' % (
cwd, GitRemoteUrl(cwd, GITHUB_REMOTE), GITHUB_REMOTE)
return
remote = GITHUB_SSH + '/'.join(GitRemoteUrl(cwd, 'origin').split('/')[-2:])
print '%s has no github remote, adding %s' % (cwd, remote)
proc.check_call(['git', 'remote', 'add', GITHUB_REMOTE, remote],
cwd=cwd)
def SetUpVSEnv(outdir):
"""Set up the VS build environment used by Chromium bots"""
# Get the chromium-packaged toolchain directory info in a JSON file
proc.check_call([VS_TOOLCHAIN, 'get_toolchain_dir'])
with open(WIN_TOOLCHAIN_JSON) as f:
paths = json.load(f)
# Write path information (usable by a non-chromium build) into an environment
# block
runtime_dirs = os.pathsep.join(paths['runtime_dirs'])
proc.check_call(
[SETUP_TOOLCHAIN, 'foo', paths['win_sdk'], runtime_dirs, 'x64'],
cwd=outdir)
return GetVSEnv(outdir)
def Emscripten(use_asm=True):
buildbot.Step('emscripten')
# Remove cached library builds (e.g. libc, libc++) to force them to be
# rebuilt in the step below.
Remove(os.path.expanduser(os.path.join('~', '.emscripten_cache')))
emscripten_dir = os.path.join(INSTALL_DIR, 'bin', 'emscripten')
Remove(emscripten_dir)
shutil.copytree(
EMSCRIPTEN_SRC_DIR,
emscripten_dir,
symlinks=True,
# Ignore the big git blob so it doesn't get archived.
ignore=shutil.ignore_patterns('.git'))
def WriteEmscriptenConfig(infile, outfile):
with open(infile) as config:
text = config.read().replace('{{WASM_INSTALL}}', INSTALL_DIR)
with open(outfile, 'w') as config:
config.write(text)
WriteEmscriptenConfig(os.path.join(SCRIPT_DIR, 'emscripten_config'),
EMSCRIPTEN_CONFIG_ASMJS)
WriteEmscriptenConfig(
os.path.join(SCRIPT_DIR, 'emscripten_config_vanilla'),
EMSCRIPTEN_CONFIG_WASM)
try:
# Build a C++ file with each active emscripten config. This causes system
# libs to be built and cached (so we don't have that happen when building
# tests in parallel). Do it with full debug output.
# This depends on binaryen already being built and installed into the
# archive/install dir.
os.environ['EMCC_DEBUG'] = '2'
configs = [EMSCRIPTEN_CONFIG_WASM
] + ([EMSCRIPTEN_CONFIG_ASMJS] if use_asm else [])
for config in configs:
os.environ['EM_CONFIG'] = config
proc.check_call([
os.path.join(emscripten_dir, 'em++'),
os.path.join(EMSCRIPTEN_SRC_DIR, 'tests',
'hello_libcxx.cpp'), '-O2', '-s', 'BINARYEN=1',
'-s', 'BINARYEN_METHOD="native-wasm"'
])
except proc.CalledProcessError:
# Note the failure but allow the build to continue.
buildbot.Fail()
finally:
del os.environ['EMCC_DEBUG']
def LLVM():
buildbot.Step('LLVM')
Mkdir(LLVM_OUT_DIR)
command = [
PREBUILT_CMAKE_BIN, '-G', 'Ninja', LLVM_SRC_DIR,
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON', '-DLLVM_BUILD_TESTS=ON',
'-DCMAKE_C_COMPILER=' + CC, '-DCMAKE_CXX_COMPILER=' + CXX,
'-DCMAKE_BUILD_TYPE=Release', '-DCMAKE_INSTALL_PREFIX=' + INSTALL_DIR,
'-DLLVM_INCLUDE_EXAMPLES=OFF', '-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DCOMPILER_RT_INCLUDE_TESTS=OFF', '-DLLVM_BUILD_LLVM_DYLIB=ON',
'-DLLVM_LINK_LLVM_DYLIB=ON', '-DLLVM_INSTALL_TOOLCHAIN_ONLY=ON',
'-DLLVM_ENABLE_ASSERTIONS=ON',
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
'-DLLVM_TARGETS_TO_BUILD=X86'
]
compiler_launcher = None
jobs = []
if 'GOMA_DIR' in os.environ:
compiler_launcher = os.path.join(os.environ['GOMA_DIR'], 'gomacc')
jobs = ['-j', '50']
else:
ccache = Which('ccache')
if ccache:
compiler_launcher = ccache
command.extend([
'-DCMAKE_%s_FLAGS=-Qunused-arguments' % c
for c in ['C', 'CXX']
])
if compiler_launcher:
command.extend([
'-DCMAKE_%s_COMPILER_LAUNCHER=%s' % (c, compiler_launcher)
for c in ['C', 'CXX']
])
proc.check_call(command, cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', '-v'] + jobs, cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', 'install'] + jobs, cwd=LLVM_OUT_DIR)
# The following isn't useful for now, and takes up space.
Remove(os.path.join(INSTALL_BIN, 'clang-check'))
# The following are useful, LLVM_INSTALL_TOOLCHAIN_ONLY did away with them.
extra_bins = [
'FileCheck', 'lli', 'llc', 'llvm-as', 'llvm-dis', 'llvm-link',
'llvm-nm', 'opt'
]
extra_libs = ['libLLVM*.so']
for p in [
glob.glob(os.path.join(LLVM_OUT_DIR, 'bin', b)) for b in extra_bins
]:
for e in p:
CopyBinaryToArchive(os.path.join(LLVM_OUT_DIR, 'bin', e))
for p in [
glob.glob(os.path.join(LLVM_OUT_DIR, 'lib', l)) for l in extra_libs
]:
for e in p:
CopyLibraryToArchive(os.path.join(LLVM_OUT_DIR, 'lib', e))
def Musl():
buildbot.Step('musl')
Mkdir(MUSL_OUT_DIR)
try:
proc.check_call([
os.path.join(MUSL_SRC_DIR, 'libc.py'),
'--clang_dir', INSTALL_BIN,
'--binaryen_dir', INSTALL_BIN,
'--sexpr_wasm', os.path.join(INSTALL_BIN, 'sexpr-wasm'),
'--musl', MUSL_SRC_DIR], cwd=MUSL_OUT_DIR)
for f in ['musl.wast', 'musl.wasm']:
CopyLibraryToArchive(os.path.join(MUSL_OUT_DIR, f))
CopyLibraryToArchive(os.path.join(MUSL_SRC_DIR,
'arch', 'wasm32', 'wasm.js'))
except proc.CalledProcessError:
buildbot.Fail()
def SetUpVSEnv(outdir):
"""Set up the VS build environment used by Chromium bots"""
# Get the chromium-packaged toolchain directory info in a JSON file
proc.check_call(VSToolchainPy() + ['get_toolchain_dir'])
with open(WinToolchainJson()) as f:
paths = json.load(f)
# Write path information (usable by a non-chromium build) into an
# environment block
runtime_dirs = os.pathsep.join(paths['runtime_dirs'])
proc.check_call(SetupToolchain() + [
'foo', paths['win_sdk'], runtime_dirs, 'win', 'x64', 'environment.x64'
],
cwd=outdir)
return GetVSEnv(outdir)
def CoreUtils():
buildbot.Step('coreutils')
Mkdir(COREUTILS_OUT_DIR)
# Build bash
proc.check_call([
'bash',
WAVIX_SRC_DIR.replace('\\', '/').replace('C:','/mnt/c') + '/coreutils/build-wavix-coreutils.sh',
WAVIX_SRC_DIR.replace('\\', '/').replace('C:','/mnt/c'),
BUILD_DIR.replace('\\', '/'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/clang'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/llvm-ar'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/llvm-ranlib'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/clang-cpp')
], cwd=COREUTILS_OUT_DIR)
def Bash():
buildbot.Step('bash')
Mkdir(BASH_OUT_DIR)
# Build bash
proc.check_call([
'bash',
WAVIX_SRC_DIR.replace('\\', '/').replace('C:','/mnt/c') + '/bash/build-wavix-bash.sh',
WAVIX_SRC_DIR.replace('\\', '/').replace('C:','/mnt/c'),
BUILD_DIR.replace('\\', '/'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/clang'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/llvm-ar'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/llvm-ranlib'),
BUILD_DIR.replace('\\', '/').replace('C:','/mnt/c') + Executable('/host/bin/clang-cpp')
], cwd=BASH_OUT_DIR)
def Emscripten(use_asm=True):
buildbot.Step('emscripten')
# Remove cached library builds (e.g. libc, libc++) to force them to be
# rebuilt in the step below.
Remove(os.path.expanduser(os.path.join('~', '.emscripten_cache')))
emscripten_dir = os.path.join(INSTALL_DIR, 'bin', 'emscripten')
Remove(emscripten_dir)
shutil.copytree(EMSCRIPTEN_SRC_DIR,
emscripten_dir,
symlinks=True,
# Ignore the big git blob so it doesn't get archived.
ignore=shutil.ignore_patterns('.git'))
def WriteEmscriptenConfig(infile, outfile):
with open(infile) as config:
text = config.read().replace('{{WASM_INSTALL}}', INSTALL_DIR)
with open(outfile, 'w') as config:
config.write(text)
WriteEmscriptenConfig(os.path.join(SCRIPT_DIR, 'emscripten_config'),
EMSCRIPTEN_CONFIG_ASMJS)
WriteEmscriptenConfig(os.path.join(SCRIPT_DIR, 'emscripten_config_vanilla'),
EMSCRIPTEN_CONFIG_WASM)
try:
# Build a C++ file with each active emscripten config. This causes system
# libs to be built and cached (so we don't have that happen when building
# tests in parallel). Do it with full debug output.
# This depends on binaryen already being built and installed into the
# archive/install dir.
os.environ['EMCC_DEBUG'] = '2'
configs = [EMSCRIPTEN_CONFIG_WASM] + (
[EMSCRIPTEN_CONFIG_ASMJS] if use_asm else [])
for config in configs:
os.environ['EM_CONFIG'] = config
proc.check_call([
os.path.join(emscripten_dir, 'em++'),
os.path.join(EMSCRIPTEN_SRC_DIR, 'tests', 'hello_libcxx.cpp'),
'-O2', '-s', 'BINARYEN=1', '-s', 'BINARYEN_METHOD="native-wasm"'])
except proc.CalledProcessError:
# Note the failure but allow the build to continue.
buildbot.Fail()
finally:
del os.environ['EMCC_DEBUG']
def Binaryen():
buildbot.Step('binaryen')
Mkdir(BINARYEN_OUT_DIR)
proc.check_call(
['cmake', '-G', 'Ninja', BINARYEN_SRC_DIR,
'-DCMAKE_C_COMPILER=' + CC,
'-DCMAKE_CXX_COMPILER=' + CXX],
cwd=BINARYEN_OUT_DIR)
proc.check_call(['ninja'], cwd=BINARYEN_OUT_DIR)
assert os.path.isdir(BINARYEN_BIN_DIR), 'Expected %s' % BINARYEN_BIN_DIR
for node in os.listdir(BINARYEN_BIN_DIR):
f = os.path.join(BINARYEN_BIN_DIR, node)
if os.path.isfile(f):
CopyBinaryToArchive(f)
CopyBinaryToArchive(os.path.join(BINARYEN_SRC_DIR, 'bin', 'wasm.js'))
Mkdir(os.path.join(INSTALL_DIR, 'src'))
Mkdir(os.path.join(INSTALL_DIR, 'src', 'js'))
shutil.copy2(os.path.join(BINARYEN_SRC_DIR, 'src', 'js', 'wasm.js-post.js'),
os.path.join(INSTALL_DIR, 'src', 'js'))
def Binaryen():
buildbot.Step('binaryen')
Mkdir(BINARYEN_OUT_DIR)
proc.check_call(
[PREBUILT_CMAKE_BIN, '-G', 'Ninja', BINARYEN_SRC_DIR,
'-DCMAKE_C_COMPILER=' + CC,
'-DCMAKE_CXX_COMPILER=' + CXX],
cwd=BINARYEN_OUT_DIR)
proc.check_call(['ninja'], cwd=BINARYEN_OUT_DIR)
bin_dir = os.path.join(BINARYEN_OUT_DIR, 'bin')
assert os.path.isdir(bin_dir), 'Expected %s' % bin_dir
for node in os.listdir(bin_dir):
f = os.path.join(bin_dir, node)
if os.path.isfile(f):
CopyBinaryToArchive(f)
CopyBinaryToArchive(os.path.join(BINARYEN_SRC_DIR, 'bin', 'wasm.js'))
js_src_dir = os.path.join(BINARYEN_SRC_DIR, 'src', 'js')
js_dest_dir = os.path.join(INSTALL_DIR, 'src', 'js')
print 'copying directory', js_src_dir, 'to', js_dest_dir
CopyTree(js_src_dir, js_dest_dir)
def Musl():
buildbot.Step('musl')
Mkdir(MUSL_OUT_DIR)
try:
proc.check_call([
sys.executable,
os.path.join(MUSL_SRC_DIR, 'libc.py'),
'--clang_dir', INSTALL_BIN,
'--binaryen_dir', os.path.join(INSTALL_BIN),
'--sexpr_wasm', os.path.join(INSTALL_BIN, 'wast2wasm'),
'--musl', MUSL_SRC_DIR], cwd=MUSL_OUT_DIR)
for f in ['musl.wast', 'musl.wasm']:
CopyLibraryToArchive(os.path.join(MUSL_OUT_DIR, f))
CopyLibraryToArchive(os.path.join(MUSL_SRC_DIR,
'arch', 'wasm32', 'wasm.js'))
CopyTree(os.path.join(MUSL_SRC_DIR, 'include'),
os.path.join(INSTALL_SYSROOT, 'include'))
CopyTree(os.path.join(MUSL_SRC_DIR, 'arch', 'wasm32'),
os.path.join(INSTALL_SYSROOT, 'include'))
except proc.CalledProcessError:
# Note the failure but allow the build to continue.
buildbot.Fail()
def Wabt():
buildbot.Step('WABT')
Mkdir(WABT_OUT_DIR)
proc.check_call([PREBUILT_CMAKE_BIN, '-G', 'Ninja', WABT_SRC_DIR,
'-DCMAKE_C_COMPILER=%s' % CC,
'-DCMAKE_CXX_COMPILER=%s' % CXX,
'-DCMAKE_INSTALL_PREFIX=%s' % INSTALL_DIR,
'-DBUILD_TESTS=OFF'],
cwd=WABT_OUT_DIR)
proc.check_call(['ninja'], cwd=WABT_OUT_DIR)
proc.check_call(['ninja', 'install'], cwd=WABT_OUT_DIR)
def LLVM():
buildbot.Step('LLVM')
Mkdir(LLVM_OUT_DIR)
command = [PREBUILT_CMAKE_BIN, '-G', 'Ninja', LLVM_SRC_DIR,
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DLLVM_BUILD_TESTS=ON',
'-DCMAKE_C_COMPILER=' + CC,
'-DCMAKE_CXX_COMPILER=' + CXX,
'-DCMAKE_BUILD_TYPE=Release',
'-DCMAKE_INSTALL_PREFIX=' + INSTALL_DIR,
'-DLLVM_INCLUDE_EXAMPLES=OFF',
'-DCLANG_INCLUDE_EXAMPLES=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DLLVM_BUILD_LLVM_DYLIB=ON',
'-DLLVM_LINK_LLVM_DYLIB=ON',
'-DLLVM_INSTALL_TOOLCHAIN_ONLY=ON',
'-DLLVM_ENABLE_ASSERTIONS=ON',
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
'-DLLVM_TARGETS_TO_BUILD=X86']
ccache = Which('ccache')
if ccache:
command.extend(['-DCMAKE_%s_COMPILER_LAUNCHER=%s' %
(c, ccache) for c in ['C', 'CXX']])
command.extend(['-DCMAKE_%s_FLAGS=-Qunused-arguments' %
c for c in ['C', 'CXX']])
proc.check_call(command, cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', '-v'], cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', 'install'], cwd=LLVM_OUT_DIR)
# The following isn't useful for now, and takes up space.
Remove(os.path.join(INSTALL_BIN, 'clang-check'))
# The following are useful, LLVM_INSTALL_TOOLCHAIN_ONLY did away with them.
extra_bins = ['FileCheck', 'lli', 'llc', 'llvm-as', 'llvm-dis', 'llvm-link',
'llvm-nm', 'opt']
extra_libs = ['libLLVM*.so']
for p in [glob.glob(os.path.join(LLVM_OUT_DIR, 'bin', b)) for b in
extra_bins]:
for e in p:
CopyBinaryToArchive(os.path.join(LLVM_OUT_DIR, 'bin', e))
for p in [glob.glob(os.path.join(LLVM_OUT_DIR, 'lib', l)) for l in
extra_libs]:
for e in p:
CopyLibraryToArchive(os.path.join(LLVM_OUT_DIR, 'lib', e))
def OCaml():
buildbot.Step('OCaml')
makefile = os.path.join(OCAML_DIR, 'config', 'Makefile')
if not os.path.isfile(makefile):
configure = os.path.join(OCAML_DIR, 'configure')
proc.check_call(
[configure, '-prefix', OCAML_OUT_DIR, '-cc', CC], cwd=OCAML_DIR)
proc.check_call(['make', 'world.opt', '-j%s' % NPROC], cwd=OCAML_DIR)
proc.check_call(['make', 'install'], cwd=OCAML_DIR)
ocamlbuild = os.path.join(OCAML_BIN_DIR, 'ocamlbuild')
assert os.path.isfile(ocamlbuild), 'Expected installed %s' % ocamlbuild
os.environ['PATH'] = OCAML_BIN_DIR + os.pathsep + os.environ['PATH']
def LLVM():
buildbot.Step('LLVM')
Mkdir(LLVM_OUT_DIR)
command = [PREBUILT_CMAKE_BIN, '-G', 'Ninja', LLVM_SRC_DIR,
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DLLVM_BUILD_TESTS=ON',
'-DCMAKE_C_COMPILER=' + CC,
'-DCMAKE_CXX_COMPILER=' + CXX,
'-DCMAKE_BUILD_TYPE=Release',
'-DCMAKE_INSTALL_PREFIX=' + INSTALL_DIR,
'-DLLVM_INCLUDE_EXAMPLES=OFF',
'-DCOMPILER_RT_BUILD_XRAY=OFF',
'-DCOMPILER_RT_INCLUDE_TESTS=OFF',
'-DLLVM_BUILD_LLVM_DYLIB=ON',
'-DLLVM_LINK_LLVM_DYLIB=ON',
'-DLLVM_INSTALL_TOOLCHAIN_ONLY=ON',
'-DLLVM_ENABLE_ASSERTIONS=ON',
'-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly',
'-DLLVM_TARGETS_TO_BUILD=X86']
compiler_launcher = None
jobs = []
if 'GOMA_DIR' in os.environ:
compiler_launcher = os.path.join(os.environ['GOMA_DIR'], 'gomacc')
jobs = ['-j', '50']
else:
ccache = Which('ccache')
if ccache:
compiler_launcher = ccache
command.extend(['-DCMAKE_%s_FLAGS=-Qunused-arguments' %
c for c in ['C', 'CXX']])
if compiler_launcher:
command.extend(['-DCMAKE_%s_COMPILER_LAUNCHER=%s' %
(c, compiler_launcher) for c in ['C', 'CXX']])
proc.check_call(command, cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', '-v'] + jobs, cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', 'check-all'], cwd=LLVM_OUT_DIR)
proc.check_call(['ninja', 'install'] + jobs, cwd=LLVM_OUT_DIR)
CopyLLVMTools(LLVM_OUT_DIR)
def Tar(directory, print_content=False):
"""Create a tar file from directory."""
if not BUILDBOT_BUILDERNAME:
return
assert os.path.isdir(directory), 'Must tar a directory to avoid tarbombs'
(up_directory, basename) = os.path.split(directory)
tar = os.path.join(up_directory, basename + '.tbz2')
Remove(tar)
if print_content:
proc.check_call(['find', basename, '-type', 'f',
'-exec', 'ls', '-lhS', '{}', '+'], cwd=up_directory)
proc.check_call(['tar', 'cjf', tar, basename], cwd=up_directory)
proc.check_call(['ls', '-lh', tar], cwd=up_directory)
return tar
def Fastcomp():
buildbot.Step('fastcomp')
Mkdir(FASTCOMP_OUT_DIR)
proc.check_call(
['cmake', '-G', 'Ninja', FASTCOMP_SRC_DIR,
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
'-DCMAKE_C_COMPILER=' + CC,
'-DCMAKE_CXX_COMPILER=' + CXX,
'-DCMAKE_BUILD_TYPE=Release',
'-DCMAKE_INSTALL_PREFIX=' + os.path.join(INSTALL_DIR, 'fastcomp'),
'-DLLVM_INCLUDE_EXAMPLES=OFF',
'-DCLANG_INCLUDE_EXAMPLES=OFF',
'-DLLVM_BUILD_LLVM_DYLIB=ON',
'-DLLVM_LINK_LLVM_DYLIB=ON',
'-DLLVM_TARGETS_TO_BUILD=X86;JSBackend',
'-DLLVM_ENABLE_ASSERTIONS=ON'], cwd=FASTCOMP_OUT_DIR)
proc.check_call(['ninja'], cwd=FASTCOMP_OUT_DIR)
try:
proc.check_call(['ninja', 'check'], cwd=FASTCOMP_OUT_DIR)
except proc.CalledProcessError:
print 'ಠ_ಠ'
proc.check_call(['ninja', 'install'], cwd=FASTCOMP_OUT_DIR)
