def _addSteps4SystemCompiler(
f,
stage_idx = 0,
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
# This stage could use incremental build.
# Clean stage1, only if requested.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
('-DCMAKE_COLOR_MAKEFILE=', 'OFF'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Unix Makefiles'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
# Note: On this stage we do not care of warnings, as we build with
# a system toolchain and cannot control the environment.
# Warnings are likely, and we ignore them.
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
flunkOnWarnings=False,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build clang by the system compiler
f.addStep(ShellCommand(name="build-stage%s-compiler"% stage_num,
command=['make', WithProperties("-j%s" % jobs), "-k"],
haltOnFailure=True,
flunkOnWarnings=False,
description=["build stage%s compiler" % stage_num],
env=env,
workdir=obj_dir))
# Test stage1 compiler
f.addStep(ShellCommand(name="test-stage%s-compiler"% stage_num,
command=["make", WithProperties("-j%s" % jobs), "check-all"], # or "check-llvm", "check-clang"
haltOnFailure=True,
flunkOnWarnings=False,
description=["test stage%s compiler" % stage_num],
env=env,
workdir=obj_dir))
# Install stage1 compiler
f.addStep(ShellCommand(name="install-stage%s-compiler"% stage_num,
command=["make", WithProperties("-j%s" % jobs), "install"],
haltOnFailure=True,
flunkOnWarnings=False,
description=["install stage%s compiler" % stage_num],
env=env,
workdir=obj_dir))
def getToolchainBuildFactory(
clean=False,
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = LLVMBuildFactory(
depends_on_projects=[
"llvm",
"clang",
"clang-tools-extra",
"compiler-rt",
"libcxx",
"libcxxabi",
"libunwind",
"lld"
])
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(RemoveDirectory(name="clean-sdk",
dir=sdk_dir,
haltOnFailure=True))
f.addStep(ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean
# Clean up llvm sources.
f.addStep(RemoveDirectory(name="clean-llvm.src",
dir=src_dir,
haltOnFailure=True,
doStepIf=cleanCheckoutRequested))
# Get sources.
for project in f.depends_on_projects:
_, baseURL = svn_repos[project]
f.addStep(SVN(name="svn-%s" % project,
workdir=src_dir + "/" + project,
baseURL=baseURL))
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
# Clean up llvm build.
f.addStep(RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties(
"-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir
))
cmake_options.append(
WithProperties(
"-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir
))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(NinjaCommand(name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=None,
testsuite_flags=None,
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env=None,
extra_cmake_args=None,
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
checkout_flang=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=None):
############# PREPARING
if nt_flags is None:
nt_flags = []
if testsuite_flags is None:
testsuite_flags = []
if env is None:
env = {}
if extra_cmake_args is None:
extra_cmake_args = []
if trigger_after_stage1 is None:
trigger_after_stage1 = []
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
if checkout_flang:
depends_on_projects.append('flang')
depends_on_projects.append('mlir')
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
# Checkout the latest code for LNT
# and the test-suite separately. Le's do this first,
# so we wouldn't poison got_revision property.
if runTestSuite or checkout_test_suite:
f.addGetSourcecodeForProject(project='lnt',
src_dir='test/lnt',
alwaysUseLatest=True)
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test/test-suite',
alwaysUseLatest=True)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs and vs != "manual":
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
CmakeCommand.applyRequiredOptions(extra_cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", rel_src_dir,
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
LitTestCommand(name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir,
stage2_build)
cmake_cmd2 = [
cmake, "-G", "Ninja", rel_src_dir,
WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install +
"/bin/" + cc),
WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" +
stage1_install + "/bin/" + cxx),
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
LitTestCommand(name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
LitTestCommand(name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def _addSteps4StagedCompiler(
f,
stage_idx = 1,
use_stage_idx = -1,
withLTOSupport = False,
jobs = None,
extra_configure_args = None,
env = None):
if use_stage_idx < 0:
use_stage_idx = stage_idx - 1
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[use_stage_idx]
# Always do a clean build for the staged compiler.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if withLTOSupport:
CmakeCommand.applyDefaultOptions(cmake_args, [
# LTO Plugin dependency:
('-DLLVM_BINUTILS_INCDIR=', '/opt/binutils/include'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
))
cmake_args.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
))
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")
))
if withLTOSupport:
# Build LTO plugin if requested.
f.addStep(NinjaCommand(name="build-stage%s-LLVMgold.so" % stage_num,
targets=['lib/LLVMgold.so'],
haltOnFailure=True,
description=["stage%s build LLVMgold.so" % stage_num],
env=env,
workdir=obj_dir,
))
# Build clang by the staged compiler
f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
haltOnFailure=True,
description=["build stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Test just built compiler
f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
targets=["check-all"],
haltOnFailure=True,
description=["test stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Install just built compiler
f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
targets=["install"],
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
def getCmakeBuildFactory(depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
install_dir=None,
clean=False,
extra_configure_args=None,
env=None,
**kwargs):
# Set defaults
if not depends_on_projects:
depends_on_projects = ['llvm', 'clang']
if extra_configure_args is None:
extra_configure_args = []
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if not obj_dir:
obj_dir = "build"
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
install_dir, obj_dir)
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir or "llvm.src",
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=cleanBuildRequested,
**kwargs # Pass through all the extra arguments.
)
# Directories to use on this stage.
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
# Do a clean checkout if requested.
f.addStep(
RemoveDirectory(
name='clean-src-dir',
dir=f.llvm_srcdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Get the source code.
f.addSVNSteps()
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % f.obj_dir,
dir=f.obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
if f.install_dir:
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % f.install_dir,
dir=f.install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
f.addStep(
CmakeCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt"),
**kwargs # Pass through all the extra arguments.
))
return f
def getSanitizerBuildFactoryII(
clean=False,
sanity_check=True,
sanitizers=[
'sanitizer', 'asan', 'lsan', 'msan', 'tsan', 'ubsan', 'dfsan'
],
common_cmake_options=None, # FIXME: For backward compatibility. Will be removed.
extra_configure_args=[],
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
env=None,
jobs="%(jobs)s",
timeout=1200):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb', # Make sure Clang doesn't use color escape sequences.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = BuildFactory()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + llvm_objdir,
dir=llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
# Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind
f.addStep(
SVN(name='svn-llvm',
mode='update',
description='svn-llvm',
descriptionDone='svn-llvm',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
description='svn-clang',
descriptionDone='svn-clang',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
description='svn-compiler-rt',
descriptionDone='svn--compiler-rt',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(
SVN(name='svn-libcxx',
mode='update',
description='svn-libcxx',
descriptionDone='svn-libcxx',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir='%s/projects/libcxx' % llvm_srcdir))
f.addStep(
SVN(name='svn-libcxxabi',
mode='update',
description='svn-libcxxabi',
descriptionDone='svn-libcxxabi',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir='%s/projects/libcxxabi' % llvm_srcdir))
f.addStep(
SVN(name='svn-libunwind',
mode='update',
description='svn-libunwind',
descriptionDone='svn-libunwind',
baseURL='http://llvm.org/svn/llvm-project/libunwind/',
defaultBranch='trunk',
workdir='%s/projects/libunwind' % llvm_srcdir))
# Run annotated command for sanitizer.
if sanity_check:
f.addStep(
AnnotatedCommand(
name="lint check",
description="lint check",
timeout=timeout,
haltOnFailure=False, #True,
warnOnWarnings=True,
command=["./check_lint.sh"],
workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts"
% llvm_srcdir,
env=merged_env))
# Always build with ninja.
cmakeCommand = ["cmake", "-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(
a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
if not any(
a.startswith('-DLLVM_ENABLE_WERROR=')
for a in extra_configure_args):
cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF')
if not any(
a.startswith('-DLLVM_ENABLE_ASSERTIONS=')
for a in extra_configure_args):
cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(
a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_C_COMPILER=clang')
if not any(
a.startswith('-DCMAKE_CXX_COMPILER')
for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++')
if not any(
a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"')
if not any(
a.startswith('-DCMAKE_EXE_LINKER_FLAGS')
for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt')
if not any(
a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=')
for a in extra_configure_args):
cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"')
cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=False, #True,
warnOnWarnings=True,
command=WithProperties(" ".join(cmakeCommand)),
env=merged_env,
workdir=llvm_objdir,
doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir)))
# Build everything.
f.addStep(
NinjaCommand(
name='build',
haltOnFailure=False, #True,
warnOnWarnings=True,
description=['building', 'with', 'ninja'],
descriptionDone=['built', 'with', 'ninja'],
workdir=llvm_objdir,
env=merged_env))
# Run tests for each of the requested sanitizers.
if sanitizers:
for s in sanitizers:
f.addStep(
NinjaCommand(
name='test %s' % s,
targets=['check-%s' % s],
haltOnFailure=False, #True,
description=['testing', '%s' % s],
descriptionDone=['test', '%s' % s],
workdir=llvm_objdir,
env=merged_env))
return f
def getLLDWinBuildFactory(
clean = True,
# Default values for VS devenv and build configuration
vs = None, # What to run to configure Visual Studio utils.
target_arch = None, # Native.
extra_configure_args = None,
env = None):
# Set defaults
if vs is None:
vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015.
if extra_configure_args is None:
extra_configure_args = []
if env is None:
env = {}
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# If set up environment step is requested, do this now.
if vs:
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
# Always build with ninja.
cmake_options = ["-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmake_options.append('-DCMAKE_BUILD_TYPE=Release')
if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_WERROR=ON')
if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"')
cmake_options += extra_configure_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
warnOnWarnings=True,
options=cmake_options,
path="../%s" % f.llvm_srcdir,
env=env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld.
f.addStep(NinjaCommand(name='build lld',
haltOnFailure=True,
warnOnWarnings=True,
description='build lld',
workdir=f.llvm_objdir,
env=env))
# Test Lld
f.addStep(NinjaCommand(name='test lld',
targets=['lld-test'],
haltOnFailure=True,
warnOnWarnings=True,
description='test lld',
workdir=f.llvm_objdir,
env=env))
return f
def getLLDBuildFactory(
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Set defaults
if jobs is None:
jobs = "%(jobs)s"
if extra_configure_args is None:
extra_configure_args = []
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'CC' : "clang",
'CXX' : "clang++",
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=extra_configure_args,
path="../%s" % f.llvm_srcdir,
env=merged_env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld
f.addStep(ShellCommand(name="build_Lld",
command=['nice', '-n', '10',
'make', WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["build lld"],
env=merged_env,
workdir=f.llvm_objdir))
# Test Lld
f.addStep(ShellCommand(name="test_lld",
command=["make", "lld-test"],
haltOnFailure=True,
description=["test lld"],
env=merged_env,
workdir=f.llvm_objdir))
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=[],
testsuite_flags=[],
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env={},
extra_cmake_args=[],
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Triggers
trigger_after_stage1=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
clean_build_requested = lambda step: clean or step.build.getProperty(
"clean")
addSVNUpdateSteps(f,
checkout_clang_tools_extra=checkout_clang_tools_extra,
checkout_compiler_rt=checkout_compiler_rt,
checkout_lld=checkout_lld,
checkout_test_suite=runTestSuite or checkout_test_suite,
checkout_libcxx=checkout_libcxx)
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url')
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
cmake_cmd2 = [
'env',
WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
cxx), cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f