def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
checks=None,
install_dir=None,
clean=False,
extra_configure_args=None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env=None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
f = getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(
SetPropertyFromCommand(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractVSEnvironment))
env = util.Property('vs_env')
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(f,
generator='Ninja',
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
checks=None,
install_dir=None,
clean=False,
extra_configure_args=None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env=None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs = r"""%VS140COMNTOOLS%"""
f = getLLVMBuildFactoryAndSVNSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=None,
testsuite_flags=None,
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env=None,
extra_cmake_args=None,
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
checkout_flang=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=None):
############# PREPARING
if nt_flags is None:
nt_flags = []
if testsuite_flags is None:
testsuite_flags = []
if env is None:
env = {}
if extra_cmake_args is None:
extra_cmake_args = []
if trigger_after_stage1 is None:
trigger_after_stage1 = []
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
if checkout_flang:
depends_on_projects.append('flang')
depends_on_projects.append('mlir')
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
# Checkout the latest code for LNT
# and the test-suite separately. Le's do this first,
# so we wouldn't poison got_revision property.
if runTestSuite or checkout_test_suite:
f.addGetSourcecodeForProject(project='lnt',
src_dir='test/lnt',
alwaysUseLatest=True)
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test/test-suite',
alwaysUseLatest=True)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs and vs != "manual":
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
CmakeCommand.applyRequiredOptions(extra_cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", rel_src_dir,
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
LitTestCommand(name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir,
stage2_build)
cmake_cmd2 = [
cmake, "-G", "Ninja", rel_src_dir,
WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install +
"/bin/" + cc),
WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" +
stage1_install + "/bin/" + cxx),
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
LitTestCommand(name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
LitTestCommand(name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=[],
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env={},
extra_cmake_args=[],
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Triggers
trigger_after_stage1=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
addSVNUpdateSteps(f,
checkout_clang_tools_extra=checkout_clang_tools_extra,
checkout_compiler_rt=checkout_compiler_rt,
checkout_test_suite=runTestSuite)
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
check_build_cmd = [
"sh", "-c", "test -e build.ninja && echo OK || echo Missing"
]
if vs:
check_build_cmd = [
"cmd", "/c",
"if exist build.ninja (echo OK) " + " else (echo Missing & exit 1)"
]
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
if clean:
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
env=env))
else:
f.addStep(
SetProperty(name="check ninja files 1",
workdir=stage1_build,
command=check_build_cmd,
flunkOnFailure=False,
property="exists_ninja_1"))
############# STAGE 1
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=lambda step: step.build.getProperty("exists_ninja_1"
) != "OK",
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url')
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.',
env=env))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
cmake_cmd2 = [
'env',
WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
cxx), cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
doStepIf=lambda step: step.build.getProperty(
"exists_ninja_2") != "OK",
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags to get the same effect)
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
if testerName:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def getCmakeWithMSVCBuildFactory(
clean = True, # False for incremental builds.
depends_on_projects = None, # List of projects to listen.
cmake_cache = None, # Path to a cmake cache file.
extra_configure_args = None, # Extra CMake args if any.
llvm_srcdir = None, # Source code root directory.
obj_dir = None, # Build tree root directory.
install_dir = None, # Directory to install the results to.
checks = None, # List of checks to test the build.
checks_on_target = None, # [(<name>,[<command tokens>])] array of
# check name and command to run on target.
jobs = None, # Restrict a degree of parallelism.
env = None, # Environmental variables for all steps.
# VS tools environment variable if using MSVC.
# For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select
# the VS 2015 toolchain, or empty string if environment is already set.
vs=None,
**kwargs):
if vs is None:
# We autodetect Visual Studio, unless otherwise is requested.
vs = "autodetect"
if install_dir is None:
install_dir = 'install'
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not vs, "Cannot have custom builder env vars with VS setup."
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if depends_on_projects is None:
depends_on_projects = [
'llvm',
'compiler-rt',
'clang',
'clang-tools-extra',
'libunwind',
'libcxx',
'libcxxabi',
'lld',
]
if checks is None:
# Check only host-side tools. Target-side tests should run on a target.
checks=[
"check-llvm",
"check-clang",
"check-lld",
]
source_remove_requested = lambda step: step.build.getProperty("clean")
clean_build_requested = lambda step: \
clean or \
step.build.getProperty("clean", \
default=step.build.getProperty("clean_obj") \
)
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=clean_build_requested,
**kwargs) # Pass through all the extra arguments.
# Remove the source code tree if requested.
# NOTE: Somehow RemoveDirectory buildbot command often fails on Windows,
# as somthing keeps a lock. We use rm command instead realying on a shell
# to support that.
f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir,
command=['rm','-rf', f.monorepo_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove the source code',
workdir='.',
env=merged_env,
doStepIf=source_remove_requested))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Remove the build directory if requested.
f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir,
command=['rm','-rf', f.obj_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove build directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
if vs:
# Configure MSVC environment if requested.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, None),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Since this is a build of a cross toolchain, we build only the host-side
# tools first by the host system compiler. Libraries will be cross-compiled.
cmake_args.append(InterpolateToPosixPath(
'-DLLVM_AR=%(workdir)s/' + f.obj_dir + '/bin/llvm-ar.exe')),
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'),
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeTo(
install_dir,
f.obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
# Remove the build directory if requested.
f.addStep(ShellCommand(name='clean-%s-dir' % install_dir,
command=['rm','-rf', install_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove install directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
src_dir_rel = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)
# Add given cmake cache at the very end.
if cmake_cache:
cmake_args.append('-C%s' % cmake_cache)
f.addStep(CmakeCommand(name="cmake-configure",
haltOnFailure=True,
description=["Cmake", "configure"],
options=cmake_args,
path=src_dir_rel,
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir,
haltOnFailure=True,
description=["Build", f.monorepo_dir],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Test the components if requested one check at a time.
for check in checks:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=["ninja", WithProperties(check)],
description=[
"Testing", "just", "built", "components", "for", check],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Run commands on a target if requested.
if checks_on_target:
for check, cmd in checks_on_target:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=cmd,
description=[
"Testing", "just", "built", "components", "for", check],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
if install_dir:
f.addStep(NinjaCommand(name="install-all",
haltOnFailure=True,
targets=["install"],
description=[
"Install", "just", "built", "components"],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=[],
testsuite_flags=[],
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env={},
extra_cmake_args=[],
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=[]):
############# PREPARING
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if runTestSuite or checkout_test_suite:
depends_on_projects.append('lnt')
depends_on_projects.append('test-suite')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
f.addSVNSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j"+str(jobs)]
lit_args += " -j"+str(jobs)+"'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(ShellCommand(name='clean stage 1',
command=['rm','-rf',stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
f.addStep(ShellCommand(name='cmake stage 1',
command=[cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE="+stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True",
"-DLLVM_LIT_ARGS="+lit_args,
"-DCMAKE_INSTALL_PREFIX=../"+stage1_install]
+ extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(lit_test_command.LitTestCommand(name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(ShellCommand(name='clean stage 1 install',
command=['rm','-rf',stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f, 'stage 1', stage1_install, stage1_upload_directory,
env, gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(ShellCommand(name='clean stage 2',
command=['rm','-rf',stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
cmake_cmd2 = ['env',
WithProperties('CC=%(workdir)s/'+stage1_install+'/bin/'+cc),
WithProperties('CXX=%(workdir)s/'+stage1_install+'/bin/'+cxx),
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE="+stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True",
"-DLLVM_LIT_ARGS="+lit_args,
"-DCMAKE_INSTALL_PREFIX=../"+stage2_install] + extra_cmake_args
f.addStep(ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(lit_test_command.LitTestCommand(name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path=stage2_install
f.addStep(ShellCommand(name='clean stage 2 install',
command=['rm','-rf',stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cc)
cxx = WithProperties('%(workdir)s/'+compiler_path+'/bin/'+cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [python, lnt, 'runtest', 'nt',
'--no-timestamp',
'--sandbox', sandbox,
'--test-suite', test_suite_dir,
'--cc', cc,
'--cxx', cxx]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/'+stage1_build+'/bin/llvm-lit')
test_suite_cmd = [python, lnt, 'runtest', 'test-suite',
'--no-timestamp',
'--sandbox', sandbox,
'--test-suite', test_suite_dir,
'--cc', cc,
'--cxx', cxx,
'--use-lit', lit]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={'configure.log' : 'build/configure.log',
'build-tools.log' : 'build/build-tools.log',
'test.log' : 'build/test.log',
'report.json' : 'build/report.json'},
env=test_suite_env))
return f
def getClangBuildFactory(
triple=None,
clean=True,
test=True,
package_dst=None,
run_cxx_tests=False,
examples=False,
valgrind=False,
valgrindLeakCheck=False,
outOfDir=False,
useTwoStage=False,
completely_clean=False,
make='make',
jobs="%(jobs)s",
stage1_config='Debug+Asserts',
stage2_config='Release+Asserts',
env={}, # Environmental variables for all steps.
extra_configure_args=[],
use_pty_in_tests=False,
trunk_revision=None,
force_checkout=False,
extra_clean_step=None,
checkout_compiler_rt=False,
run_gdb=False,
run_modern_gdb=False,
run_gcc=False,
merge_functions=False):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if run_gdb or run_gcc or run_modern_gdb:
outOfDir = True
# Don't use in-dir builds with a two stage build process.
inDir = not outOfDir and not useTwoStage
if inDir:
llvm_srcdir = "llvm"
llvm_1_objdir = "llvm"
llvm_1_installdir = None
else:
llvm_srcdir = "llvm.src"
llvm_1_objdir = "llvm.obj"
llvm_1_installdir = "llvm.install.1"
llvm_2_objdir = "llvm.obj.2"
llvm_2_installdir = "llvm.install"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Blow away completely, if requested.
if completely_clean:
f.addStep(ShellCommand(name="rm-llvm.src",
command=["rm", "-rf", llvm_srcdir],
haltOnFailure=True,
description=["rm src dir", "llvm"],
workdir=".",
env=merged_env))
# Checkout sources.
if trunk_revision:
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout']
if force_checkout:
svn_co += ['--force']
svn_co += ['--revision', WithProperties(trunk_revision)]
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' %
trunk_revision),
llvm_srcdir]
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' %
trunk_revision),
'%s/tools/clang' % llvm_srcdir]
svn_co_clang_tools_extra = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' %
trunk_revision),
'%s/tools/clang/tools/extra' % llvm_srcdir]
f.addStep(ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(ShellCommand(name='svn-clang-tools-extra',
command=svn_co_clang_tools_extra,
haltOnFailure=True,
workdir='.'))
else:
f.addStep(SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if checkout_compiler_rt:
f.addStep(SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
# Revert and apply patch mergeFunctions in required
if merge_functions:
f.addStep(ShellCommand(name="revert_patch_MergeFunctions",
command=["svn", "-R", "revert", '.'],
haltOnFailure=True,
description=["revert patch MergeFunctions"],
workdir='%s/tools/clang' % llvm_srcdir,
env=merged_env))
if merge_functions:
f.addStep(ShellCommand(name="patch_MergeFunctions",
command=["patch", "-Np0", "-i", '../../utils/Misc/mergefunctions.clang.svn.patch'],
haltOnFailure=True,
description=["patch MergeFunctions"],
workdir='%s/tools/clang' % llvm_srcdir,
env=merged_env))
# Clean up llvm (stage 1); unless in-dir.
if clean and llvm_srcdir != llvm_1_objdir:
f.addStep(ShellCommand(name="rm-llvm.obj.stage1",
command=["rm", "-rf", llvm_1_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
# Force without llvm-gcc so we don't run afoul of Frontend test failures.
base_configure_args = [WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
'--disable-bindings']
base_configure_args += extra_configure_args
if triple:
base_configure_args += ['--build=%s' % triple,
'--host=%s' % triple]
args = base_configure_args + [WithProperties("--prefix=%%(builddir)s/%s" % llvm_1_installdir)]
args += builders_util.getConfigArgs(stage1_config)
if not clean:
f.addStep(SetProperty(name="Makefile_isready",
workdir=llvm_1_objdir,
command=["sh", "-c",
"test -e Makefile.config && echo OK || echo Missing"],
flunkOnFailure=False,
property="exists_Makefile"))
f.addStep(Configure(command=args,
workdir=llvm_1_objdir,
description=['configuring',stage1_config],
descriptionDone=['configure',stage1_config],
env=merged_env,
doStepIf=lambda step: step.build.getProperty("exists_Makefile") != "OK"))
# Make clean if using in-dir builds.
if clean and llvm_srcdir == llvm_1_objdir:
f.addStep(WarningCountingShellCommand(name="clean-llvm",
command=[make, "clean"],
haltOnFailure=True,
description="cleaning llvm",
descriptionDone="clean llvm",
workdir=llvm_1_objdir,
doStepIf=clean,
env=merged_env))
if extra_clean_step:
f.addStep(extra_clean_step)
f.addStep(WarningCountingShellCommand(name="compile",
command=['nice', '-n', '10',
make, WithProperties("-j%s" % jobs)],
haltOnFailure=True,
flunkOnFailure=not run_gdb,
description=["compiling", stage1_config],
descriptionDone=["compile", stage1_config],
workdir=llvm_1_objdir,
env=merged_env))
if examples:
f.addStep(WarningCountingShellCommand(name="compile.examples",
command=['nice', '-n', '10',
make, WithProperties("-j%s" % jobs),
"BUILD_EXAMPLES=1"],
haltOnFailure=True,
description=["compilinge", stage1_config, "examples"],
descriptionDone=["compile", stage1_config, "examples"],
workdir=llvm_1_objdir,
env=merged_env))
clangTestArgs = '-v -j %s' % jobs
if valgrind:
clangTestArgs += ' --vg'
if valgrindLeakCheck:
clangTestArgs += ' --vg-leak'
clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp'
extraTestDirs = ''
if run_cxx_tests:
extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests'
if test:
f.addStep(lit_test_command.LitTestCommand(name='check-all',
command=[make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)],
flunkOnFailure=not run_gdb,
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_1_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# Install llvm and clang.
if llvm_1_installdir:
f.addStep(ShellCommand(name="rm-install.clang.stage1",
command=["rm", "-rf", llvm_1_installdir],
haltOnFailure=True,
description=["rm install dir", "clang"],
workdir=".",
env=merged_env))
f.addStep(WarningCountingShellCommand(name="install.clang.stage1",
command = ['nice', '-n', '10',
make, 'install-clang'],
haltOnFailure=True,
description=["install", "clang",
stage1_config],
workdir=llvm_1_objdir,
env=merged_env))
if run_gdb or run_gcc or run_modern_gdb:
ignores = getClangTestsIgnoresFromPath(os.path.expanduser('~/public/clang-tests'), 'clang-x86_64-darwin10')
install_prefix = "%%(builddir)s/%s" % llvm_1_installdir
if run_gdb:
addClangGDBTests(f, ignores, install_prefix)
if run_modern_gdb:
addModernClangGDBTests(f, jobs, install_prefix)
if run_gcc:
addClangGCCTests(f, ignores, install_prefix)
if not useTwoStage:
if package_dst:
name = WithProperties(
"%(builddir)s/" + llvm_1_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_1_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(ShellCommand(name='pkg.upload',
description="upload root",
command=["scp", name,
WithProperties(
package_dst + "/%(buildername)s")],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
# Clean up llvm (stage 2).
if clean:
f.addStep(ShellCommand(name="rm-llvm.obj.stage2",
command=["rm", "-rf", llvm_2_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm", "(stage 2)"],
workdir=".",
env=merged_env))
# Configure llvm (stage 2).
args = base_configure_args + [WithProperties("--prefix=%(builddir)s/" + llvm_2_installdir)]
args += builders_util.getConfigArgs(stage2_config)
local_env = dict(merged_env)
local_env.update({
'CC' : WithProperties("%%(builddir)s/%s/bin/clang" % llvm_1_installdir),
'CXX' : WithProperties("%%(builddir)s/%s/bin/clang++" % llvm_1_installdir)})
f.addStep(Configure(name="configure.llvm.stage2",
command=args,
haltOnFailure=True,
workdir=llvm_2_objdir,
description=["configure", "llvm", "(stage 2)",
stage2_config],
env=local_env))
# Build llvm (stage 2).
f.addStep(WarningCountingShellCommand(name="compile.llvm.stage2",
command=['nice', '-n', '10',
make, WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["compiling", "(stage 2)",
stage2_config],
descriptionDone=["compile", "(stage 2)",
stage2_config],
workdir=llvm_2_objdir,
env=merged_env))
if test:
f.addStep(lit_test_command.LitTestCommand(name='check-all',
command=[make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)],
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_2_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# Install clang (stage 2).
f.addStep(ShellCommand(name="rm-install.clang.stage2",
command=["rm", "-rf", llvm_2_installdir],
haltOnFailure=True,
description=["rm install dir", "clang"],
workdir=".",
env=merged_env))
f.addStep(WarningCountingShellCommand(name="install.clang.stage2",
command = ['nice', '-n', '10',
make, 'install-clang'],
haltOnFailure=True,
description=["install", "clang",
"(stage 2)"],
workdir=llvm_2_objdir,
env=merged_env))
if package_dst:
name = WithProperties(
"%(builddir)s/" + llvm_2_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_2_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(ShellCommand(name='pkg.upload',
description="upload root",
command=["scp", name,
WithProperties(
package_dst + "/%(buildername)s")],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
def getClangAndLLDBuildFactory(
clean=True,
env=None,
withLLD=True,
extraCmakeOptions=None,
extraCompilerOptions=None,
buildWithSanitizerOptions=None,
triple=None,
isMSVC=False,
# Choose VS tools to build with. For example,
# "autodetect" to find the latest installed Visual Studio, or
# %VS140COMNTOOLS% to selects the 2015 toolchain.
vs=None,
target_arch=None,
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
extraLitArgs=None):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not (isMSVC
and vs), "Can't have custom builder env vars with VS"
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Get LLVM, Clang and LLD.
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Sanitizer runtime in compiler-rt, and cannot be built with sanitizer compiler.
if buildWithSanitizerOptions is None:
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if withLLD:
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='%s/tools/lld' % llvm_srcdir))
# Clean directory, if requested.
if clean:
shellCommand = ["rm", "-rf", llvm_objdir]
if isMSVC:
shellCommand = ["rmdir", "/S", "/Q", llvm_objdir]
f.addStep(
ShellCommand(name="rm-llvm_objdir",
command=shellCommand,
haltOnFailure=False,
flunkOnFailure=False,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
# Create configuration files with cmake.
options = ["-Wdocumentation", "-Wno-documentation-deprecated-sync"]
if isMSVC:
options = []
if extraCompilerOptions:
options += extraCompilerOptions
if buildWithSanitizerOptions:
options += buildWithSanitizerOptions
lit_args = ["-v"]
if extraLitArgs:
lit_args += extraLitArgs
lit_args = ["-DLLVM_LIT_ARGS=\"%s\"" % " ".join(lit_args)]
cmakeCommand = [
"cmake", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_ENABLE_ASSERTIONS=ON"
]
if buildWithSanitizerOptions:
cmakeCommand += [
"-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++"
]
if triple:
cmakeCommand += ["-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % triple]
if extraCmakeOptions:
assert not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extraCmakeOptions), \
"Please use extraLitArgs for LIT arguments instead of defining them in extraCmakeOptions."
cmakeCommand += extraCmakeOptions
if not isMSVC:
cmakeCommand += [
"-DCMAKE_C_FLAGS=\"%s\"" % (" ".join(options)),
"-DCMAKE_CXX_FLAGS=\"-std=c++11 %s\"" % (" ".join(options)),
]
cmakeCommand += lit_args
cmakeCommand += ["-GNinja", "../%s" % llvm_srcdir]
if isMSVC and vs:
# Set up VS environment, if requested.
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand)),
workdir=llvm_objdir,
env=merged_env))
# Build everything.
f.addStep(
NinjaCommand(name="build",
prefixCommand=prefixCommand,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
# Test everything.
f.addStep(
NinjaCommand(name="test",
prefixCommand=prefixCommand,
targets=["check-all"],
haltOnFailure=True,
description=["test"],
workdir=llvm_objdir,
env=merged_env))
return f
def getClangAndLLDBuildFactory(
clean=True,
env=None,
withLLD=True,
extraCmakeOptions=None,
extraCompilerOptions=None,
buildWithSanitizerOptions=None,
triple=None,
isMSVC=False,
# Choose VS tools to build with. For example,
# "autodetect" to find the latest installed Visual Studio, or
# %VS140COMNTOOLS% to selects the 2015 toolchain.
vs=None,
target_arch=None,
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
extraLitArgs=None
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not (isMSVC and vs), "Can't have custom builder env vars with VS"
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Get LLVM, Clang and LLD.
f.addStep(SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Sanitizer runtime in compiler-rt, and cannot be built with sanitizer compiler.
if buildWithSanitizerOptions is None:
f.addStep(SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if withLLD:
f.addStep(SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='%s/tools/lld' % llvm_srcdir))
# Clean directory, if requested.
if clean:
shellCommand = ["rm", "-rf", llvm_objdir]
if isMSVC:
shellCommand = ["rmdir", "/S", "/Q", llvm_objdir]
f.addStep(ShellCommand(name="rm-llvm_objdir",
command=shellCommand,
haltOnFailure=False,
flunkOnFailure=False,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
# Create configuration files with cmake.
options = ["-Wdocumentation", "-Wno-documentation-deprecated-sync"]
if isMSVC:
options = []
if extraCompilerOptions:
options += extraCompilerOptions
if buildWithSanitizerOptions:
options += buildWithSanitizerOptions
lit_args = ["-v"]
if extraLitArgs:
lit_args += extraLitArgs
lit_args = ["-DLLVM_LIT_ARGS=\"%s\"" % " ".join(lit_args)]
cmakeCommand = [
"cmake",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_ENABLE_ASSERTIONS=ON"
]
if buildWithSanitizerOptions:
cmakeCommand += [
"-DCMAKE_C_COMPILER=clang",
"-DCMAKE_CXX_COMPILER=clang++"
]
if triple:
cmakeCommand += ["-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % triple]
if extraCmakeOptions:
assert not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extraCmakeOptions), \
"Please use extraLitArgs for LIT arguments instead of defining them in extraCmakeOptions."
cmakeCommand += extraCmakeOptions
if not isMSVC:
cmakeCommand += [
"-DCMAKE_C_FLAGS=\"%s\"" % (" ".join(options)),
"-DCMAKE_CXX_FLAGS=\"-std=c++11 %s\"" % (" ".join(options)),
]
cmakeCommand += lit_args
cmakeCommand += [
"-GNinja",
"../%s" % llvm_srcdir
]
if isMSVC and vs:
# Set up VS environment, if requested.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(ShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand)),
workdir=llvm_objdir,
env=merged_env))
# Build everything.
f.addStep(NinjaCommand(name="build",
prefixCommand=prefixCommand,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
# Test everything.
f.addStep(NinjaCommand(name="test",
prefixCommand=prefixCommand,
targets=["check-all"],
haltOnFailure=True,
description=["test"],
workdir=llvm_objdir,
env=merged_env))
return f
def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
checks = None,
install_dir = None,
clean = False,
extra_configure_args = None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env = None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs=r"""%VS140COMNTOOLS%"""
f = getLLVMBuildFactoryAndSVNSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(
f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(
f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f