def getCmakeWithNinjaBuildFactory(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
checks = None,
install_dir = None,
clean = False,
extra_configure_args = None,
env = None,
**kwargs):
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
f = getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
clean=clean,
extra_configure_args=cmake_args,
env=env,
**kwargs) # Pass through all the extra arguments.
addNinjaSteps(
f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def getFlangOutOfTreeBuildFactory(checks=None,
clean=False,
llvm_extra_configure_args=None,
flang_extra_configure_args=None,
env=None,
**kwargs):
f = getCmakeWithNinjaBuildFactory(
depends_on_projects=['llvm', 'mlir'],
obj_dir="build_llvm",
checks=[],
clean=clean,
extra_configure_args=llvm_extra_configure_args,
env=env,
**kwargs)
if checks is None:
checks = ['check-all']
# Make a local copy of the flang configure args, as we are going to modify that.
if flang_extra_configure_args:
flang_cmake_args = flang_extra_configure_args[:]
else:
flang_cmake_args = list()
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(flang_cmake_args, [
('-G', 'Ninja'),
])
flang_obj_dir = "build_flang"
flang_src_dir = "%s/flang" % f.monorepo_dir
# Add LLVM_DIR and MLIR_DIR to the CMake invocation.
llvm_dir = "%s/lib/cmake/llvm" % f.obj_dir
mlir_dir = "%s/lib/cmake/mlir" % f.obj_dir
CmakeCommand.applyRequiredOptions(
flang_cmake_args,
[
# We actually need the paths to be relative to the source directory,
# otherwise find_package can't locate the config files.
('-DLLVM_DIR:PATH=',
LLVMBuildFactory.pathRelativeTo(llvm_dir, flang_src_dir)),
('-DMLIR_DIR:PATH=',
LLVMBuildFactory.pathRelativeTo(mlir_dir, flang_src_dir)),
])
# We can't use addCmakeSteps as that would use the path in f.llvm_srcdir.
f.addStep(
CmakeCommand(name="cmake-configure-flang",
haltOnFailure=True,
description=["CMake", "configure", "flang"],
options=flang_cmake_args,
path=LLVMBuildFactory.pathRelativeTo(
flang_src_dir, flang_obj_dir),
env=env,
workdir=flang_obj_dir,
**kwargs))
addNinjaSteps(f,
obj_dir=flang_obj_dir,
checks=checks,
env=env,
stage_name="flang",
**kwargs)
return f
def getLibcxxAndAbiBuilder(f=None,
env=None,
additional_features=None,
cmake_extra_opts=None,
lit_extra_opts=None,
lit_extra_args=None,
check_libcxx_abilist=False,
check_libcxx_benchmarks=None,
depends_on_projects=None,
**kwargs):
if env is None:
env = {}
if additional_features is None:
additional_features = set()
if cmake_extra_opts is None:
cmake_extra_opts = {}
if lit_extra_opts is None:
lit_extra_opts = {}
if lit_extra_args is None:
lit_extra_args = []
if depends_on_projects is None:
depends_on_projects = ['libcxx', 'libcxxabi', 'libunwind']
src_root = 'llvm'
build_path = 'build'
if f is None:
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=src_root,
obj_dir=build_path,
**kwargs) # Pass through all the extra arguments.
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
build_path)
# Specify the max number of threads using properties so LIT doesn't use
# all the threads on the system.
litTestArgs = '-vv --show-unsupported --show-xfail --threads=%(jobs)s'
if lit_extra_args:
litTestArgs += ' ' + ' '.join(lit_extra_args)
if additional_features:
litTestArgs += (' --param=additional_features=' +
','.join(additional_features))
for key in lit_extra_opts:
litTestArgs += (' --param=' + key + '=' + lit_extra_opts[key])
cmake_opts = [properties.WithProperties('-DLLVM_LIT_ARGS=' + litTestArgs)]
for key in cmake_extra_opts:
cmake_opts.append('-D' + key + '=' + cmake_extra_opts[key])
# FIXME: The libc++ abilist's are generated in release mode with debug
# symbols Other configurations may contain additional non-inlined symbols.
if check_libcxx_abilist and not 'CMAKE_BUILD_TYPE' in cmake_extra_opts:
cmake_opts.append('-DCMAKE_BUILD_TYPE=RELWITHDEBINFO')
# Force libc++ to use the in-tree libc++abi unless otherwise specified.
if 'LIBCXX_CXX_ABI' not in cmake_extra_opts:
cmake_opts.append('-DLIBCXX_CXX_ABI=libcxxabi')
# Nuke/remake build directory and run CMake
f.addStep(
buildbot.steps.shell.ShellCommand(name='rm.builddir',
command=['rm', '-rf', build_path],
workdir=".",
haltOnFailure=False))
if not f.is_legacy_mode:
CmakeCommand.applyRequiredOptions(cmake_opts, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
f.addStep(
buildbot.steps.shell.ShellCommand(name='cmake',
command=['cmake', rel_src_dir] +
cmake_opts,
haltOnFailure=True,
workdir=build_path,
env=env))
# Build libcxxabi
jobs_flag = properties.WithProperties('-j%(jobs)s')
f.addStep(
buildbot.steps.shell.ShellCommand(
name='build.libcxxabi',
command=['make', jobs_flag, 'cxxabi'],
haltOnFailure=True,
workdir=build_path))
# Build libcxx
f.addStep(
buildbot.steps.shell.ShellCommand(name='build.libcxx',
command=['make', jobs_flag, 'cxx'],
haltOnFailure=True,
workdir=build_path))
# Test libc++abi
f.addStep(
LitTestCommand(name='test.libcxxabi',
command=['make', jobs_flag, 'check-cxxabi'],
description=['testing', 'libcxxabi'],
descriptionDone=['test', 'libcxxabi'],
workdir=build_path))
# Test libc++
f.addStep(
LitTestCommand(name='test.libcxx',
command=['make', jobs_flag, 'check-cxx'],
description=['testing', 'libcxx'],
descriptionDone=['test', 'libcxx'],
workdir=build_path))
if check_libcxx_abilist:
f.addStep(
buildbot.steps.shell.ShellCommand(
name='test.libcxx.abilist',
command=['make', 'check-cxx-abilist'],
description=['testing', 'libcxx', 'abi'],
descriptionDone=['test', 'libcxx', 'abi'],
workdir=build_path))
if check_libcxx_benchmarks:
# Build the libc++ benchmarks
f.addStep(
buildbot.steps.shell.ShellCommand(
name='build.libcxx.benchmarks',
command=['make', jobs_flag, 'cxx-benchmarks'],
haltOnFailure=True,
workdir=build_path))
# Run the benchmarks
f.addStep(
LitTestCommand(name='test.libcxx.benchmarks',
command=['make', jobs_flag, 'check-cxx-benchmarks'],
description=['testing', 'libcxx', 'benchmarks'],
descriptionDone=['test', 'libcxx', 'benchmarks'],
workdir=build_path))
return f
def getClangWithLTOBuildFactory(
depends_on_projects = None,
clean = False,
jobs = None,
extra_configure_args = None,
compare_last_2_stages = True,
lto = None, # The string gets passed to -flto flag as is. Like -flto=thin.
env = None,
**kwargs):
# Set defaults
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
# By default we link with LLD.
depends_on_projects = ['llvm', 'clang', 'lld']
if lto is None:
lto = 'ON'
if jobs is None:
jobs = "%(jobs)s"
if extra_configure_args is None:
extra_configure_args = []
else:
extra_configure_args = list(extra_configure_args)
# Make sure CMAKE_INSTALL_PREFIX and -G are not specified
# in the extra_configure_args. We set them internally as needed.
# TODO: assert extra_configure_args.
install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args)
assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build."
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects=depends_on_projects,
stage_objdirs=[
"build/stage1",
"build/stage2",
"build/stage3",
"build/stage4",
],
stage_installdirs=[
"install/stage1",
"install/stage2",
"install/stage3",
"install/stage4",
],
staged_compiler_idx = 1,
**kwargs)
# Consume is_legacy_mode if given.
# TODO: Remove this once legacy mode gets dropped.
kwargs.pop('is_legacy_mode', None)
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
# Get the source code.
f.addGetSourcecodeSteps()
# Build with the system compiler first
_addSteps4SystemCompiler(f,
stage_idx=0,
clean=cleanBuildRequested,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Then build the compiler we would use for the bootstrap.
_addSteps4StagedCompiler(f,
stage_idx=1,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Build all the remaining stages with exactly the same configuration.
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LTO=', lto),
])
# If we build LLD, we would link with LLD.
# Otherwise we link with the system linker.
if 'lld' in depends_on_projects:
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LLD=', 'ON'),
])
# The rest are test stages, which depend on the staged compiler we are ultimately after.
s = f.staged_compiler_idx + 1
staged_install = f.stage_installdirs[f.staged_compiler_idx]
for i in range(s, len(f.stage_objdirs[s:]) + s):
configure_args = extra_configure_args[:]
configure_args.append(
WithProperties(
"-DCMAKE_AR=%(workdir)s/" + staged_install + "/bin/llvm-ar"
))
configure_args.append(
WithProperties(
"-DCMAKE_RANLIB=%(workdir)s/" + staged_install + "/bin/llvm-ranlib"
))
_addSteps4StagedCompiler(f,
stage_idx=i,
use_stage_idx=f.staged_compiler_idx,
jobs=jobs,
extra_configure_args=configure_args,
env=merged_env)
if compare_last_2_stages:
# Compare the compilers built on the last 2 stages if requested.
diff_command = [
"diff",
"-q",
f.stage_installdirs[-2] + "/bin/clang",
f.stage_installdirs[-1] + "/bin/clang",
]
f.addStep(
ShellCommand(
name="compare-compilers",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs)-1),
"and",
"stage%d" % len(f.stage_installdirs),
"compilers",
],
haltOnFailure=False,
command=WithProperties(" ".join(diff_command)),
workdir=".",
env=merged_env
)
)
# Only if the compare-compilers step has failed.
def _prevStepFailed(step):
steps = step.build.getStatus().getSteps()
prev_step = steps[-2]
(result, _) = prev_step.getResults()
return (result == FAILURE)
dir1 = f.stage_objdirs[-2]
dir2 = f.stage_objdirs[-1]
inc_pattern = "-type f -not -name *.inc -printf '%f\n'"
find_cmd = "find %s %s" % (dir1, dir2)
diff_cmd = "diff -ru %s %s -x '*.tmp*' -X -" % (dir1, dir2)
# Note: Use a string here as we want the command executed by a shell.
diff_tablegen_inc_files_command = "%s %s | %s" % (find_cmd, inc_pattern, diff_cmd)
f.addStep(
ShellCommand(
name="compare-tablegen-inc-files",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs)-1),
"and",
"stage%d" % len(f.stage_installdirs),
"Tablegen inc files",
],
command=diff_tablegen_inc_files_command,
workdir=".",
env=merged_env,
doStepIf=_prevStepFailed,
)
)
return f
def _addSteps4SystemCompiler(
f,
stage_idx = 0,
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
# This stage could use incremental build.
# Clean stage1, only if requested.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
if not f.is_legacy_mode:
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Note: On this stage we do not care of warnings, as we build with
# a system toolchain and cannot control the environment.
# Warnings are likely, and we ignore them.
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
flunkOnWarnings=False,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")
))
# Build clang by the system compiler
f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["build stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Test stage1 compiler
f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
targets=["check-all"], # or "check-llvm", "check-clang"
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["test stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Install stage1 compiler
f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
def addCmakeSteps(
f,
cleanBuildRequested,
obj_dir,
install_dir = None,
extra_configure_args = None,
env = None,
stage_name = None,
**kwargs):
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if obj_dir is None:
obj_dir = f.obj_dir
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeTo(
install_dir,
obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
f.addStep(RemoveDirectory(name='clean-%s-dir' % install_dir,
dir=install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
if stage_name:
step_name = "cmake-configure-%s" % stage_name
else:
stage_name = ""
step_name = "cmake-configure"
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
f.addStep(CmakeCommand(name=step_name,
haltOnFailure=True,
description=["Cmake", "configure", stage_name],
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
checks = None,
install_dir = None,
clean = False,
extra_configure_args = None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env = None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs=r"""%VS140COMNTOOLS%"""
f = getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(
f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(
f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def getLLVMDocsBuildFactory(clean=False,
depends_on_projects=None,
extra_configure_args=None,
env=None,
**kwargs):
if depends_on_projects is None:
# All the projects by default.
_depends_on_projects = [
"llvm",
"clang",
"clang-tools-extra",
"libcxx",
"libcxxabi",
"libunwind",
"lld",
"lldb",
"flang",
"openmp",
"polly",
]
else:
# Make a local copy of depends_on_projects, as we are going to modify
# that.
_depends_on_projects = depends_on_projects[:]
# Some projects are interdependent for the purpose of documentation.
# Enforce the dependencies.
if ("clang-tools-extra" in _depends_on_projects or \
"lldb" in _depends_on_projects
) and "clang" not in _depends_on_projects:
_depends_on_projects.append("clang")
if "libcxx" in _depends_on_projects and \
"libcxxabi" not in _depends_on_projects:
_depends_on_projects.append("libcxxabi")
if "libcxxabi" in _depends_on_projects and \
"libcxx" not in _depends_on_projects:
_depends_on_projects.append("libcxx")
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
CmakeCommand.applyDefaultOptions(cmake_args, [
("-G", "Ninja"),
("-DLLVM_ENABLE_SPHINX=", "ON"),
("-DSPHINX_OUTPUT_HTML=", "ON"),
("-DSPHINX_OUTPUT_MAN=", "OFF"),
("-DSPHINX_WARNINGS_AS_ERRORS=", "OFF"),
("-DLLVM_ENABLE_ASSERTIONS=", "OFF"),
("-DCMAKE_BUILD_TYPE=", "Release"),
])
# Build docs for each of the projects this builder depends on
docs = [
llvm_docs[project] for project in llvm_docs.keys()
if project in _depends_on_projects
]
f = UnifiedTreeBuilder.getCmakeBuildFactory(
clean=clean,
depends_on_projects=_depends_on_projects,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
UnifiedTreeBuilder.addNinjaSteps(f,
targets=[d[0] for d in docs],
checks=[],
env=merged_env,
**kwargs)
# Publish just built documentation
for target, local_path, remote_path in docs:
f.addStep(
ShellCommand(
name="Publish {}".format(target),
description=[
"Publish",
"just",
"built",
"documentation",
"for",
"{}".format(target),
],
command=[
'rsync',
'-vrl',
'--delete',
'--force',
'--delay-updates',
'--delete-delay',
'--ignore-times',
'--checksum',
'-p',
'--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r',
"{}".format(local_path),
"lists.llvm.org:web/{}".format(remote_path),
],
env=merged_env,
))
return f
def getLLDWinBuildFactory(
clean = True,
# Default values for VS devenv and build configuration
vs = None, # What to run to configure Visual Studio utils.
target_arch = None, # Native.
extra_configure_args = None,
env = None):
# Set defaults
if vs is None:
vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015.
if extra_configure_args is None:
extra_configure_args = []
if env is None:
env = {}
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# If set up environment step is requested, do this now.
if vs:
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
# Always build with ninja.
cmake_options = ["-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmake_options.append('-DCMAKE_BUILD_TYPE=Release')
if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_WERROR=ON')
if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"')
cmake_options += extra_configure_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
warnOnWarnings=True,
options=cmake_options,
path="../%s" % f.llvm_srcdir,
env=env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld.
f.addStep(NinjaCommand(name='build lld',
haltOnFailure=True,
warnOnWarnings=True,
description='build lld',
workdir=f.llvm_objdir,
env=env))
# Test Lld
f.addStep(NinjaCommand(name='test lld',
targets=['lld-test'],
haltOnFailure=True,
warnOnWarnings=True,
description='test lld',
workdir=f.llvm_objdir,
env=env))
return f
def getLLDBuildFactory(
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Set defaults
if jobs is None:
jobs = "%(jobs)s"
if extra_configure_args is None:
extra_configure_args = []
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'CC' : "clang",
'CXX' : "clang++",
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=extra_configure_args,
path="../%s" % f.llvm_srcdir,
env=merged_env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld
f.addStep(ShellCommand(name="build_Lld",
command=['nice', '-n', '10',
'make', WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["build lld"],
env=merged_env,
workdir=f.llvm_objdir))
# Test Lld
f.addStep(ShellCommand(name="test_lld",
command=["make", "lld-test"],
haltOnFailure=True,
description=["test lld"],
env=merged_env,
workdir=f.llvm_objdir))
return f
def addCmakeSteps(
f,
cleanBuildRequested,
obj_dir,
install_dir = None,
extra_configure_args = None,
env = None,
stage_name = None,
**kwargs):
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
install_dir,
obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
f.addStep(RemoveDirectory(name='clean-%s-dir' % install_dir,
dir=install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
('-DLLVM_LIT_ARGS=', '"-v"'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
if stage_name:
step_name = "cmake-configure-%s" % stage_name
else:
stage_name = ""
step_name = "cmake-configure"
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
f.addStep(CmakeCommand(name=step_name,
description=["Cmake", "configure", stage_name],
options=cmake_args,
path=src_dir,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def getCmakeWithNinjaMultistageBuildFactory(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
checks = None,
install_dir = None,
clean = False,
extra_configure_args = None,
env = None,
stages=2,
stage_names=None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
assert stages > 1, "It should be at least 2 stages in a multistage build."
if stage_names is None:
stage_names = list()
for i in range(1, stages + 1):
stage_names.append("stage%s" % i)
else:
assert len(stage_names) == stages, "Please specify names for none or all of the requested stages."
if obj_dir is None:
obj_dir = "build"
if install_dir is None:
install_dir = "install"
if checks is None:
checks = ['check-all']
stage_objdirs = list()
stage_installdirs = list()
for s in stage_names:
stage_objdirs.append("%s/%s" % (obj_dir, s))
stage_installdirs.append("%s/%s" % (install_dir, s))
f = getLLVMBuildFactoryAndSVNSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
env=env,
stage_objdirs=stage_objdirs,
stage_installdirs=stage_installdirs,
stage_names=stage_names,
**kwargs) # Pass through all the extra arguments.
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if 'clang' in depends_on_projects:
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
])
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
# The stage 1 is special, though. We use the system compiler and
# do incremental build, unless a clean one has been requested.
cmake_args_stage1 = cmake_args[:]
CmakeCommand.applyDefaultOptions(cmake_args_stage1, [
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(
f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=stage_objdirs[0],
install_dir=stage_installdirs[0],
extra_configure_args=cmake_args_stage1,
env=env,
stage_name=stage_names[0],
**kwargs)
addNinjaSteps(
f,
obj_dir=stage_objdirs[0],
checks=checks,
install_dir=stage_installdirs[0],
env=env,
stage_name=stage_names[0],
**kwargs)
# Build the rest stage by stage, using just built compiler to compile
# the next stage.
CmakeCommand.applyDefaultOptions(cmake_args, [
# We should be warnings free when use just built compiler.
('-DLLVM_ENABLE_WERROR=', 'ON'),
])
# If we build LLD, we would link with LLD.
# Otherwise we link with a system linker.
if 'lld' in f.depends_on_projects:
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DLLVM_ENABLE_LLD=', 'ON'),
])
for stage_idx in range(1, stages):
# Directories to use in this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[stage_idx - 1]
# Configure the compiler to use in this stage.
cmake_args_stageN = cmake_args[:]
CmakeCommand.applyRequiredOptions(cmake_args_stageN, [
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args_stageN.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
))
cmake_args_stageN.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
))
addCmakeSteps(
f,
True, # We always do a clean build for the staged builds.
obj_dir=stage_objdirs[stage_idx],
install_dir=stage_installdirs[stage_idx],
extra_configure_args=cmake_args_stageN,
env=env,
stage_name=stage_names[stage_idx],
**kwargs)
addNinjaSteps(
f,
obj_dir=stage_objdirs[stage_idx],
checks=checks,
install_dir=stage_installdirs[stage_idx],
env=env,
stage_name=stage_names[stage_idx],
**kwargs)
return f
def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
checks = None,
install_dir = None,
clean = False,
extra_configure_args = None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env = None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs=r"""%VS140COMNTOOLS%"""
f = getLLVMBuildFactoryAndSVNSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(
f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(
f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def getABITestsuitBuildFactory(
clean = True,
depends_on_projects = None,
extra_configure_args = None, # Extra CMake args for all stages.
jobs = None, # Restrict a degree of parallelism if needed.
env = None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld']
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(NinjaCommand(name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Checkout the test-suite.
f.addStep(SVN(name='svn-test-suite',
mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir='test-suite'))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH' : WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"),
'PATH' : WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f
def getClangWithLTOBuildFactory(
depends_on_projects = None,
clean = False,
jobs = None,
extra_configure_args = None,
compare_last_2_stages = True,
lto = None, # The string gets passed to -flto flag as is. Like -flto=thin.
env = None):
# Set defaults
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
# By default we link with LLD.
depends_on_projects = ['llvm', 'clang', 'lld']
if lto is None:
lto = 'ON'
if jobs is None:
jobs = "%(jobs)s"
if extra_configure_args is None:
extra_configure_args = []
else:
extra_configure_args = list(extra_configure_args)
# Make sure CMAKE_INSTALL_PREFIX and -G are not specified
# in the extra_configure_args. We set them internally as needed.
# TODO: assert extra_configure_args.
install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args)
assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build."
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm.src",
stage_objdirs=[
"build/stage1",
"build/stage2",
"build/stage3",
"build/stage4",
],
stage_installdirs=[
"install/stage1",
"install/stage2",
"install/stage3",
"install/stage4",
],
staged_compiler_idx = 1)
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
# Do a clean checkout if requested.
f.addStep(RemoveDirectory(name='clean-src-dir',
dir=f.llvm_srcdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Get the source code.
f.addSVNSteps()
# Build with the system compiler first
_addSteps4SystemCompiler(f,
stage_idx=0,
clean=cleanBuildRequested,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Then build the compiler we would use for the bootstrap.
_addSteps4StagedCompiler(f,
stage_idx=1,
withLTOSupport=True,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Build all the remaining stages with exactly the same configuration.
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LTO=', lto),
])
# If we build LLD, we would link with LLD.
# Otherwise we link with the system linker.
if 'lld' in depends_on_projects:
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LLD=', 'ON'),
])
# The rest are test stages, which depend on the staged compiler we are ultimately after.
s = f.staged_compiler_idx + 1
staged_install = f.stage_installdirs[f.staged_compiler_idx]
for i in range(s, len(f.stage_objdirs[s:]) + s):
configure_args = extra_configure_args[:]
configure_args.append(
WithProperties(
"-DCMAKE_AR=%(workdir)s/" + staged_install + "/bin/llvm-ar"
))
configure_args.append(
WithProperties(
"-DCMAKE_RANLIB=%(workdir)s/" + staged_install + "/bin/llvm-ranlib"
))
_addSteps4StagedCompiler(f,
stage_idx=i,
use_stage_idx=f.staged_compiler_idx,
jobs=jobs,
extra_configure_args=configure_args,
env=merged_env)
if compare_last_2_stages:
# Compare the compilers built on the last 2 stages if requested.
diff_command = [
"diff",
"-q",
f.stage_installdirs[-2] + "/bin/clang",
f.stage_installdirs[-1] + "/bin/clang",
]
f.addStep(
ShellCommand(
name="compare-compilers",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs)-1),
"and",
"stage%d" % len(f.stage_installdirs),
"compilers",
],
haltOnFailure=False,
command=WithProperties(" ".join(diff_command)),
workdir=".",
env=merged_env
)
)
# Only if the compare-compilers step has failed.
def _prevStepFailed(step):
steps = step.build.getStatus().getSteps()
prev_step = steps[-2]
(result, _) = prev_step.getResults()
return (result == FAILURE)
dir1 = f.stage_objdirs[-2]
dir2 = f.stage_objdirs[-1]
inc_pattern = "-type f -not -name *.inc -printf '%f\n'"
find_cmd = "find %s %s" % (dir1, dir2)
diff_cmd = "diff -ru %s %s -x '*.tmp*' -X -" % (dir1, dir2)
# Note: Use a string here as we want the command executed by a shell.
diff_tablegen_inc_files_command = "%s %s | %s" % (find_cmd, inc_pattern, diff_cmd)
f.addStep(
ShellCommand(
name="compare-tablegen-inc-files",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs)-1),
"and",
"stage%d" % len(f.stage_installdirs),
"Tablegen inc files",
],
command=diff_tablegen_inc_files_command,
workdir=".",
env=merged_env,
doStepIf=_prevStepFailed,
)
)
return f
def getLLDBCMakeBuildFactory(
clean=False,
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
testTimeout=2400,
install=False):
############# PREPARING
# Global configurations
build_dir = 'build'
f = LLVMBuildFactory(depends_on_projects=["llvm", "clang", "lldb", "lld"],
obj_dir=build_dir)
env = {}
# Determine Slave Environment and Set MSVC environment.
if vs and vs != 'manual':
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractVSEnvironment))
env = Property('vs_env')
f.addGetSourcecodeSteps()
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
############# CLEANING
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
f.addStep(
steps.RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)
cmake_options = [
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
]
if python_source_dir:
cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_options += extra_cmake_args
f.addStep(
CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_options,
path=rel_src_dir,
env=env,
workdir=build_dir))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=env))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=env))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=testTimeout,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=env))
return f
def getABITestsuitBuildFactory(
clean=True,
depends_on_projects=None,
extra_configure_args=None, # Extra CMake args for all stages.
jobs=None, # Restrict a degree of parallelism if needed.
env=None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = [
'llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld'
]
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
cleanBuildRequested=cleanBuildRequested,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
# First of all, we shall checkout the latest test-suite.
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test-suite',
alwaysUseLatest=True,
**kwargs)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps(**kwargs)
UnifiedTreeBuilder.addCmakeSteps(f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
f.addStep(
NinjaCommand(
name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH':
WithProperties("%(workdir)s/" + f.llvm_srcdir +
"/utils/lit:${PYTHONPATH}"),
'PATH':
WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(
LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f
def getFactory(depends_on_projects=None,
targets=None,
checks=None,
clean=False,
extra_configure_args=None,
env=None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
# Consume is_legacy_mode if given.
# TODO: Remove this once legacy mode gets dropped.
kwargs.pop('is_legacy_mode', None)
if targets:
step_name = "build-%s" % ("-".join(targets))
step_description = ["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description = ["Build", "unified", "tree"]
f.addStep(
NinjaCommand(
name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(
NinjaCommand(
name="test-%s" % ("-".join(checks)),
targets=checks,
description=["Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy",
"LLD",
"executable",
"to",
"the",
"performance",
"test",
"suite",
],
command=["cp", "-aL", "./bin/ld.lld", "../lld-speed-test/ld.lld"],
workdir=f.obj_dir,
env=merged_env))
# Run the performance test suite.
perf_command = [
"python",
"%(workdir)s/lld-benchmark.py",
"--machine=%(slavename)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(name="performance-test-suite",
description=[
"LLD",
"performance",
"test",
"suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env))
return f
def getSphinxDocsBuildFactory(
llvm_html=False, # Build LLVM HTML documentation
llvm_man=False, # Build LLVM man pages
clang_html=False, # Build Clang HTML documentation
clang_tools_html=False, # Build Clang Extra Tools HTML documentation
lld_html=False, # Build LLD HTML documentation
libcxx_html=False, # Build Libc++ HTML documentation
libunwind_html=False, # Build libunwind HTML documentation
lldb_html=False, # Build LLDB HTML documentation
extra_configure_args=None,
**kwargs):
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults for the config flags.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_SPHINX=', 'ON'),
('-DSPHINX_OUTPUT_HTML=', 'ON'),
('-DSPHINX_OUTPUT_MAN=', 'ON'),
('-DLLDB_INCLUDE_TESTS=', 'OFF'),
('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
])
llvm_srcdir = 'llvm/src'
llvm_objdir = 'llvm/build'
depends_on_projects = ['llvm']
if clang_html or clang_tools_html or lldb_html:
depends_on_projects.append('clang')
if clang_tools_html:
depends_on_projects.append('clang-tools-extra')
if lld_html:
depends_on_projects.append('lld')
if lldb_html:
depends_on_projects.append('lldb')
if libcxx_html:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
if libunwind_html:
depends_on_projects.append('libunwind')
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
extra_configure_args=cmake_args,
**kwargs) # Pass through all the extra arguments.
if llvm_html:
f.addStep(
NinjaCommand(name="docs-llvm-html",
haltOnFailure=True,
description=["Build LLVM Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-llvm-html']))
if llvm_man:
f.addStep(
NinjaCommand(name="docs-llvm-man",
haltOnFailure=True,
description=["Build LLVM Sphinx man pages"],
workdir=llvm_objdir,
targets=['docs-llvm-man']))
if clang_html:
f.addStep(
NinjaCommand(name="docs-clang-html",
haltOnFailure=True,
description=["Build Clang Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-clang-html']))
if clang_tools_html:
f.addStep(
NinjaCommand(
name="docs-clang-tools-html",
haltOnFailure=True,
description=[
"Build Clang Extra Tools Sphinx HTML documentation"
],
workdir=llvm_objdir,
targets=['docs-clang-tools-html']))
if lld_html:
f.addStep(
NinjaCommand(name="docs-lld-html",
haltOnFailure=True,
description=["Build LLD Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lld-html']))
if lldb_html:
f.addStep(
NinjaCommand(name="docs-lldb-html",
haltOnFailure=True,
description=["Build LLDB Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lldb-html']))
if libcxx_html:
f.addStep(
NinjaCommand(
name="docs-libcxx-html",
haltOnFailure=True,
description=["Build Libc++ Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libcxx-html']))
if libunwind_html:
f.addStep(
NinjaCommand(
name="docs-libunwind-html",
haltOnFailure=True,
description=["Build libunwind Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libunwind-html']))
return f
def getFuchsiaToolchainBuildFactory(
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM": "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=[
"llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx",
"libcxxabi", "libunwind", "lld"
],
llvm_srcdir=src_dir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(
RemoveDirectory(name="clean-sdk", dir=sdk_dir, haltOnFailure=True))
f.addStep(
ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(
ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanBuildRequested = lambda step: step.build.getProperty("clean",
default=True)
# Clean up llvm build.
f.addStep(
RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
f.addStep(
RemoveDirectory(name="clean-llvm.install",
dir=install_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties("-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir))
cmake_options.append(WithProperties("-DFUCHSIA_SDK=%(workdir)s/" +
sdk_dir))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(
CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(
NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(
NinjaCommand(
name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(
NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def getCmakeWithNinjaMultistageBuildFactory(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
checks = None,
install_dir = None,
clean = False,
extra_configure_args = None,
env = None,
stages=2,
stage_names=None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
assert stages > 1, "It should be at least 2 stages in a multistage build."
if stage_names is None:
stage_names = list()
for i in range(1, stages + 1):
stage_names.append("stage%s" % i)
else:
assert len(stage_names) == stages, "Please specify names for none or all of the requested stages."
if obj_dir is None:
obj_dir = "build"
if install_dir is None:
install_dir = "install"
if checks is None:
checks = ['check-all']
stage_objdirs = list()
stage_installdirs = list()
for s in stage_names:
stage_objdirs.append("%s/%s" % (obj_dir, s))
stage_installdirs.append("%s/%s" % (install_dir, s))
f = getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
env=env,
stage_objdirs=stage_objdirs,
stage_installdirs=stage_installdirs,
stage_names=stage_names,
**kwargs) # Pass through all the extra arguments.
# Get the source code.
# We have consumed kwargs specific to this factory, so
# it is safe to pass all the remaining kwargs down.
f.addGetSourcecodeSteps(**kwargs)
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if 'clang' in depends_on_projects:
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
])
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
# The stage 1 is special, though. We use the system compiler and
# do incremental build, unless a clean one has been requested.
cmake_args_stage1 = cmake_args[:]
CmakeCommand.applyDefaultOptions(cmake_args_stage1, [
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(
f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=stage_objdirs[0],
install_dir=stage_installdirs[0],
extra_configure_args=cmake_args_stage1,
env=env,
stage_name=stage_names[0],
**kwargs)
addNinjaSteps(
f,
obj_dir=stage_objdirs[0],
checks=checks,
install_dir=stage_installdirs[0],
env=env,
stage_name=stage_names[0],
**kwargs)
# Build the rest stage by stage, using just built compiler to compile
# the next stage.
CmakeCommand.applyDefaultOptions(cmake_args, [
# We should be warnings free when use just built compiler.
('-DLLVM_ENABLE_WERROR=', 'ON'),
])
# If we build LLD, we would link with LLD.
# Otherwise we link with a system linker.
if 'lld' in f.depends_on_projects:
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DLLVM_ENABLE_LLD=', 'ON'),
])
for stage_idx in range(1, stages):
# Directories to use in this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeTo(f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[stage_idx - 1]
# Configure the compiler to use in this stage.
cmake_args_stageN = cmake_args[:]
CmakeCommand.applyRequiredOptions(cmake_args_stageN, [
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args_stageN.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
))
cmake_args_stageN.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
))
addCmakeSteps(
f,
True, # We always do a clean build for the staged builds.
obj_dir=stage_objdirs[stage_idx],
install_dir=stage_installdirs[stage_idx],
extra_configure_args=cmake_args_stageN,
env=env,
stage_name=stage_names[stage_idx],
**kwargs)
addNinjaSteps(
f,
obj_dir=stage_objdirs[stage_idx],
checks=checks,
install_dir=stage_installdirs[stage_idx],
env=env,
stage_name=stage_names[stage_idx],
**kwargs)
return f
def getCUDATestsuiteBuildFactory(
externals, # Directory with CUDA, thrust and gcc versions for testing.
always_clean=True,
test=False,
useTwoStage=False,
cmake='cmake',
extra_cmake_args=None, # Extra CMake args for all stages.
extra_ts_cmake_args=None, # extra cmake args for testsuite.
jobs=None,
cuda_jobs=1, # number of simultaneous CUDA apps to run
env=None, # Environmental variables for all steps.
enable_thrust_tests=False,
split_thrust_tests=False, # Each thrust test is a separate executable.
run_thrust_tests=False,
enable_libcxx=True, # checkout/build libcxx to test with.
gpu_arch_list=None,
gpu_devices=None, # List of devices to make visible to CUDA
stage1_config='Release',
stage2_config='Release'):
if extra_cmake_args is None:
extra_cmake_args = []
if extra_ts_cmake_args is None:
extra_ts_cmake_args = []
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory.
stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory.
stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory.
install_dir = 'clang.install'
if useTwoStage:
clang_build_dir = stage2_build_dir
else:
clang_build_dir = stage1_build_dir
# Build clang.
f = ClangBuilder.getClangCMakeBuildFactory(
clean=always_clean,
test=test,
cmake=cmake,
extra_cmake_args=extra_cmake_args,
jobs=jobs,
env=merged_env,
useTwoStage=useTwoStage,
stage1_config=stage1_config,
stage2_config=stage2_config,
checkout_clang_tools_extra=False,
checkout_compiler_rt=False,
checkout_libcxx=enable_libcxx,
checkout_lld=False,
checkout_test_suite=True)
cuda_test_env = {
'PYTHONPATH': WithProperties("%(workdir)s/" + source_dir +
"/utils/lit:${PYTHONPATH}"),
'DESTDIR': WithProperties("%(workdir)s/" + install_dir),
'PATH': WithProperties("%(workdir)s/" + install_dir +
"/usr/local/bin:${PATH}"),
}
merged_env.update(cuda_test_env)
ts_build_dir = 'test-suite-build'
f.addStep(
RemoveDirectory(name="Remove old clang install directory",
dir=install_dir))
# Install clang into directory pointed by $DESTDIR
f.addStep(NinjaCommand(
name='ninja install clang',
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=split("installing clang"),
descriptionDone=split("Clang installation is done."),
workdir=clang_build_dir,
env=merged_env))
# Completely remove test suite build dir.
f.addStep(
RemoveDirectory(name="Remove old test-suite build directory",
dir=ts_build_dir))
if extra_ts_cmake_args:
cmake_args = extra_ts_cmake_args[:]
else:
cmake_args = []
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cmake_args.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" +
clang_build_dir + "/bin/clang++"
))
cmake_args.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang"
))
cmake_args.append('-DTEST_SUITE_SUBDIRS=External'),
# Limit to External tests only.
if externals:
cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals)
if split_thrust_tests:
cmake_args.append('-DTHRUST_SPLIT_TESTS=1')
if gpu_arch_list:
cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list))
if cuda_jobs:
cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs)
# Then do fresh cmake configuration.
f.addStep(CmakeCommand(name='cmake test-suite',
description='cmake test-suite',
haltOnFailure=True,
options=cmake_args,
path="../test/test-suite",
workdir=ts_build_dir,
env=merged_env))
# Always build simple CUDA tests. They serve as compilation
# smoketests and will fail quickly if compiler has obvious issues
# compiling CUDA files.
f.addStep(NinjaCommand(
name='ninja build simple CUDA tests',
targets=["cuda-tests-simple"],
jobs=jobs,
haltOnFailure=True,
description=split("building simple CUDA tests"),
descriptionDone=split("simple CUDA tests built."),
workdir=ts_build_dir,
env=merged_env))
# Limit GPUs visible to CUDA.
if gpu_devices:
for gpu_id in gpu_devices:
# make ID a string as it may be either an integer or a UUID string.
gpu_id = str(gpu_id)
gpu_env = dict(merged_env)
gpu_env["CUDA_VISIBLE_DEVICES"] = gpu_id
f.addStep(NinjaCommand(
name='run simple CUDA tests on gpu %s' % gpu_id,
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=split("Running simple CUDA tests on GPU %s" % gpu_id),
descriptionDone=split("simple CUDA tests on GPU %s done." % gpu_id),
workdir=ts_build_dir,
env=gpu_env))
else:
f.addStep(NinjaCommand(
name='run simple CUDA tests',
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=split("Running simple CUDA tests"),
descriptionDone=split("simple CUDA tests done."),
workdir=ts_build_dir,
env=merged_env))
# If we've enabled thrust tests, build them now.
# WARNING: This takes a lot of time to build.
if (enable_thrust_tests):
f.addStep(NinjaCommand(
name='ninja build thrust',
targets=["cuda-tests-thrust"],
jobs=jobs,
haltOnFailure=True,
description=split("building thrust tests"),
descriptionDone=split("thrust tests built."),
workdir=ts_build_dir,
env=merged_env))
# Run them. That also takes a while.
# TODO(tra) we may want to run more than one instance so one
# can be compiling tests while another is running them on GPU.
if run_thrust_tests:
f.addStep(NinjaCommand(
name='run all CUDA tests',
targets=["check"],
jobs=1, # lit will parallelize the jobs.
haltOnFailure=True,
description=split("running all CUDA tests."),
descriptionDone=split("all cuda tests done."),
workdir=ts_build_dir,
env=merged_env))
return f
def getCmakeWithMSVCBuildFactory(
clean = True, # False for incremental builds.
depends_on_projects = None, # List of projects to listen.
cmake_cache = None, # Path to a cmake cache file.
extra_configure_args = None, # Extra CMake args if any.
llvm_srcdir = None, # Source code root directory.
obj_dir = None, # Build tree root directory.
install_dir = None, # Directory to install the results to.
checks = None, # List of checks to test the build.
checks_on_target = None, # [(<name>,[<command tokens>])] array of
# check name and command to run on target.
jobs = None, # Restrict a degree of parallelism.
env = None, # Environmental variables for all steps.
# VS tools environment variable if using MSVC.
# For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select
# the VS 2015 toolchain, or empty string if environment is already set.
vs=None,
**kwargs):
if vs is None:
# We autodetect Visual Studio, unless otherwise is requested.
vs = "autodetect"
if install_dir is None:
install_dir = 'install'
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not vs, "Cannot have custom builder env vars with VS setup."
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if depends_on_projects is None:
depends_on_projects = [
'llvm',
'compiler-rt',
'clang',
'clang-tools-extra',
'libunwind',
'libcxx',
'libcxxabi',
'lld',
]
if checks is None:
# Check only host-side tools. Target-side tests should run on a target.
checks=[
"check-llvm",
"check-clang",
"check-lld",
]
source_remove_requested = lambda step: step.build.getProperty("clean")
clean_build_requested = lambda step: \
clean or \
step.build.getProperty("clean", \
default=step.build.getProperty("clean_obj") \
)
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=clean_build_requested,
**kwargs) # Pass through all the extra arguments.
# Remove the source code tree if requested.
# NOTE: Somehow RemoveDirectory buildbot command often fails on Windows,
# as somthing keeps a lock. We use rm command instead realying on a shell
# to support that.
f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir,
command=['rm','-rf', f.monorepo_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove the source code',
workdir='.',
env=merged_env,
doStepIf=source_remove_requested))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Remove the build directory if requested.
f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir,
command=['rm','-rf', f.obj_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove build directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
if vs:
# Configure MSVC environment if requested.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, None),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Since this is a build of a cross toolchain, we build only the host-side
# tools first by the host system compiler. Libraries will be cross-compiled.
cmake_args.append(InterpolateToPosixPath(
'-DLLVM_AR=%(workdir)s/' + f.obj_dir + '/bin/llvm-ar.exe')),
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'),
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeTo(
install_dir,
f.obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
# Remove the build directory if requested.
f.addStep(ShellCommand(name='clean-%s-dir' % install_dir,
command=['rm','-rf', install_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove install directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
src_dir_rel = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)
# Add given cmake cache at the very end.
if cmake_cache:
cmake_args.append('-C%s' % cmake_cache)
f.addStep(CmakeCommand(name="cmake-configure",
haltOnFailure=True,
description=["Cmake", "configure"],
options=cmake_args,
path=src_dir_rel,
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir,
haltOnFailure=True,
description=["Build", f.monorepo_dir],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Test the components if requested one check at a time.
for check in checks:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=["ninja", WithProperties(check)],
description=[
"Testing", "just", "built", "components", "for", check],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Run commands on a target if requested.
if checks_on_target:
for check, cmd in checks_on_target:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=cmd,
description=[
"Testing", "just", "built", "components", "for", check],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
if install_dir:
f.addStep(NinjaCommand(name="install-all",
haltOnFailure=True,
targets=["install"],
description=[
"Install", "just", "built", "components"],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
return f
def getOpenMPCMakeBuildFactory(
jobs = '%(jobs)s', # Number of concurrent jobs.
clean = True, # "clean" step is requested if true
env = None, # Environmental variables for all steps.
ompt = False, # Whether to enable the OpenMP Tools Interface.
test = True, # Test the built libraries.
depends_on_projects = None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
llvm_srcdir = 'llvm.src'
llvm_builddir = 'llvm.build'
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
if depends_on_projects is None:
# Monorepo configuration requires llvm and clang to get cmake work.
depends_on_projects = ['llvm', 'clang', 'openmp']
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_builddir,
cleanBuildRequested=cleanBuildRequested,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(
ShellCommand(
name = 'clean',
command = ['rm', '-rf', f.obj_dir],
warnOnFailure = True,
description = ['clean'],
doStepIf = cleanBuildRequested,
workdir = '.',
env = merged_env))
# Configure LLVM and OpenMP (and Clang, if requested).
cmake_args = ['cmake', '-G', 'Ninja']
cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON']
if ompt:
cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON']
if test:
lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs
cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)]
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')})
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir)
f.addStep(CmakeCommand(name='configure-openmp',
description=['configure','openmp'],
options=cmake_args,
path=src_dir,
env=merged_env,
workdir=f.obj_dir,
haltOnFailure=True,
**kwargs # Pass through all the extra arguments.
))
# Build OpenMP runtime libraries.
f.addStep(
NinjaCommand(
name = 'compile-openmp',
description = 'compile openmp',
workdir = f.obj_dir,
env = merged_env,
haltOnFailure=True))
# Test OpenMP runtime libraries, if requested.
if test:
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')})
ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)]
f.addStep(
LitTestCommand(
name = 'test-openmp',
command = ninja_test_args + ['check-openmp'],
description = 'test openmp',
workdir = f.obj_dir,
env = merged_env,
haltOnFailure=True))
return f
def _addSteps4StagedCompiler(
f,
stage_idx = 1,
use_stage_idx = -1,
jobs = None,
extra_configure_args = None,
env = None):
if use_stage_idx < 0:
use_stage_idx = stage_idx - 1
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[use_stage_idx]
# Always do a clean build for the staged compiler.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
))
cmake_args.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
))
if not f.is_legacy_mode:
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")
))
# Build clang by the staged compiler
f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
description=["build stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Test just built compiler
f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
targets=["check-all"],
jobs=jobs,
haltOnFailure=True,
description=["test stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Install just built compiler
f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
def _addSteps4SystemCompiler(
f,
stage_idx = 0,
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
# This stage could use incremental build.
# Clean stage1, only if requested.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
# Note: On this stage we do not care of warnings, as we build with
# a system toolchain and cannot control the environment.
# Warnings are likely, and we ignore them.
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
flunkOnWarnings=False,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")
))
# Build clang by the system compiler
f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["build stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Test stage1 compiler
f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
targets=["check-all"], # or "check-llvm", "check-clang"
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["test stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Install stage1 compiler
f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
def _getClangCMakeBuildFactory(
is_legacy_mode=False,
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=None,
testsuite_flags=None,
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env=None,
extra_cmake_args=None,
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=None):
############# PREPARING
if nt_flags is None:
nt_flags = []
if testsuite_flags is None:
testsuite_flags = []
if env is None:
env = {}
if extra_cmake_args is None:
extra_cmake_args = []
if trigger_after_stage1 is None:
trigger_after_stage1 = []
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
# Some projects are not a part of the monorepo.
# So, depending on the legacy mode, we
# would have to checkout them differently.
if is_legacy_mode and (runTestSuite or checkout_test_suite):
depends_on_projects.append('lnt')
depends_on_projects.append('test-suite')
f = LLVMBuildFactory(is_legacy_mode=is_legacy_mode,
depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
# If we get the source code form the monorepo,
# we need to checkout the latest code for LNT
# and the test-suite separately. Le's do this first,
# so we wouldn't poison got_revision property.
if not is_legacy_mode and (runTestSuite or checkout_test_suite):
f.addGetSourcecodeForProject(project='lnt',
src_dir='test/lnt',
alwaysUseLatest=True)
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test/test-suite',
alwaysUseLatest=True)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
if not f.is_legacy_mode:
CmakeCommand.applyRequiredOptions(extra_cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
stage1_build)
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", rel_src_dir,
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(
f.llvm_srcdir, stage2_build)
cmake_cmd2 = [
'env',
WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
cxx), cmake, "-G", "Ninja", rel_src_dir,
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def _addSteps4StagedCompiler(
f,
stage_idx = 1,
use_stage_idx = -1,
withLTOSupport = False,
jobs = None,
extra_configure_args = None,
env = None):
if use_stage_idx < 0:
use_stage_idx = stage_idx - 1
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[use_stage_idx]
# Always do a clean build for the staged compiler.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if withLTOSupport:
CmakeCommand.applyDefaultOptions(cmake_args, [
# LTO Plugin dependency:
('-DLLVM_BINUTILS_INCDIR=', '/opt/binutils/include'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
))
cmake_args.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
))
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")
))
if withLTOSupport:
# Build LTO plugin if requested.
f.addStep(NinjaCommand(name="build-stage%s-LLVMgold.so" % stage_num,
targets=['lib/LLVMgold.so'],
jobs=jobs,
haltOnFailure=True,
description=["stage%s build LLVMgold.so" % stage_num],
env=env,
workdir=obj_dir,
))
# Build clang by the staged compiler
f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
description=["build stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Test just built compiler
f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
targets=["check-all"],
jobs=jobs,
haltOnFailure=True,
description=["test stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Install just built compiler
f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
def getFactory(
depends_on_projects = None,
targets = None,
checks = None,
clean = False,
extra_configure_args = None,
env = None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
if targets:
step_name = "build-%s" % ("-".join(targets))
step_description=["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description=["Build", "unified", "tree"]
f.addStep(NinjaCommand(name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(NinjaCommand(name="test-%s" % ("-".join(checks)),
targets=checks,
description=[
"Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy", "LLD", "executable", "to", "the", "performance",
"test", "suite",
],
command=[
"cp", "-aL", "./bin/ld.lld", "../lld-speed-test/ld.lld"
],
workdir=f.obj_dir,
env=merged_env
)
)
# Run the performance test suite.
perf_command = [
"python",
"%(workdir)s/lld-benchmark.py",
"--machine=%(slavename)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(
name="performance-test-suite",
description=[
"LLD", "performance","test","suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env
)
)
return f
def getCUDATestsuiteBuildFactory(
externals, # Directory with CUDA, thrust and gcc versions for testing.
always_clean=True,
test=False,
useTwoStage=False,
cmake='cmake',
extra_cmake_args=None, # Extra CMake args for all stages.
extra_ts_cmake_args=None, # extra cmake args for testsuite.
jobs=None,
cuda_jobs=1, # number of simultaneous CUDA apps to run
env=None, # Environmental variables for all steps.
enable_thrust_tests=False,
split_thrust_tests=False, # Each thrust test is a separate executable.
run_thrust_tests=False,
enable_libcxx=True, # checkout/build libcxx to test with.
gpu_arch_list=None,
gpu_devices=None, # List of devices to make visible to CUDA
stage1_config='Release',
stage2_config='Release'):
if extra_cmake_args is None:
extra_cmake_args = []
if extra_ts_cmake_args is None:
extra_ts_cmake_args = []
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory.
stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory.
stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory.
if useTwoStage:
clang_build_dir = stage2_build_dir
else:
clang_build_dir = stage1_build_dir
# Build clang.
f = ClangBuilder.getClangCMakeBuildFactory(
clean=always_clean,
test=test,
cmake=cmake,
extra_cmake_args=extra_cmake_args,
jobs=jobs,
env=merged_env,
useTwoStage=useTwoStage,
stage1_config=stage1_config,
stage2_config=stage2_config,
checkout_libcxx=enable_libcxx,
checkout_test_suite=True)
cuda_test_env = {
'PYTHONPATH': WithProperties("%(workdir)s/" + source_dir +
"/utils/lit:${PYTHONPATH}"),
'PATH': WithProperties("%(workdir)s/" + clang_build_dir +
"/bin:${PATH}"),
}
# Limit GPUs visible to CUDA.
if gpu_devices:
cuda_test_env["CUDA_VISIBLE_DEVICES"] = ",".join(
str(x) for x in gpu_devices)
merged_env.update(cuda_test_env)
ts_build_dir = 'test-suite-build'
# Completely remove test suite build dir.
f.addStep(
RemoveDirectory(name="Remove old test-suite build directory",
dir=ts_build_dir))
if extra_ts_cmake_args:
cmake_args = extra_ts_cmake_args[:]
else:
cmake_args = []
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cmake_args.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" +
clang_build_dir + "/bin/clang++"
))
cmake_args.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir + "/bin/clang"
))
cmake_args.append('-DTEST_SUITE_SUBDIRS=External'),
# Limit to External tests only.
if externals:
cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals)
if split_thrust_tests:
cmake_args.append('-DTHRUST_SPLIT_TESTS=1')
if gpu_arch_list:
cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list))
if cuda_jobs:
cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs)
# Then do fresh cmake configuration.
f.addStep(CmakeCommand(name='cmake test-suite',
description='cmake test-suite',
haltOnFailure=True,
options=cmake_args,
path="../test/test-suite",
workdir=ts_build_dir,
env=merged_env))
# Always build simple CUDA tests. They serve as compilation
# smoketests and will fail quickly if compiler has obvious issues
# compiling CUDA files.
f.addStep(NinjaCommand(
name='ninja build simple CUDA tests',
targets=["cuda-tests-simple"],
jobs=jobs,
haltOnFailure=True,
description=["building simple CUDA tests"],
descriptionDone=["simple CUDA tests built."],
workdir=ts_build_dir,
env=merged_env))
f.addStep(NinjaCommand(
name='run simple CUDA tests',
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=["Running simple CUDA tests"],
descriptionDone=["simple CUDA tests done."],
workdir=ts_build_dir,
env=merged_env))
# If we've enabled thrust tests, build them now.
# WARNING: This takes a lot of time to build.
if (enable_thrust_tests):
f.addStep(NinjaCommand(
name='ninja build thrust',
targets=["cuda-tests-thrust"],
jobs=jobs,
haltOnFailure=True,
description=["building thrust tests"],
descriptionDone=["thrust tests built."],
workdir=ts_build_dir,
env=merged_env))
# Run them. That also takes a while.
# TODO(tra) we may want to run more than one instance so one
# can be compiling tests while another is running them on GPU.
if run_thrust_tests:
f.addStep(NinjaCommand(
name='run all CUDA tests',
targets=["check"],
jobs=1, # lit will parallelize the jobs.
haltOnFailure=True,
description=["running all CUDA tests."],
descriptionDone=["all cuda tests done."],
workdir=ts_build_dir,
env=merged_env))
return f
