def getFuchsiaToolchainBuildFactory(
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM": "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=[
"llvm", "clang", "clang-tools-extra", "compiler-rt", "libcxx",
"libcxxabi", "libunwind", "lld"
],
llvm_srcdir=src_dir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(
steps.RemoveDirectory(name="clean-sdk",
dir=sdk_dir,
haltOnFailure=True))
f.addStep(
ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(
ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanBuildRequested = lambda step: step.build.getProperty("clean",
default=True)
# Clean up llvm build.
f.addStep(
steps.RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
f.addStep(
steps.RemoveDirectory(name="clean-llvm.install",
dir=install_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties("-DCMAKE_INSTALL_PREFIX=%(builddir)s/" + install_dir))
cmake_options.append(
WithProperties("-DFUCHSIA_SDK=%(builddir)s/" + sdk_dir))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(
CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(
NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(
NinjaCommand(
name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(
NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def getClangWithLTOBuildFactory(
depends_on_projects=None,
clean=False,
jobs=None,
extra_configure_args=None,
extra_configure_args_lto_stage=None,
compare_last_2_stages=True,
lto=None, # The string gets passed to -flto flag as is. Like -flto=thin.
env=None,
**kwargs):
# Set defaults
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
# By default we link with LLD.
depends_on_projects = ['llvm', 'clang', 'lld']
if lto is None:
lto = 'ON'
if extra_configure_args is None:
extra_configure_args = []
else:
extra_configure_args = list(extra_configure_args)
if extra_configure_args_lto_stage is None:
extra_configure_args_lto_stage = []
else:
extra_configure_args_lto_stage = list(extra_configure_args_lto_stage)
# Make sure CMAKE_INSTALL_PREFIX and -G are not specified
# in the extra_configure_args. We set them internally as needed.
# TODO: assert extra_configure_args.
install_prefix_specified = (any(
a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args)
or any(
a.startswith('-DCMAKE_INSTALL_PREFIX=')
for a in extra_configure_args_lto_stage))
assert not install_prefix_specified, "Please do not explicitly specify the install prefix for multi-stage build."
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects=depends_on_projects,
stage_objdirs=[
"build/stage1",
"build/stage2",
"build/stage3",
"build/stage4",
],
stage_installdirs=[
"install/stage1",
"install/stage2",
"install/stage3",
"install/stage4",
],
staged_compiler_idx=1,
**kwargs)
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
# Get the source code.
f.addGetSourcecodeSteps()
# Build with the system compiler first
_addSteps4SystemCompiler(f,
stage_idx=0,
clean=cleanBuildRequested,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Then build the compiler we would use for the bootstrap.
_addSteps4StagedCompiler(f,
stage_idx=1,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Build all the remaining stages with exactly the same configuration.
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LTO=', lto),
])
# If we build LLD, we would link with LLD.
# Otherwise we link with the system linker.
if 'lld' in depends_on_projects:
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LLD=', 'ON'),
])
# The rest are test stages, which depend on the staged compiler we are ultimately after.
s = f.staged_compiler_idx + 1
staged_install = f.stage_installdirs[f.staged_compiler_idx]
for i in range(s, len(f.stage_objdirs[s:]) + s):
configure_args = extra_configure_args[:] + extra_configure_args_lto_stage[:]
configure_args.append(
WithProperties("-DCMAKE_AR=%(builddir)s/" + staged_install +
"/bin/llvm-ar"))
configure_args.append(
WithProperties("-DCMAKE_RANLIB=%(builddir)s/" + staged_install +
"/bin/llvm-ranlib"))
_addSteps4StagedCompiler(f,
stage_idx=i,
use_stage_idx=f.staged_compiler_idx,
jobs=jobs,
extra_configure_args=configure_args,
env=merged_env)
if compare_last_2_stages:
# Compare the compilers built on the last 2 stages if requested.
diff_command = [
"diff",
"-q",
f.stage_installdirs[-2] + "/bin/clang",
f.stage_installdirs[-1] + "/bin/clang",
]
f.addStep(
ShellCommand(name="compare-compilers",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs) - 1),
"and",
"stage%d" % len(f.stage_installdirs),
"compilers",
],
haltOnFailure=False,
command=WithProperties(" ".join(diff_command)),
workdir=".",
env=merged_env))
# Only if the compare-compilers step has failed.
def _prevStepFailed(step):
steps = step.build.executedSteps
prev_step = steps[-2]
return (prev_step.results == FAILURE)
dir1 = f.stage_objdirs[-2]
dir2 = f.stage_objdirs[-1]
inc_pattern = "-type f -not -name *.inc -printf '%f\n'"
find_cmd = "find %s %s" % (dir1, dir2)
diff_cmd = "diff -ru %s %s -x '*.tmp*' -X -" % (dir1, dir2)
# Note: Use a string here as we want the command executed by a shell.
diff_tablegen_inc_files_command = "%s %s | %s" % (
find_cmd, inc_pattern, diff_cmd)
f.addStep(
ShellCommand(
name="compare-tablegen-inc-files",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs) - 1),
"and",
"stage%d" % len(f.stage_installdirs),
"Tablegen inc files",
],
command=diff_tablegen_inc_files_command,
workdir=".",
env=merged_env,
doStepIf=_prevStepFailed,
))
return f
def getSphinxDocsBuildFactory(
llvm_html=False, # Build LLVM HTML documentation
llvm_man=False, # Build LLVM man pages
clang_html=False, # Build Clang HTML documentation
clang_tools_html=False, # Build Clang Extra Tools HTML documentation
lld_html=False, # Build LLD HTML documentation
libcxx_html=False, # Build Libc++ HTML documentation
libunwind_html=False, # Build libunwind HTML documentation
lldb_html=False, # Build LLDB HTML documentation
extra_configure_args=None,
**kwargs):
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults for the config flags.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_SPHINX=', 'ON'),
('-DSPHINX_OUTPUT_HTML=', 'ON'),
('-DSPHINX_OUTPUT_MAN=', 'ON'),
('-DLLDB_INCLUDE_TESTS=', 'OFF'),
('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
])
llvm_srcdir = 'llvm/src'
llvm_objdir = 'llvm/build'
depends_on_projects = ['llvm']
if clang_html or clang_tools_html or lldb_html:
depends_on_projects.append('clang')
if clang_tools_html:
depends_on_projects.append('clang-tools-extra')
if lld_html:
depends_on_projects.append('lld')
if lldb_html:
depends_on_projects.append('lldb')
if libcxx_html:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
if libunwind_html:
depends_on_projects.append('libunwind')
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
extra_configure_args=cmake_args,
**kwargs) # Pass through all the extra arguments.
if llvm_html:
f.addStep(
NinjaCommand(name="docs-llvm-html",
haltOnFailure=True,
description=["Build LLVM Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-llvm-html']))
if llvm_man:
f.addStep(
NinjaCommand(name="docs-llvm-man",
haltOnFailure=True,
description=["Build LLVM Sphinx man pages"],
workdir=llvm_objdir,
targets=['docs-llvm-man']))
if clang_html:
f.addStep(
NinjaCommand(name="docs-clang-html",
haltOnFailure=True,
description=["Build Clang Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-clang-html']))
if clang_tools_html:
f.addStep(
NinjaCommand(
name="docs-clang-tools-html",
haltOnFailure=True,
description=[
"Build Clang Extra Tools Sphinx HTML documentation"
],
workdir=llvm_objdir,
targets=['docs-clang-tools-html']))
if lld_html:
f.addStep(
NinjaCommand(name="docs-lld-html",
haltOnFailure=True,
description=["Build LLD Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lld-html']))
if lldb_html:
f.addStep(
NinjaCommand(name="docs-lldb-html",
haltOnFailure=True,
description=["Build LLDB Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lldb-html']))
if libcxx_html:
f.addStep(
NinjaCommand(
name="docs-libcxx-html",
haltOnFailure=True,
description=["Build Libc++ Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libcxx-html']))
if libunwind_html:
f.addStep(
NinjaCommand(
name="docs-libunwind-html",
haltOnFailure=True,
description=["Build libunwind Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libunwind-html']))
return f
def getFactory(
depends_on_projects = None,
targets = None,
checks = None,
clean = False,
extra_configure_args = None,
llvm_srcdir = None,
obj_dir = None,
env = None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
if targets:
step_name = "build-{}".format("-".join(targets))
step_description=["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description=["Build", "unified", "tree"]
f.addStep(NinjaCommand(name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(NinjaCommand(name="test-{}".format("-".join(checks)),
targets=checks,
description=[
"Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy", "LLD", "executable", "to", "the", "performance",
"test", "suite",
],
command=[
"cp", "-aL", "./{}/bin/ld.lld".format(f.obj_dir), "./lld-speed-test/ld.lld"
],
env=merged_env,
workdir='.'
)
)
# Run the performance test suite.
perf_command = [
"python",
"%(builddir)s/lld-benchmark.py",
"--machine=%(workername)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(
name="performance-test-suite",
description=[
"LLD", "performance","test","suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env
)
)
return f
def getFactory(
depends_on_projects = None,
targets = None,
checks = None,
clean = False,
extra_configure_args = None,
env = None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
if targets:
step_name = "build-%s" % ("-".join(targets))
step_description=["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description=["Build", "unified", "tree"]
f.addStep(NinjaCommand(name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(NinjaCommand(name="test-%s" % ("-".join(checks)),
targets=checks,
description=[
"Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy", "LLD", "executable", "to", "the", "performance",
"test", "suite",
],
command=[
"cp", "-aL", "./bin/ld.lld", "../lld-speed-test/ld.lld"
],
workdir=f.obj_dir,
env=merged_env
)
)
# Run the performance test suite.
perf_command = [
"python",
"%(workdir)s/lld-benchmark.py",
"--machine=%(slavename)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(
name="performance-test-suite",
description=[
"LLD", "performance","test","suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env
)
)
return f
def getLLVMDocsBuildFactory(clean=False,
depends_on_projects=None,
extra_configure_args=None,
env=None,
**kwargs):
if depends_on_projects is None:
# All the projects by default.
_depends_on_projects = [
"llvm",
"clang",
"clang-tools-extra",
"libcxx",
"libcxxabi",
"libunwind",
"lld",
"lldb",
"flang",
"openmp",
"polly",
]
else:
# Make a local copy of depends_on_projects, as we are going to modify
# that.
_depends_on_projects = depends_on_projects[:]
# Some projects are interdependent for the purpose of documentation.
# Enforce the dependencies.
if ("clang-tools-extra" in _depends_on_projects or \
"lldb" in _depends_on_projects
) and "clang" not in _depends_on_projects:
_depends_on_projects.append("clang")
if "libcxx" in _depends_on_projects and \
"libcxxabi" not in _depends_on_projects:
_depends_on_projects.append("libcxxabi")
if "libcxxabi" in _depends_on_projects and \
"libcxx" not in _depends_on_projects:
_depends_on_projects.append("libcxx")
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
CmakeCommand.applyDefaultOptions(cmake_args, [
("-G", "Ninja"),
("-DLLVM_ENABLE_SPHINX=", "ON"),
("-DSPHINX_OUTPUT_HTML=", "ON"),
("-DSPHINX_OUTPUT_MAN=", "OFF"),
("-DSPHINX_WARNINGS_AS_ERRORS=", "OFF"),
("-DLLVM_ENABLE_ASSERTIONS=", "OFF"),
("-DCMAKE_BUILD_TYPE=", "Release"),
])
# Build docs for each of the projects this builder depends on
docs = [
llvm_docs[project] for project in llvm_docs.keys()
if project in _depends_on_projects
]
f = UnifiedTreeBuilder.getCmakeBuildFactory(
clean=clean,
depends_on_projects=_depends_on_projects,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
UnifiedTreeBuilder.addNinjaSteps(f,
targets=[d[0] for d in docs],
checks=[],
env=merged_env,
**kwargs)
# Publish just built documentation
for target, local_path, remote_path in docs:
f.addStep(
ShellCommand(
name="Publish %s" % target,
description=[
"Publish",
"just",
"built",
"documentation",
"fior",
"%s" % target,
],
command=[
'rsync',
'-vrl',
'--delete',
'--force',
'--delay-updates',
'--delete-delay',
'--ignore-times',
'--checksum',
'-p',
'--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r',
"%s" % local_path,
"lists.llvm.org:web/%s" % remote_path,
],
env=merged_env,
))
return f
def getLibcxxAndAbiBuilder(f=None, env=None,
cmake_extra_opts=None, lit_extra_opts=None,
lit_extra_args=None, check_libcxx_abilist=False,
check_libcxx_benchmarks=None,
depends_on_projects=None,
use_cache=None,
**kwargs):
if env is None:
env = {}
if cmake_extra_opts is None:
cmake_extra_opts = {}
if lit_extra_opts is None:
lit_extra_opts = {}
if lit_extra_args is None:
lit_extra_args = []
if depends_on_projects is None:
depends_on_projects = ['libcxx','libcxxabi','libunwind']
src_root = 'llvm'
build_path = 'build'
if f is None:
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=src_root,
obj_dir=build_path,
**kwargs) # Pass through all the extra arguments.
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, build_path)
# Specify the max number of threads using properties so LIT doesn't use
# all the threads on the system.
litTestArgs = '-vv --show-unsupported --show-xfail --threads=%(jobs)s'
if lit_extra_args:
litTestArgs += ' ' + ' '.join(lit_extra_args)
for key in lit_extra_opts:
litTestArgs += (' --param=' + key + '=' + lit_extra_opts[key])
cmake_opts = [properties.WithProperties('-DLLVM_LIT_ARGS='+litTestArgs)]
for key in cmake_extra_opts:
cmake_opts.append('-D' + key + '=' + cmake_extra_opts[key])
if use_cache:
libcxx_cache_dir = '%s/../libcxx/cmake/caches' % rel_src_dir
cache = '%s/%s' % (libcxx_cache_dir, use_cache)
cmake_opts.append('-C' + cache)
# FIXME: The libc++ abilist's are generated in release mode with debug
# symbols Other configurations may contain additional non-inlined symbols.
if check_libcxx_abilist and not 'CMAKE_BUILD_TYPE' in cmake_extra_opts:
cmake_opts.append('-DCMAKE_BUILD_TYPE=RELWITHDEBINFO')
# Force libc++ to use the in-tree libc++abi unless otherwise specified.
if 'LIBCXX_CXX_ABI' not in cmake_extra_opts:
cmake_opts.append('-DLIBCXX_CXX_ABI=libcxxabi')
# Nuke/remake build directory and run CMake
f.addStep(buildbot.steps.shell.ShellCommand(
name='rm.builddir', command=['rm', '-rf', build_path],
workdir=".",
haltOnFailure=False))
CmakeCommand.applyRequiredOptions(cmake_opts, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
f.addStep(buildbot.steps.shell.ShellCommand(
name='cmake', command=['cmake', rel_src_dir] + cmake_opts,
haltOnFailure=True, workdir=build_path, env=env))
# Build libcxxabi
jobs_flag = properties.WithProperties('-j%(jobs)s')
f.addStep(buildbot.steps.shell.ShellCommand(
name='build.libcxxabi', command=['make', jobs_flag, 'cxxabi'],
haltOnFailure=True, workdir=build_path))
# Build libcxx
f.addStep(buildbot.steps.shell.ShellCommand(
name='build.libcxx', command=['make', jobs_flag, 'cxx'],
haltOnFailure=True, workdir=build_path))
# Test libc++abi
f.addStep(LitTestCommand(
name = 'test.libcxxabi',
command = ['make', jobs_flag, 'check-cxxabi'],
description = ['testing', 'libcxxabi'],
descriptionDone = ['test', 'libcxxabi'],
workdir = build_path))
# Test libc++
f.addStep(LitTestCommand(
name = 'test.libcxx',
command = ['make', jobs_flag, 'check-cxx'],
description = ['testing', 'libcxx'],
descriptionDone = ['test', 'libcxx'],
workdir = build_path))
if check_libcxx_abilist:
f.addStep(buildbot.steps.shell.ShellCommand(
name = 'test.libcxx.abilist',
command = ['make', 'check-cxx-abilist'],
description = ['testing', 'libcxx', 'abi'],
descriptionDone = ['test', 'libcxx', 'abi'],
workdir = build_path))
if check_libcxx_benchmarks:
# Build the libc++ benchmarks
f.addStep(buildbot.steps.shell.ShellCommand(
name='build.libcxx.benchmarks',
command=['make', jobs_flag, 'cxx-benchmarks'],
haltOnFailure=True, workdir=build_path))
# Run the benchmarks
f.addStep(LitTestCommand(
name = 'test.libcxx.benchmarks',
command = ['make', jobs_flag, 'check-cxx-benchmarks'],
description = ['testing', 'libcxx', 'benchmarks'],
descriptionDone = ['test', 'libcxx', 'benchmarks'],
workdir = build_path))
return f
def getABITestsuitBuildFactory(
clean=True,
depends_on_projects=None,
extra_configure_args=None, # Extra CMake args for all stages.
jobs=None, # Restrict a degree of parallelism if needed.
env=None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = [
'llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld'
]
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
cleanBuildRequested=cleanBuildRequested,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
# First of all, we shall checkout the latest test-suite.
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test-suite',
alwaysUseLatest=True,
**kwargs)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps(**kwargs)
UnifiedTreeBuilder.addCmakeSteps(f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
f.addStep(
NinjaCommand(
name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH':
WithProperties("%(workdir)s/" + f.llvm_srcdir +
"/utils/lit:${PYTHONPATH}"),
'PATH':
WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(
LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f
def getOpenMPCMakeBuildFactory(
jobs = '%(jobs)s', # Number of concurrent jobs.
clean = True, # "clean" step is requested if true
env = None, # Environmental variables for all steps.
ompt = False, # Whether to enable the OpenMP Tools Interface.
test = True, # Test the built libraries.
depends_on_projects = None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
llvm_srcdir = 'llvm.src'
llvm_builddir = 'llvm.build'
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
if depends_on_projects is None:
# Monorepo configuration requires llvm and clang to get cmake work.
depends_on_projects = ['llvm', 'clang', 'openmp']
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_builddir,
cleanBuildRequested=cleanBuildRequested,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(
ShellCommand(
name = 'clean',
command = ['rm', '-rf', f.obj_dir],
warnOnFailure = True,
description = ['clean'],
doStepIf = cleanBuildRequested,
workdir = '.',
env = merged_env))
# Configure LLVM and OpenMP (and Clang, if requested).
cmake_args = ['cmake', '-G', 'Ninja']
cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON']
if ompt:
cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON']
if test:
lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs
cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)]
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')})
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir)
f.addStep(CmakeCommand(name='configure-openmp',
description=['configure','openmp'],
options=cmake_args,
path=src_dir,
env=merged_env,
workdir=f.obj_dir,
haltOnFailure=True,
**kwargs # Pass through all the extra arguments.
))
# Build OpenMP runtime libraries.
f.addStep(
NinjaCommand(
name = 'compile-openmp',
description = 'compile openmp',
workdir = f.obj_dir,
env = merged_env,
haltOnFailure=True))
# Test OpenMP runtime libraries, if requested.
if test:
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')})
ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)]
f.addStep(
LitTestCommand(
name = 'test-openmp',
command = ninja_test_args + ['check-openmp'],
description = 'test openmp',
workdir = f.obj_dir,
env = merged_env,
haltOnFailure=True))
return f
def getLLVMDocsBuildFactory(clean=True,
depends_on_projects=None,
extra_configure_args=None,
env=None,
**kwargs):
if depends_on_projects is None:
# All the projects from llvm_docs, and remove all duplicates.
_depends_on_projects = list(
set([project for project in llvm_docs if project]))
else:
# Make a local copy of depends_on_projects, as we are going to modify
# that.
_depends_on_projects = depends_on_projects[:]
# Some projects are interdependent for the purpose of documentation.
# Enforce the dependencies.
# TODO: Check later the dependencies for doxygen docs and enforce them
# here if needed.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [
("-G", "Ninja"),
("-DLLVM_ENABLE_DOXYGEN=", "ON"),
("-DLLVM_BUILD_DOCS=", "ON"),
("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"),
("-DLLVM_ENABLE_ASSERTIONS=", "OFF"),
("-DCMAKE_BUILD_TYPE=", "Release"),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
clean=clean,
depends_on_projects=_depends_on_projects,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
# Build the documentation for all the projects.
for project in llvm_docs:
target = llvm_docs[project][0]
# Build only those with specifies targets.
if target:
UnifiedTreeBuilder.addNinjaSteps(
f,
# Doxygen builds the final result for really
# long time without any output.
# We have to have a long timeout at this step.
timeout=10800,
targets=[target],
checks=[],
env=merged_env,
**kwargs)
# Publish just built documentation
for project in llvm_docs:
target, local_path, remote_path = llvm_docs[project]
f.addStep(
ShellCommand(
name="Publish {}".format(project or target),
description=[
"Publish", "just", "built", "documentation", "for",
"{}".format(project or target)
],
command=[
'rsync',
'-vrl',
'--delete',
'--force',
'--delay-updates',
'--delete-delay',
'--ignore-times',
'--checksum',
'-p',
'--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r',
"{}".format(local_path),
"lists.llvm.org:web/doxygen/{}".format(remote_path),
],
env=merged_env,
))
return f
def getABITestsuitBuildFactory(
clean = True,
depends_on_projects = None,
extra_configure_args = None, # Extra CMake args for all stages.
jobs = None, # Restrict a degree of parallelism if needed.
env = None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld']
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(NinjaCommand(name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Checkout the test-suite.
f.addStep(SVN(name='svn-test-suite',
mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir='test-suite'))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH' : WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"),
'PATH' : WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f