def addNinjaSteps(
f,
obj_dir = None,
targets = None,
checks = None,
install_dir = None,
env = None,
stage_name = None,
**kwargs):
# Build the unified tree.
if stage_name:
step_name = "%s-" % stage_name
else:
stage_name = ""
step_name = ""
if obj_dir is None:
obj_dir = f.obj_dir
f.addStep(NinjaCommand(name="build-%sunified-tree" % step_name,
haltOnFailure=True,
targets=targets,
description=["Build", stage_name, "unified", "tree"],
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
for check in checks:
f.addStep(LitTestCommand(name="test-%s%s" % (step_name, check),
command=['ninja', check],
description=[
"Test", "just", "built", "components", "for",
check,
],
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
if install_dir:
# TODO: Run this step only if none of the prevous failed.
f.addStep(NinjaCommand(name="install-%sall" % step_name,
targets=["install"],
description=["Install", "just", "built", "components"],
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def getCmakeWithMSVCBuildFactory(
clean = True, # False for incremental builds.
depends_on_projects = None, # List of projects to listen.
cmake_cache = None, # Path to a cmake cache file.
extra_configure_args = None, # Extra CMake args if any.
llvm_srcdir = None, # Source code root directory.
obj_dir = None, # Build tree root directory.
install_dir = None, # Directory to install the results to.
checks = None, # List of checks to test the build.
checks_on_target = None, # [(<name>,[<command tokens>])] array of
# check name and command to run on target.
jobs = None, # Restrict a degree of parallelism.
env = None, # Environmental variables for all steps.
# VS tools environment variable if using MSVC.
# For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select
# the VS 2015 toolchain, or empty string if environment is already set.
vs=None,
**kwargs):
if vs is None:
# We autodetect Visual Studio, unless otherwise is requested.
vs = "autodetect"
if install_dir is None:
install_dir = 'install'
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not vs, "Cannot have custom builder env vars with VS setup."
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if depends_on_projects is None:
depends_on_projects = [
'llvm',
'compiler-rt',
'clang',
'clang-tools-extra',
'libunwind',
'libcxx',
'libcxxabi',
'lld',
]
if checks is None:
# Check only host-side tools. Target-side tests should run on a target.
checks=[
"check-llvm",
"check-clang",
"check-lld",
]
source_remove_requested = lambda step: step.build.getProperty("clean")
clean_build_requested = lambda step: \
clean or \
step.build.getProperty("clean", \
default=step.build.getProperty("clean_obj") \
)
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=clean_build_requested,
**kwargs) # Pass through all the extra arguments.
# Remove the source code tree if requested.
# NOTE: Somehow RemoveDirectory buildbot command often fails on Windows,
# as somthing keeps a lock. We use rm command instead realying on a shell
# to support that.
f.addStep(ShellCommand(name='clean-%s-dir' % f.monorepo_dir,
command=['rm','-rf', f.monorepo_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove the source code',
workdir='.',
env=merged_env,
doStepIf=source_remove_requested))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Remove the build directory if requested.
f.addStep(ShellCommand(name='clean-%s-dir' % f.obj_dir,
command=['rm','-rf', f.obj_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove build directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
if vs:
# Configure MSVC environment if requested.
f.addStep(SetProperty(
command=builders_util.getVisualStudioEnvironment(vs, None),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Since this is a build of a cross toolchain, we build only the host-side
# tools first by the host system compiler. Libraries will be cross-compiled.
cmake_args.append(InterpolateToPosixPath(
'-DLLVM_AR=%(workdir)s/' + f.obj_dir + '/bin/llvm-ar.exe')),
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'),
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeTo(
install_dir,
f.obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
# Remove the build directory if requested.
f.addStep(ShellCommand(name='clean-%s-dir' % install_dir,
command=['rm','-rf', install_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove install directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
src_dir_rel = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)
# Add given cmake cache at the very end.
if cmake_cache:
cmake_args.append('-C%s' % cmake_cache)
f.addStep(CmakeCommand(name="cmake-configure",
haltOnFailure=True,
description=["Cmake", "configure"],
options=cmake_args,
path=src_dir_rel,
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
f.addStep(NinjaCommand(name="build-%s" % f.monorepo_dir,
haltOnFailure=True,
description=["Build", f.monorepo_dir],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Test the components if requested one check at a time.
for check in checks:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=["ninja", WithProperties(check)],
description=[
"Testing", "just", "built", "components", "for", check],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Run commands on a target if requested.
if checks_on_target:
for check, cmd in checks_on_target:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=cmd,
description=[
"Testing", "just", "built", "components", "for", check],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
if install_dir:
f.addStep(NinjaCommand(name="install-all",
haltOnFailure=True,
targets=["install"],
description=[
"Install", "just", "built", "components"],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
return f
def getToolchainBuildFactory(
clean=False,
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = LLVMBuildFactory(
depends_on_projects=[
"llvm",
"clang",
"clang-tools-extra",
"compiler-rt",
"libcxx",
"libcxxabi",
"libunwind",
"lld"
])
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(RemoveDirectory(name="clean-sdk",
dir=sdk_dir,
haltOnFailure=True))
f.addStep(ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean
# Clean up llvm sources.
f.addStep(RemoveDirectory(name="clean-llvm.src",
dir=src_dir,
haltOnFailure=True,
doStepIf=cleanCheckoutRequested))
# Get sources.
for project in f.depends_on_projects:
_, baseURL = svn_repos[project]
f.addStep(SVN(name="svn-%s" % project,
workdir=src_dir + "/" + project,
baseURL=baseURL))
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
# Clean up llvm build.
f.addStep(RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties(
"-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir
))
cmake_options.append(
WithProperties(
"-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir
))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(NinjaCommand(name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def get3StageClangLTOBuildFactory(
clean=True,
env=None,
build_gold=False,
cmake_cache_file=None,
extra_cmake_options=None,
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm-stage1.obj"
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
merged_env.update(env)
f = BuildFactory()
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
# We have to programatically determine the current llvm version.
def getClangVer(exit_status, stdout, stderr):
# We expect something like this:
# release = '3.9'
if exit_status:
return {}
k, v = stdout.split('=')
clang_ver = v.strip().strip("\'\"")
return {'clang_ver': clang_ver}
f.addStep(
SetProperty(
name="get_clang_ver",
command=["grep", "release =", "./tools/clang/docs/conf.py"],
extract_fn=getClangVer,
description="get clang release ver",
workdir=llvm_srcdir,
env=merged_env))
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
ShellCommand(doStepIf=cleanBuildRequested,
name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
cmake_command = ["cmake"]
if cmake_cache_file:
cmake_command += ['-C', cmake_cache_file]
if extra_cmake_options:
cmake_command += extra_cmake_options
cmake_command += ["../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
WarningCountingShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(
" ".join(cmake_command)),
workdir=llvm_objdir,
env=merged_env))
if build_gold:
f.addStep(
NinjaCommand(name='build-LLVMgold.so',
targets=['lib/LLVMgold.so'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
workdir=llvm_objdir,
env=merged_env))
f.addStep(
NinjaCommand(
name='build-stage3-clang',
targets=['stage3-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
timeout=3600, # Default value is not enough.
workdir=llvm_objdir,
env=merged_env))
f.addStep(
NinjaCommand(name='build-stage3-check-clang',
targets=['stage3-check-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["Check Clang"],
workdir=llvm_objdir,
env=merged_env))
# Compare stage2 & stage3 clang
shell_command = [
"diff", "-q", "tools/clang/stage2-bins/bin/clang-%(clang_ver)s",
"tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang-%(clang_ver)s"
]
f.addStep(
ShellCommand(name="compare-clang",
description=["comapre stage2 & stage3 clang"],
haltOnFailure=True,
command=WithProperties(" ".join(shell_command)),
workdir=llvm_objdir,
env=merged_env))
return f
def getSphinxDocsBuildFactory(
llvm_html=False, # Build LLVM HTML documentation
llvm_man=False, # Build LLVM man pages
clang_html=False, # Build Clang HTML documentation
clang_tools_html=False, # Build Clang Extra Tools HTML documentation
lld_html=False, # Build LLD HTML documentation
libcxx_html=False, # Build Libc++ HTML documentation
libunwind_html=False, # Build libunwind HTML documentation
lldb_html=False, # Build LLDB HTML documentation
extra_configure_args=None,
**kwargs):
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults for the config flags.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_SPHINX=', 'ON'),
('-DSPHINX_OUTPUT_HTML=', 'ON'),
('-DSPHINX_OUTPUT_MAN=', 'ON'),
('-DLLDB_INCLUDE_TESTS=', 'OFF'),
('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
])
llvm_srcdir = 'llvm/src'
llvm_objdir = 'llvm/build'
depends_on_projects = ['llvm']
if clang_html or clang_tools_html or lldb_html:
depends_on_projects.append('clang')
if clang_tools_html:
depends_on_projects.append('clang-tools-extra')
if lld_html:
depends_on_projects.append('lld')
if lldb_html:
depends_on_projects.append('lldb')
if libcxx_html:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
if libunwind_html:
depends_on_projects.append('libunwind')
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
extra_configure_args=cmake_args,
**kwargs) # Pass through all the extra arguments.
if llvm_html:
f.addStep(
NinjaCommand(name="docs-llvm-html",
haltOnFailure=True,
description=["Build LLVM Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-llvm-html']))
if llvm_man:
f.addStep(
NinjaCommand(name="docs-llvm-man",
haltOnFailure=True,
description=["Build LLVM Sphinx man pages"],
workdir=llvm_objdir,
targets=['docs-llvm-man']))
if clang_html:
f.addStep(
NinjaCommand(name="docs-clang-html",
haltOnFailure=True,
description=["Build Clang Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-clang-html']))
if clang_tools_html:
f.addStep(
NinjaCommand(
name="docs-clang-tools-html",
haltOnFailure=True,
description=[
"Build Clang Extra Tools Sphinx HTML documentation"
],
workdir=llvm_objdir,
targets=['docs-clang-tools-html']))
if lld_html:
f.addStep(
NinjaCommand(name="docs-lld-html",
haltOnFailure=True,
description=["Build LLD Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lld-html']))
if lldb_html:
f.addStep(
NinjaCommand(name="docs-lldb-html",
haltOnFailure=True,
description=["Build LLDB Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lldb-html']))
if libcxx_html:
f.addStep(
NinjaCommand(
name="docs-libcxx-html",
haltOnFailure=True,
description=["Build Libc++ Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libcxx-html']))
if libunwind_html:
f.addStep(
NinjaCommand(
name="docs-libunwind-html",
haltOnFailure=True,
description=["Build libunwind Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libunwind-html']))
return f
def _addSteps4SystemCompiler(f,
stage_idx=0,
clean=True,
jobs=None,
extra_configure_args=None,
env=None):
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeTo(
f.stage_installdirs[stage_idx], obj_dir)
# This stage could use incremental build.
# Clean stage1, only if requested.
f.addStep(
steps.RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean))
f.addStep(
steps.RemoveDirectory(name='clean-%s-dir' %
f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(
cmake_args,
[
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Note: On this stage we do not care of warnings, as we build with
# a system toolchain and cannot control the environment.
# Warnings are likely, and we ignore them.
# Create configuration files with cmake
f.addStep(
CmakeCommand(
name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
flunkOnWarnings=False,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
))
# Build clang by the system compiler
f.addStep(
NinjaCommand(
name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["build stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Test stage1 compiler
f.addStep(
NinjaCommand(
name="test-stage%s-compiler" % stage_num,
targets=["check-all"], # or "check-llvm", "check-clang"
jobs=jobs,
haltOnFailure=True,
flunkOnWarnings=False,
description=["test stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
# Install stage1 compiler
f.addStep(
NinjaCommand(
name="install-stage%s-compiler" % stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
env=env,
workdir=obj_dir,
))
def getSphinxDocsBuildFactory(
llvm_html=False, # Build LLVM HTML documentation
llvm_man=False, # Build LLVM man pages
clang_html=False, # Build Clang HTML documentation
clang_tools_html=False, # Build Clang Extra Tools HTML documentation
lld_html=False, # Build LLD HTML documentation
libcxx_html=False, # Build Libc++ HTML documentation
libunwind_html=False, # Build libunwind HTML documentation
lldb_html=False # Build LLDB HTML documentation
):
f = buildbot.process.factory.BuildFactory()
llvm_srcdir = 'llvm/src'
llvm_objdir = 'llvm/build'
clang_srcdir = llvm_srcdir + '/tools/clang'
clang_tools_srcdir = llvm_srcdir + '/tools/clang/tools/extra'
lld_srcdir = llvm_srcdir + '/tools/lld'
lldb_srcdir = llvm_srcdir + '/tools/lldb'
libcxx_srcdir = llvm_srcdir + '/projects/libcxx'
libcxxabi_srcdir = llvm_srcdir + '/projects/libcxxabi'
libunwind_srcdir = llvm_srcdir + '/projects/libunwind'
# Get LLVM. This is essential for all builds
# because we build all subprojects in tree
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
if clang_html or clang_tools_html or lldb_html:
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir=clang_srcdir))
if clang_tools_html:
f.addStep(
SVN(name='svn-clang-tools',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir=clang_tools_srcdir))
if lld_html:
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir=lld_srcdir))
if lldb_html:
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
workdir=lldb_srcdir))
if libcxx_html:
f.addStep(
SVN(name='svn-libcxx',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir=libcxx_srcdir))
f.addStep(
SVN(name='svn-libcxxabi',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir=libcxxabi_srcdir))
if libunwind_html:
f.addStep(
SVN(name='svn-libunwind',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libunwind/',
defaultBranch='trunk',
workdir=libunwind_srcdir))
# Use CMake to configure
cmakeCommand = [
"cmake",
WithProperties('%s/' + llvm_srcdir, 'workdir'),
'-G',
'Ninja',
'-DLLVM_ENABLE_SPHINX:BOOL=ON',
'-DSPHINX_OUTPUT_HTML:BOOL=ON',
'-DSPHINX_OUTPUT_MAN:BOOL=ON',
'-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON',
'-DLLDB_INCLUDE_TESTS=OFF',
]
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
description=["cmake configure"],
workdir=llvm_objdir))
if llvm_html:
f.addStep(
NinjaCommand(name="docs-llvm-html",
haltOnFailure=True,
description=["Build LLVM Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-llvm-html']))
if llvm_man:
f.addStep(
NinjaCommand(name="docs-llvm-man",
haltOnFailure=True,
description=["Build LLVM Sphinx man pages"],
workdir=llvm_objdir,
targets=['docs-llvm-man']))
if clang_html:
f.addStep(
NinjaCommand(name="docs-clang-html",
haltOnFailure=True,
description=["Build Clang Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-clang-html']))
if clang_tools_html:
f.addStep(
NinjaCommand(
name="docs-clang-tools-html",
haltOnFailure=True,
description=[
"Build Clang Extra Tools Sphinx HTML documentation"
],
workdir=llvm_objdir,
targets=['docs-clang-tools-html']))
if lld_html:
f.addStep(
NinjaCommand(name="docs-lld-html",
haltOnFailure=True,
description=["Build LLD Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lld-html']))
if lldb_html:
f.addStep(
NinjaCommand(name="docs-lldb-html",
haltOnFailure=True,
description=["Build LLDB Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lldb-html']))
if libcxx_html:
f.addStep(
NinjaCommand(
name="docs-libcxx-html",
haltOnFailure=True,
description=["Build Libc++ Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libcxx-html']))
if libunwind_html:
f.addStep(
NinjaCommand(
name="docs-libunwind-html",
haltOnFailure=True,
description=["Build libunwind Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libunwind-html']))
return f
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Kill any stale symbolizer processes for the last run. If there are any
# stale processes, the build will fail during linking. This can happen to
# any process, but it is most likely to happen to llvm-symbolizer if its
# pipe isn't closed.
taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
f.addStep(
ShellCommand(name='taskkill',
description='kill stale processes',
command=['cmd', '/c', taskkill_cmd],
haltOnFailure=False))
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f, 'llvm')
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(
ShellCommand(
name='cmake',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON"
] + extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizer runtimes.
f.addStep(
NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Build Clang and LLD next so that most compilation errors occur in a build
# step.
f.addStep(
NinjaCommand(name='build clang lld',
targets=['clang', 'lld'],
haltOnFailure=True,
description='ninja clang lld',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
for target in [
'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi'
]:
f.addStep(
NinjaCommand(name='run %s' % target,
targets=[target],
haltOnFailure=False,
description='ninja %s' % target,
workdir=build_dir,
env=Property('slave_env')))
return f
def getLLGoBuildFactory(
clean=True,
build_type='Release+Asserts',
test_libgo=True, # run 'check-libgo' target if True
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llgo_srcdir = '%s/tools/llgo' % llvm_srcdir
clang_srcdir = '%s/tools/clang' % llvm_srcdir
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Get LLVM, clang and llgo
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir=clang_srcdir))
f.addStep(
SVN(name='svn-llgo',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llgo/',
defaultBranch='trunk',
workdir=llgo_srcdir))
# Clean build directory, if requested.
f.addStep(
ShellCommand(name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
doStepIf=clean))
# Create configuration files with cmake
f.addStep(
ShellCommand(name="create-build-dir",
command=["mkdir", "-p", llvm_objdir],
haltOnFailure=False,
description=["create build dir"],
workdir="."))
cmakeCommand = [
"cmake",
"-G",
"Ninja",
"../%s" % llvm_srcdir,
"-DCMAKE_BUILD_TYPE=" + build_type,
]
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir))
# Build llgo
f.addStep(
NinjaCommand(name="build_llgo",
targets=["llgo"],
haltOnFailure=True,
description=["build llgo"],
workdir=llvm_objdir))
# Build libgo
f.addStep(
NinjaCommand(
name="build_libgo",
targets=["libgo"],
haltOnFailure=True,
logfiles={
'libgo-build-out':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-build-out.log',
'libgo-build-err':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-build-err.log',
'libgo-configure-out':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-configure-out.log',
'libgo-configure-err':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-configure-err.log',
},
lazylogfiles=True,
description=["build libgo"],
workdir=llvm_objdir))
# Build llgoi
f.addStep(
NinjaCommand(name="build_llgoi",
targets=["llgoi"],
haltOnFailure=True,
description=["build llgoi"],
workdir=llvm_objdir))
# Test llgo
f.addStep(
NinjaCommand(name="test_llgo",
targets=["check-llgo"],
haltOnFailure=True,
description=["test llgo"],
workdir=llvm_objdir))
# Test libgo
f.addStep(
NinjaCommand(name="test_libgo",
targets=["check-libgo"],
haltOnFailure=True,
description=["test libgo"],
workdir=llvm_objdir))
return f
def addNinjaSteps(f,
obj_dir=None,
targets=None,
checks=None,
install_dir=None,
env=None,
stage_name=None,
**kwargs):
# Build the unified tree.
if stage_name:
step_name = "%s-" % stage_name
else:
stage_name = ""
step_name = ""
if obj_dir is None:
obj_dir = f.obj_dir
f.addStep(
NinjaCommand(
name="build-%sunified-tree" % step_name,
haltOnFailure=True,
targets=targets,
description=["Build", stage_name, "unified", "tree"],
env=env or {},
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
# Note: At this point env could be None, a dictionary, or a Property object.
if isinstance(env, dict):
check_env = env.copy() if env else dict()
check_env['NINJA_STATUS'] = check_env.get('NINJA_STATUS',
"%e [%u/%r/%f] ")
else:
check_env = env or {}
for check in checks:
f.addStep(
LitTestCommand(
name="test-%s%s" % (step_name, check),
command=['ninja', check],
description=[
"Test",
"just",
"built",
"components",
"for",
check,
],
env=check_env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
if install_dir:
# TODO: Run this step only if none of the prevous failed.
f.addStep(
NinjaCommand(
name="install-%sall" % step_name,
targets=["install"],
description=["Install", "just", "built", "components"],
env=env or {},
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def getSanitizerBuildFactoryII(
clean=False,
sanity_check=True,
sanitizers=[
'sanitizer', 'asan', 'lsan', 'msan', 'tsan', 'ubsan', 'dfsan'
],
common_cmake_options=None, # FIXME: For backward compatibility. Will be removed.
extra_configure_args=[],
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
env=None,
jobs="%(jobs)s",
timeout=1200):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb', # Make sure Clang doesn't use color escape sequences.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = BuildFactory()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + llvm_objdir,
dir=llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
# Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind
f.addStep(
SVN(name='svn-llvm',
mode='update',
description='svn-llvm',
descriptionDone='svn-llvm',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
description='svn-clang',
descriptionDone='svn-clang',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
description='svn-compiler-rt',
descriptionDone='svn--compiler-rt',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(
SVN(name='svn-libcxx',
mode='update',
description='svn-libcxx',
descriptionDone='svn-libcxx',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir='%s/projects/libcxx' % llvm_srcdir))
f.addStep(
SVN(name='svn-libcxxabi',
mode='update',
description='svn-libcxxabi',
descriptionDone='svn-libcxxabi',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir='%s/projects/libcxxabi' % llvm_srcdir))
f.addStep(
SVN(name='svn-libunwind',
mode='update',
description='svn-libunwind',
descriptionDone='svn-libunwind',
baseURL='http://llvm.org/svn/llvm-project/libunwind/',
defaultBranch='trunk',
workdir='%s/projects/libunwind' % llvm_srcdir))
# Run annotated command for sanitizer.
if sanity_check:
f.addStep(
AnnotatedCommand(
name="lint check",
description="lint check",
timeout=timeout,
haltOnFailure=False, #True,
warnOnWarnings=True,
command=["./check_lint.sh"],
workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts"
% llvm_srcdir,
env=merged_env))
# Always build with ninja.
cmakeCommand = ["cmake", "-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(
a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
if not any(
a.startswith('-DLLVM_ENABLE_WERROR=')
for a in extra_configure_args):
cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF')
if not any(
a.startswith('-DLLVM_ENABLE_ASSERTIONS=')
for a in extra_configure_args):
cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(
a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_C_COMPILER=clang')
if not any(
a.startswith('-DCMAKE_CXX_COMPILER')
for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++')
if not any(
a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"')
if not any(
a.startswith('-DCMAKE_EXE_LINKER_FLAGS')
for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt')
if not any(
a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=')
for a in extra_configure_args):
cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"')
cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=False, #True,
warnOnWarnings=True,
command=WithProperties(" ".join(cmakeCommand)),
env=merged_env,
workdir=llvm_objdir,
doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir)))
# Build everything.
f.addStep(
NinjaCommand(
name='build',
haltOnFailure=False, #True,
warnOnWarnings=True,
description=['building', 'with', 'ninja'],
descriptionDone=['built', 'with', 'ninja'],
workdir=llvm_objdir,
env=merged_env))
# Run tests for each of the requested sanitizers.
if sanitizers:
for s in sanitizers:
f.addStep(
NinjaCommand(
name='test %s' % s,
targets=['check-%s' % s],
haltOnFailure=False, #True,
description=['testing', '%s' % s],
descriptionDone=['test', '%s' % s],
workdir=llvm_objdir,
env=merged_env))
return f
def getLLDWinBuildFactory(
clean = True,
# Default values for VS devenv and build configuration
vs = None, # What to run to configure Visual Studio utils.
target_arch = None, # Native.
extra_configure_args = None,
env = None):
# Set defaults
if vs is None:
vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015.
if extra_configure_args is None:
extra_configure_args = []
if env is None:
env = {}
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# If set up environment step is requested, do this now.
if vs:
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
# Always build with ninja.
cmake_options = ["-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmake_options.append('-DCMAKE_BUILD_TYPE=Release')
if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_WERROR=ON')
if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"')
cmake_options += extra_configure_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
warnOnWarnings=True,
options=cmake_options,
path="../%s" % f.llvm_srcdir,
env=env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld.
f.addStep(NinjaCommand(name='build lld',
haltOnFailure=True,
warnOnWarnings=True,
description='build lld',
workdir=f.llvm_objdir,
env=env))
# Test Lld
f.addStep(NinjaCommand(name='test lld',
targets=['lld-test'],
haltOnFailure=True,
warnOnWarnings=True,
description='test lld',
workdir=f.llvm_objdir,
env=env))
return f
def getABITestsuitBuildFactory(
clean = True,
depends_on_projects = None,
extra_configure_args = None, # Extra CMake args for all stages.
jobs = None, # Restrict a degree of parallelism if needed.
env = None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld']
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(NinjaCommand(name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Checkout the test-suite.
f.addStep(SVN(name='svn-test-suite',
mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir='test-suite'))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH' : WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"),
'PATH' : WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f
def getLibompCMakeBuildFactory(clean=True,
env=None,
ompt=False,
test=True,
c_compiler="gcc",
cxx_compiler="g++"):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
openmp_srcdir = "openmp.src"
openmp_builddir = "openmp.build"
llvm_srcdir = "llvm.src"
llvm_builddir = "llvm.build"
f = buildbot.process.factory.BuildFactory()
# Get libomp
f.addStep(
SVN(name='svn-libomp',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/openmp/',
defaultBranch='trunk',
workdir=openmp_srcdir))
# Get llvm to build llvm-lit
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Get clang if c/c++ compilers is clang/clang++
if c_compiler == "clang":
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
# Clean directory, if requested.
if clean:
f.addStep(
ShellCommand(name="clean",
command=["rm", "-rf", openmp_builddir, llvm_builddir],
warnOnFailure=True,
description=["clean"],
workdir='.',
env=merged_env))
# CMake llvm
f.addStep(
ShellCommand(name='cmake llvm',
command=[
"cmake", "../" + llvm_srcdir, "-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_ENABLE_ASSERTIONS=ON",
"-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
],
haltOnFailure=True,
description='cmake llvm',
workdir=llvm_builddir,
env=merged_env))
# Make clang build or just llvm utils build
if c_compiler == "clang":
f.addStep(
NinjaCommand(name="make clang build",
haltOnFailure=True,
description=["make clang build"],
workdir=llvm_builddir,
env=merged_env))
else:
f.addStep(
NinjaCommand(name="make llvm utils build",
targets=["LLVMX86Utils"],
haltOnFailure=True,
description=["make llvm utils build"],
workdir=llvm_builddir,
env=merged_env))
if ompt:
f.addStep(
NinjaCommand(name="make ompt test utils",
targets=["FileCheck"],
haltOnFailure=True,
description=["make ompt test utils"],
workdir=llvm_builddir,
env=merged_env))
# Add clang/llvm-lit to PATH
merged_env.update({
'PATH':
WithProperties("%(workdir)s/" + llvm_builddir + "/bin:" + "${PATH}")
})
# CMake libomp
command = [
"cmake", "../" + openmp_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
]
if ompt:
command.append("-DLIBOMP_OMPT_SUPPORT=ON")
f.addStep(
ShellCommand(name='cmake libomp',
command=command,
haltOnFailure=True,
description='cmake libomp',
workdir=openmp_builddir,
env=merged_env))
# Make libomp
f.addStep(
WarningCountingShellCommand(name='make libomp build',
command=['make'],
haltOnFailure=True,
description='make libomp build',
workdir=openmp_builddir,
env=merged_env))
# Test, if requested
if test:
f.addStep(
LitTestCommand(name="make check-libomp",
command=["make", "check-libomp"],
haltOnFailure=True,
description=["make check-libomp"],
descriptionDone=["build checked"],
workdir=openmp_builddir,
env=merged_env))
return f
def get3StageClangLTOBuildFactory(
clean=True,
env=None,
build_gold=False,
cmake_cache_file=None,
extra_cmake_options=None,
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm-stage1.obj"
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
merged_env.update(env)
f = BuildFactory()
f.addStep(
SVN(
name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir
)
)
f.addStep(
SVN(
name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir
)
)
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
ShellCommand(
doStepIf=cleanBuildRequested,
name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env
)
)
cmake_command = ["cmake"]
if cmake_cache_file:
cmake_command += ['-C', cmake_cache_file]
if extra_cmake_options:
cmake_command += extra_cmake_options
cmake_command += [
"../%s" % llvm_srcdir
]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
WarningCountingShellCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmake_command)),
workdir=llvm_objdir,
env=merged_env
)
)
if build_gold:
f.addStep(
NinjaCommand(name='build-LLVMgold.so',
targets=['lib/LLVMgold.so'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
workdir=llvm_objdir,
env=merged_env)
)
f.addStep(
NinjaCommand(name='build-stage3-clang',
targets=['stage3-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
timeout=3600, # Default value is not enough.
workdir=llvm_objdir,
env=merged_env)
)
f.addStep(
NinjaCommand(name='build-stage3-check-clang',
targets=['stage3-check-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["Check Clang"],
timeout=3600, # Default value is not enough.
workdir=llvm_objdir,
env=merged_env)
)
# Compare stage2 & stage3 clang
shell_command = [
"diff",
"-q",
"tools/clang/stage2-bins/bin/clang",
"tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang"
]
f.addStep(
ShellCommand(
name="compare-clang",
description=["comapre stage2 & stage3 clang"],
haltOnFailure=True,
command=WithProperties(" ".join(shell_command)),
workdir=llvm_objdir,
env=merged_env
)
)
return f
def getClangAndLLDBuildFactory(
clean=True,
env=None,
withLLD=True,
extraCmakeOptions=None,
extraCompilerOptions=None,
buildWithSanitizerOptions=None,
triple=None,
isMSVC=False,
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
extraLitArgs=None):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Get LLVM, Clang and LLD.
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Sanitizer runtime in compiler-rt, and cannot be built with sanitizer compiler.
if buildWithSanitizerOptions is None:
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if withLLD:
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='%s/tools/lld' % llvm_srcdir))
# Clean directory, if requested.
if clean:
shellCommand = ["rm", "-rf", llvm_objdir]
if isMSVC:
shellCommand = ["rmdir", "/S", "/Q", llvm_objdir]
f.addStep(
ShellCommand(name="rm-llvm_objdir",
command=shellCommand,
haltOnFailure=False,
flunkOnFailure=False,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
# Create configuration files with cmake.
options = ["-Wdocumentation", "-Wno-documentation-deprecated-sync"]
if isMSVC:
options = []
if extraCompilerOptions:
options += extraCompilerOptions
if buildWithSanitizerOptions:
options += buildWithSanitizerOptions
lit_args = ["-v"]
if extraLitArgs:
lit_args += extraLitArgs
lit_args = ["-DLLVM_LIT_ARGS=\"%s\"" % " ".join(lit_args)]
cmakeCommand = [
"cmake", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_ENABLE_ASSERTIONS=ON"
]
if buildWithSanitizerOptions:
cmakeCommand += [
"-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++"
]
if triple:
cmakeCommand += ["-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % triple]
if extraCmakeOptions:
assert not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extraCmakeOptions), \
"Please use extraLitArgs for LIT arguments instead of defining them in extraCmakeOptions."
cmakeCommand += extraCmakeOptions
if not isMSVC:
cmakeCommand += [
"-DCMAKE_C_FLAGS=\"%s\"" % (" ".join(options)),
"-DCMAKE_CXX_FLAGS=\"-std=c++11 %s\"" % (" ".join(options)),
]
cmakeCommand += lit_args
cmakeCommand += ["-GNinja", "../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand)),
workdir=llvm_objdir,
env=merged_env))
# Build everything.
f.addStep(
NinjaCommand(name="build",
prefixCommand=prefixCommand,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
# Test everything.
f.addStep(
NinjaCommand(name="test",
prefixCommand=prefixCommand,
targets=["check-all"],
haltOnFailure=True,
description=["test"],
workdir=llvm_objdir,
env=merged_env))
return f
def getOpenMPCMakeBuildFactory(
c_compiler='gcc', # The C compiler to use.
cxx_compiler='g++', # The C++ compiler to use.
jobs='%(jobs)s', # Number of concurrent jobs.
clean=True, # "clean" step is requested if true
env=None, # Environmental variables for all steps.
ompt=False, # Whether to enable the OpenMP Tools Interface.
test=True, # Test the built libraries.
):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
openmp_srcdir = 'openmp.src'
openmp_builddir = 'openmp.build'
llvm_srcdir = 'llvm.src'
llvm_builddir = 'llvm.build'
build_clang = c_compiler == 'clang'
f = buildbot.process.factory.BuildFactory()
# Checkout sources.
f.addStep(
SVN(name='svn-openmp',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/openmp/',
defaultBranch='trunk',
workdir=openmp_srcdir))
# LLVM sources are needed to build utilities for testing.
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Get Clang sources, if requested.
if build_clang:
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean")
f.addStep(
ShellCommand(name='clean',
command=['rm', '-rf', openmp_builddir, llvm_builddir],
warnOnFailure=True,
description=['clean'],
doStepIf=cleanBuildRequested,
workdir='.',
env=merged_env))
# Configure LLVM (and Clang, if requested).
cmake_args = ['cmake', '-G', 'Ninja', '../%s' % llvm_srcdir]
cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON']
cmake_args += ['-DCMAKE_C_COMPILER=%s' % c_compiler]
cmake_args += ['-DCMAKE_CXX_COMPILER=%s' % cxx_compiler]
f.addStep(
Configure(name='configure-llvm',
command=cmake_args,
description='configure llvm',
workdir=llvm_builddir,
env=merged_env))
# Build LLVM utils.
f.addStep(
NinjaCommand(name='compile-llvm-utils',
targets=['FileCheck'],
description='compile llvm utils',
workdir=llvm_builddir,
env=merged_env))
# Build Clang if requested.
if build_clang:
f.addStep(
NinjaCommand(name='compile-clang',
description='compile clang',
workdir=llvm_builddir,
env=merged_env))
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH':
WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')
})
# Configure OpenMP runtime libraries.
cmake_args = ['cmake', '-G', 'Ninja', '../%s' % openmp_srcdir]
cmake_args += ['-DCMAKE_C_COMPILER=%s' % c_compiler]
cmake_args += ['-DCMAKE_CXX_COMPILER=%s' % cxx_compiler]
if ompt:
cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON']
if test:
lit_args = '-v --show-unsupported --show-xfail -j %s' % jobs
cmake_args += [WithProperties('-DOPENMP_LIT_ARGS=%s' % lit_args)]
f.addStep(
Configure(name='configure-openmp',
command=cmake_args,
description='configure openmp',
workdir=openmp_builddir,
env=merged_env))
# Build OpenMP runtime libraries.
f.addStep(
NinjaCommand(name='compile-openmp',
description='compile openmp',
workdir=openmp_builddir,
env=merged_env))
# Test OpenMP runtime libraries, if requested.
if test:
ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)]
f.addStep(
LitTestCommand(name='test-openmp',
command=ninja_test_args + ['check-openmp'],
description='test openmp',
workdir=openmp_builddir,
env=merged_env))
return f
def getCmakeWithNinjaBuildFactory(depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
install_dir=None,
clean=False,
extra_configure_args=None,
env=None,
**kwargs):
# Reconcile the cmake options for this build.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# To set proper defaults, uncomment and modify the following lines.
#CmakeCommand.applyDefaultOptions(cmake_args, [
# ('-DCMAKE_BUILD_TYPE=', 'Release'),
# ])
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
f = getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir or "install",
extra_configure_args=cmake_args,
env=env,
**kwargs # Pass through all the extra arguments.
)
# Directories to use for this build.
# obj_dir = f.obj_dir
# src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
# install_dir = LLVMBuildFactory.pathRelativeToBuild(f.install_dir, obj_dir)
# Build the unified tree.
f.addStep(
NinjaCommand(
name="build-unified-tree",
haltOnFailure=True,
description=["build unified tree"],
env=env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components
f.addStep(
NinjaCommand(
name="test-check-all",
targets=["check-all"],
haltOnFailure=True,
description=["test just built components"],
env=env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
f.addStep(
NinjaCommand(
name="install-all",
targets=["install"],
haltOnFailure=True,
description=["install just built components"],
env=env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
return f
def getOpenMPCMakeBuildFactory(
jobs = '%(jobs)s', # Number of concurrent jobs.
clean = True, # "clean" step is requested if true
env = None, # Environmental variables for all steps.
ompt = False, # Whether to enable the OpenMP Tools Interface.
test = True, # Test the built libraries.
depends_on_projects = None,
**kwargs):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
llvm_srcdir = 'llvm.src'
llvm_builddir = 'llvm.build'
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
if depends_on_projects is None:
# Monorepo configuration requires llvm and clang to get cmake work.
depends_on_projects = ['llvm', 'clang', 'openmp']
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_builddir,
cleanBuildRequested=cleanBuildRequested,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(
ShellCommand(
name = 'clean',
command = ['rm', '-rf', f.obj_dir],
warnOnFailure = True,
description = ['clean'],
doStepIf = cleanBuildRequested,
workdir = '.',
env = merged_env))
# Configure LLVM and OpenMP (and Clang, if requested).
cmake_args = ['cmake', '-G', 'Ninja']
cmake_args += ['-DCMAKE_BUILD_TYPE=Release', '-DLLVM_ENABLE_ASSERTIONS=ON']
if ompt:
cmake_args += ['-DLIBOMP_OMPT_SUPPORT=ON']
if test:
lit_args = '-vv --show-unsupported --show-xfail -j %s' % jobs
cmake_args += [WithProperties('-DLLVM_LIT_ARGS=%s' % lit_args)]
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')})
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, f.obj_dir)
f.addStep(CmakeCommand(name='configure-openmp',
description=['configure','openmp'],
options=cmake_args,
path=src_dir,
env=merged_env,
workdir=f.obj_dir,
haltOnFailure=True,
**kwargs # Pass through all the extra arguments.
))
# Build OpenMP runtime libraries.
f.addStep(
NinjaCommand(
name = 'compile-openmp',
description = 'compile openmp',
workdir = f.obj_dir,
env = merged_env,
haltOnFailure=True))
# Test OpenMP runtime libraries, if requested.
if test:
# Add llvm-lit and clang (if built) to PATH
merged_env.update({
'PATH': WithProperties('%(workdir)s/' + llvm_builddir + '/bin:${PATH}')})
ninja_test_args = ['ninja', WithProperties('-j %s' % jobs)]
f.addStep(
LitTestCommand(
name = 'test-openmp',
command = ninja_test_args + ['check-openmp'],
description = 'test openmp',
workdir = f.obj_dir,
env = merged_env,
haltOnFailure=True))
return f
def getFactory(
depends_on_projects = None,
targets = None,
checks = None,
clean = False,
extra_configure_args = None,
llvm_srcdir = None,
obj_dir = None,
env = None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
if targets:
step_name = "build-{}".format("-".join(targets))
step_description=["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description=["Build", "unified", "tree"]
f.addStep(NinjaCommand(name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(NinjaCommand(name="test-{}".format("-".join(checks)),
targets=checks,
description=[
"Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy", "LLD", "executable", "to", "the", "performance",
"test", "suite",
],
command=[
"cp", "-aL", "./{}/bin/ld.lld".format(f.obj_dir), "./lld-speed-test/ld.lld"
],
env=merged_env,
workdir='.'
)
)
# Run the performance test suite.
perf_command = [
"python",
"%(builddir)s/lld-benchmark.py",
"--machine=%(workername)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(
name="performance-test-suite",
description=[
"LLD", "performance","test","suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env
)
)
return f
def getSphinxDocsBuildFactory(
llvm_html=False, # Build LLVM HTML documentation
llvm_man=False, # Build LLVM man pages
clang_html=False, # Build Clang HTML documentation
lld_html=False, # Build LLD HTML documentation
libcxx_html=False # Build Libc++ HTML documentation
):
f = buildbot.process.factory.BuildFactory()
llvm_srcdir = 'llvm/src'
llvm_objdir = 'llvm/build'
clang_srcdir = llvm_srcdir + '/tools/clang'
lld_srcdir = llvm_srcdir + '/tools/lld'
libcxx_srcdir = llvm_srcdir + '/projects/libcxx'
libcxxabi_srcdir = llvm_srcdir + '/projects/libcxxabi'
# Get LLVM. This is essential for all builds
# because we build all subprojects in tree
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
if clang_html:
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir=clang_srcdir))
if lld_html:
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir=lld_srcdir))
if libcxx_html:
f.addStep(
SVN(name='svn-libcxx',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir=libcxx_srcdir))
f.addStep(
SVN(name='svn-libcxxabi',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir=libcxxabi_srcdir))
f.addStep(
ShellCommand(
name="create-build-dir",
command=["mkdir", "-p", llvm_objdir],
haltOnFailure=
False, # We might of already created the directory in a previous build
description=["create build dir"],
workdir="."))
# Use CMake to configure
cmakeCommand = [
"cmake",
WithProperties('%s/' + llvm_srcdir, 'workdir'), '-G', 'Ninja',
'-DLLVM_ENABLE_SPHINX:BOOL=ON', '-DSPHINX_OUTPUT_HTML:BOOL=ON',
'-DSPHINX_OUTPUT_MAN:BOOL=ON'
]
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
description=["cmake configure"],
workdir=llvm_objdir))
if llvm_html:
f.addStep(
NinjaCommand(name="docs-llvm-html",
haltOnFailure=True,
description=["Build LLVM Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-llvm-html']))
if llvm_man:
f.addStep(
NinjaCommand(name="docs-llvm-man",
haltOnFailure=True,
description=["Build LLVM Sphinx man pages"],
workdir=llvm_objdir,
targets=['docs-llvm-man']))
if clang_html:
f.addStep(
NinjaCommand(name="docs-clang-html",
haltOnFailure=True,
description=["Build Clang Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-clang-html']))
if lld_html:
f.addStep(
NinjaCommand(name="docs-lld-html",
haltOnFailure=True,
description=["Build LLD Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lld-html']))
if libcxx_html:
f.addStep(
NinjaCommand(
name="docs-libcxx-html",
haltOnFailure=True,
description=["Build Libc++ Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libcxx-html']))
return f
def _addSteps4StagedCompiler(f,
stage_idx=1,
use_stage_idx=-1,
jobs=None,
extra_configure_args=None,
env=None):
if use_stage_idx < 0:
use_stage_idx = stage_idx - 1
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeTo(
f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[use_stage_idx]
# Always do a clean build for the staged compiler.
f.addStep(
steps.RemoveDirectory(
name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
))
f.addStep(
steps.RemoveDirectory(
name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args.append(
WithProperties("-DCMAKE_CXX_COMPILER=%(builddir)s/" + staged_install +
"/bin/clang++"))
cmake_args.append(
WithProperties("-DCMAKE_C_COMPILER=%(builddir)s/" + staged_install +
"/bin/clang"))
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
# Create configuration files with cmake
f.addStep(
CmakeCommand(
name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
))
# Build clang by the staged compiler
f.addStep(
NinjaCommand(
name="build-stage%s-compiler" % stage_num,
jobs=jobs,
haltOnFailure=True,
description=["build stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Test just built compiler
f.addStep(
NinjaCommand(
name="test-stage%s-compiler" % stage_num,
targets=["check-all"],
jobs=jobs,
haltOnFailure=True,
description=["test stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Install just built compiler
f.addStep(
NinjaCommand(
name="install-stage%s-compiler" % stage_num,
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
def getABITestsuitBuildFactory(
clean=True,
depends_on_projects=None,
extra_configure_args=None, # Extra CMake args for all stages.
jobs=None, # Restrict a degree of parallelism if needed.
env=None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = [
'llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld'
]
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
f = UnifiedTreeBuilder.getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
cleanBuildRequested=cleanBuildRequested,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
# First of all, we shall checkout the latest test-suite.
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test-suite',
alwaysUseLatest=True,
**kwargs)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps(**kwargs)
UnifiedTreeBuilder.addCmakeSteps(f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
f.addStep(
NinjaCommand(
name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH':
WithProperties("%(workdir)s/" + f.llvm_srcdir +
"/utils/lit:${PYTHONPATH}"),
'PATH':
WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(
LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f
def getCUDATestsuiteBuildFactory(
externals, # Directory with CUDA, thrust and gcc versions for testing.
always_clean=True,
test=False,
useTwoStage=False,
cmake='cmake',
extra_cmake_args=None, # Extra CMake args for all stages.
extra_ts_cmake_args=None, # extra cmake args for testsuite.
jobs=None,
cuda_jobs=1, # number of simultaneous CUDA apps to run
env=None, # Environmental variables for all steps.
enable_thrust_tests=False,
split_thrust_tests=False, # Each thrust test is a separate executable.
run_thrust_tests=False,
enable_libcxx=True, # checkout/build libcxx to test with.
gpu_arch_list=None,
gpu_devices=None, # List of devices to make visible to CUDA
stage1_config='Release',
stage2_config='Release'):
if extra_cmake_args is None:
extra_cmake_args = []
if extra_ts_cmake_args is None:
extra_ts_cmake_args = []
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory.
stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory.
stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory.
install_dir = 'clang.install'
if useTwoStage:
clang_build_dir = stage2_build_dir
else:
clang_build_dir = stage1_build_dir
# Build clang.
f = ClangBuilder.getClangCMakeBuildFactory(
clean=always_clean,
test=test,
cmake=cmake,
extra_cmake_args=extra_cmake_args,
jobs=jobs,
env=merged_env,
useTwoStage=useTwoStage,
stage1_config=stage1_config,
stage2_config=stage2_config,
checkout_clang_tools_extra=False,
checkout_compiler_rt=False,
checkout_libcxx=enable_libcxx,
checkout_lld=False,
checkout_test_suite=True)
cuda_test_env = {
'PYTHONPATH':
WithProperties("%(workdir)s/" + source_dir +
"/utils/lit:${PYTHONPATH}"),
'DESTDIR':
WithProperties("%(workdir)s/" + install_dir),
'PATH':
WithProperties("%(workdir)s/" + install_dir +
"/usr/local/bin:${PATH}"),
}
merged_env.update(cuda_test_env)
ts_build_dir = 'test-suite-build'
f.addStep(
RemoveDirectory(name="Remove old clang install directory",
dir=install_dir))
# Install clang into directory pointed by $DESTDIR
f.addStep(
NinjaCommand(name='ninja install clang',
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=split("installing clang"),
descriptionDone=split("Clang installation is done."),
workdir=clang_build_dir,
env=merged_env))
# Completely remove test suite build dir.
f.addStep(
RemoveDirectory(name="Remove old test-suite build directory",
dir=ts_build_dir))
if extra_ts_cmake_args:
cmake_args = extra_ts_cmake_args[:]
else:
cmake_args = []
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cmake_args.append(
WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + clang_build_dir +
"/bin/clang++"))
cmake_args.append(
WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir +
"/bin/clang"))
cmake_args.append('-DTEST_SUITE_SUBDIRS=External'),
# Limit to External tests only.
if externals:
cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals)
if split_thrust_tests:
cmake_args.append('-DTHRUST_SPLIT_TESTS=1')
if gpu_arch_list:
cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list))
if cuda_jobs:
cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs)
# Then do fresh cmake configuration.
f.addStep(
CmakeCommand(name='cmake test-suite',
description='cmake test-suite',
haltOnFailure=True,
options=cmake_args,
path="../test/test-suite",
workdir=ts_build_dir,
env=merged_env))
# Always build simple CUDA tests. They serve as compilation
# smoketests and will fail quickly if compiler has obvious issues
# compiling CUDA files.
f.addStep(
NinjaCommand(name='ninja build simple CUDA tests',
targets=["cuda-tests-simple"],
jobs=jobs,
haltOnFailure=True,
description=split("building simple CUDA tests"),
descriptionDone=split("simple CUDA tests built."),
workdir=ts_build_dir,
env=merged_env))
# Limit GPUs visible to CUDA.
if gpu_devices:
for gpu_id in gpu_devices:
# make ID a string as it may be either an integer or a UUID string.
gpu_id = str(gpu_id)
gpu_env = dict(merged_env)
gpu_env["CUDA_VISIBLE_DEVICES"] = gpu_id
f.addStep(
NinjaCommand(
name='run simple CUDA tests on gpu %s' % gpu_id,
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=split("Running simple CUDA tests on GPU %s" %
gpu_id),
descriptionDone=split("simple CUDA tests on GPU %s done." %
gpu_id),
workdir=ts_build_dir,
env=gpu_env))
else:
f.addStep(
NinjaCommand(
name='run simple CUDA tests',
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=split("Running simple CUDA tests"),
descriptionDone=split("simple CUDA tests done."),
workdir=ts_build_dir,
env=merged_env))
# If we've enabled thrust tests, build them now.
# WARNING: This takes a lot of time to build.
if (enable_thrust_tests):
f.addStep(
NinjaCommand(name='ninja build thrust',
targets=["cuda-tests-thrust"],
jobs=jobs,
haltOnFailure=True,
description=split("building thrust tests"),
descriptionDone=split("thrust tests built."),
workdir=ts_build_dir,
env=merged_env))
# Run them. That also takes a while.
# TODO(tra) we may want to run more than one instance so one
# can be compiling tests while another is running them on GPU.
if run_thrust_tests:
f.addStep(
NinjaCommand(
name='run all CUDA tests',
targets=["check"],
jobs=1, # lit will parallelize the jobs.
haltOnFailure=True,
description=split("running all CUDA tests."),
descriptionDone=split("all cuda tests done."),
workdir=ts_build_dir,
env=merged_env))
return f
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f,'llvm')
# Global configurations
build_dir='build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean '+build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
f.addStep(ShellCommand(name='cmake',
command=[cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE="+config,
"-DLLVM_ENABLE_ASSERTIONS=ON"]
+ extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizers runtime.
f.addStep(NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
test_targets = ['check-asan','check-asan-dynamic','check-sanitizer', 'check-cfi']
f.addStep(NinjaCommand(name='run tests',
targets=test_targets,
haltOnFailure=True,
description='ninja test',
workdir=build_dir,
env=Property('slave_env')))
return f