def getFactory(
depends_on_projects = None,
targets = None,
checks = None,
clean = False,
extra_configure_args = None,
llvm_srcdir = None,
obj_dir = None,
env = None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
if targets:
step_name = "build-{}".format("-".join(targets))
step_description=["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description=["Build", "unified", "tree"]
f.addStep(NinjaCommand(name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(NinjaCommand(name="test-{}".format("-".join(checks)),
targets=checks,
description=[
"Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy", "LLD", "executable", "to", "the", "performance",
"test", "suite",
],
command=[
"cp", "-aL", "./{}/bin/ld.lld".format(f.obj_dir), "./lld-speed-test/ld.lld"
],
env=merged_env,
workdir='.'
)
)
# Run the performance test suite.
perf_command = [
"python",
"%(builddir)s/lld-benchmark.py",
"--machine=%(workername)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(
name="performance-test-suite",
description=[
"LLD", "performance","test","suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env
)
)
return f
def getSphinxDocsBuildFactory(
llvm_html=False, # Build LLVM HTML documentation
llvm_man=False, # Build LLVM man pages
clang_html=False, # Build Clang HTML documentation
clang_tools_html=False, # Build Clang Extra Tools HTML documentation
lld_html=False, # Build LLD HTML documentation
libcxx_html=False, # Build Libc++ HTML documentation
libunwind_html=False, # Build libunwind HTML documentation
lldb_html=False, # Build LLDB HTML documentation
extra_configure_args=None,
**kwargs):
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults for the config flags.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_SPHINX=', 'ON'),
('-DSPHINX_OUTPUT_HTML=', 'ON'),
('-DSPHINX_OUTPUT_MAN=', 'ON'),
('-DLLDB_INCLUDE_TESTS=', 'OFF'),
('-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
])
llvm_srcdir = 'llvm/src'
llvm_objdir = 'llvm/build'
depends_on_projects = ['llvm']
if clang_html or clang_tools_html or lldb_html:
depends_on_projects.append('clang')
if clang_tools_html:
depends_on_projects.append('clang-tools-extra')
if lld_html:
depends_on_projects.append('lld')
if lldb_html:
depends_on_projects.append('lldb')
if libcxx_html:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
if libunwind_html:
depends_on_projects.append('libunwind')
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
extra_configure_args=cmake_args,
**kwargs) # Pass through all the extra arguments.
if llvm_html:
f.addStep(
NinjaCommand(name="docs-llvm-html",
haltOnFailure=True,
description=["Build LLVM Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-llvm-html']))
if llvm_man:
f.addStep(
NinjaCommand(name="docs-llvm-man",
haltOnFailure=True,
description=["Build LLVM Sphinx man pages"],
workdir=llvm_objdir,
targets=['docs-llvm-man']))
if clang_html:
f.addStep(
NinjaCommand(name="docs-clang-html",
haltOnFailure=True,
description=["Build Clang Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-clang-html']))
if clang_tools_html:
f.addStep(
NinjaCommand(
name="docs-clang-tools-html",
haltOnFailure=True,
description=[
"Build Clang Extra Tools Sphinx HTML documentation"
],
workdir=llvm_objdir,
targets=['docs-clang-tools-html']))
if lld_html:
f.addStep(
NinjaCommand(name="docs-lld-html",
haltOnFailure=True,
description=["Build LLD Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lld-html']))
if lldb_html:
f.addStep(
NinjaCommand(name="docs-lldb-html",
haltOnFailure=True,
description=["Build LLDB Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-lldb-html']))
if libcxx_html:
f.addStep(
NinjaCommand(
name="docs-libcxx-html",
haltOnFailure=True,
description=["Build Libc++ Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libcxx-html']))
if libunwind_html:
f.addStep(
NinjaCommand(
name="docs-libunwind-html",
haltOnFailure=True,
description=["Build libunwind Sphinx HTML documentation"],
workdir=llvm_objdir,
targets=['docs-libunwind-html']))
return f
def getLLVMDocsBuildFactory(clean=False,
depends_on_projects=None,
extra_configure_args=None,
env=None,
**kwargs):
if depends_on_projects is None:
# All the projects by default.
_depends_on_projects = [
"llvm",
"clang",
"clang-tools-extra",
"libcxx",
"libcxxabi",
"libunwind",
"lld",
"lldb",
"flang",
"openmp",
"polly",
]
else:
# Make a local copy of depends_on_projects, as we are going to modify
# that.
_depends_on_projects = depends_on_projects[:]
# Some projects are interdependent for the purpose of documentation.
# Enforce the dependencies.
if ("clang-tools-extra" in _depends_on_projects or \
"lldb" in _depends_on_projects
) and "clang" not in _depends_on_projects:
_depends_on_projects.append("clang")
if "libcxx" in _depends_on_projects and \
"libcxxabi" not in _depends_on_projects:
_depends_on_projects.append("libcxxabi")
if "libcxxabi" in _depends_on_projects and \
"libcxx" not in _depends_on_projects:
_depends_on_projects.append("libcxx")
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
CmakeCommand.applyDefaultOptions(cmake_args, [
("-G", "Ninja"),
("-DLLVM_ENABLE_SPHINX=", "ON"),
("-DSPHINX_OUTPUT_HTML=", "ON"),
("-DSPHINX_OUTPUT_MAN=", "OFF"),
("-DSPHINX_WARNINGS_AS_ERRORS=", "OFF"),
("-DLLVM_ENABLE_ASSERTIONS=", "OFF"),
("-DCMAKE_BUILD_TYPE=", "Release"),
])
# Build docs for each of the projects this builder depends on
docs = [
llvm_docs[project] for project in llvm_docs.keys()
if project in _depends_on_projects
]
f = UnifiedTreeBuilder.getCmakeBuildFactory(
clean=clean,
depends_on_projects=_depends_on_projects,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
UnifiedTreeBuilder.addNinjaSteps(f,
targets=[d[0] for d in docs],
checks=[],
env=merged_env,
**kwargs)
# Publish just built documentation
for target, local_path, remote_path in docs:
f.addStep(
ShellCommand(
name="Publish %s" % target,
description=[
"Publish",
"just",
"built",
"documentation",
"fior",
"%s" % target,
],
command=[
'rsync',
'-vrl',
'--delete',
'--force',
'--delay-updates',
'--delete-delay',
'--ignore-times',
'--checksum',
'-p',
'--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r',
"%s" % local_path,
"lists.llvm.org:web/%s" % remote_path,
],
env=merged_env,
))
return f
def getFactory(
depends_on_projects = None,
targets = None,
checks = None,
clean = False,
extra_configure_args = None,
env = None,
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'lld']
if checks is None:
checks = [] # No check unless requested specifically.
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'OFF'),
('-DLLVM_BUILD_STATIC=', 'ON'),
('-DLLVM_ENABLE_PIC=', 'OFF'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
if targets:
step_name = "build-%s" % ("-".join(targets))
step_description=["Build"]
step_description.extend(targets)
else:
step_name = "build-unified-tree"
step_description=["Build", "unified", "tree"]
f.addStep(NinjaCommand(name=step_name,
targets=targets,
description=step_description,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Test just built components if requested.
if checks:
f.addStep(NinjaCommand(name="test-%s" % ("-".join(checks)),
targets=checks,
description=[
"Test", "just", "built", "components"],
haltOnFailure=kwargs.get('haltOnFailure', True),
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Copy just built LLD executable to the test suite directory
# to avoid load from a hard drive overhead.
f.addStep(
ShellCommand(
name="copy-lld-to-test-suite",
description=[
"Copy", "LLD", "executable", "to", "the", "performance",
"test", "suite",
],
command=[
"cp", "-aL", "./bin/ld.lld", "../lld-speed-test/ld.lld"
],
workdir=f.obj_dir,
env=merged_env
)
)
# Run the performance test suite.
perf_command = [
"python",
"%(workdir)s/lld-benchmark.py",
"--machine=%(slavename)s",
"--revision=%(got_revision)s",
"--linker=./ld.lld",
".",
]
f.addStep(
ShellCommand(
name="performance-test-suite",
description=[
"LLD", "performance","test","suite",
],
command=WithProperties(" ".join(perf_command)),
workdir="./lld-speed-test",
env=merged_env
)
)
return f
def getLLVMDocsBuildFactory(clean=True,
depends_on_projects=None,
extra_configure_args=None,
env=None,
**kwargs):
if depends_on_projects is None:
# All the projects from llvm_docs, and remove all duplicates.
_depends_on_projects = list(
set([project for project in llvm_docs if project]))
else:
# Make a local copy of depends_on_projects, as we are going to modify
# that.
_depends_on_projects = depends_on_projects[:]
# Some projects are interdependent for the purpose of documentation.
# Enforce the dependencies.
# TODO: Check later the dependencies for doxygen docs and enforce them
# here if needed.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
CmakeCommand.CmakeCommand.applyDefaultOptions(cmake_args, [
("-G", "Ninja"),
("-DLLVM_ENABLE_DOXYGEN=", "ON"),
("-DLLVM_BUILD_DOCS=", "ON"),
("-DCLANG_TOOLS_EXTRA_INCLUDE_DOCS=", "ON"),
("-DLLVM_ENABLE_ASSERTIONS=", "OFF"),
("-DCMAKE_BUILD_TYPE=", "Release"),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
clean=clean,
depends_on_projects=_depends_on_projects,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
# Build the documentation for all the projects.
for project in llvm_docs:
target = llvm_docs[project][0]
# Build only those with specifies targets.
if target:
UnifiedTreeBuilder.addNinjaSteps(
f,
# Doxygen builds the final result for really
# long time without any output.
# We have to have a long timeout at this step.
timeout=10800,
targets=[target],
checks=[],
env=merged_env,
**kwargs)
# Publish just built documentation
for project in llvm_docs:
target, local_path, remote_path = llvm_docs[project]
f.addStep(
ShellCommand(
name="Publish {}".format(project or target),
description=[
"Publish", "just", "built", "documentation", "for",
"{}".format(project or target)
],
command=[
'rsync',
'-vrl',
'--delete',
'--force',
'--delay-updates',
'--delete-delay',
'--ignore-times',
'--checksum',
'-p',
'--chmod=Du=rwx,Dg=rwx,Do=rx,Fu=rw,Fg=rw,Fo=r',
"{}".format(local_path),
"lists.llvm.org:web/doxygen/{}".format(remote_path),
],
env=merged_env,
))
return f
def getABITestsuitBuildFactory(
clean = True,
depends_on_projects = None,
extra_configure_args = None, # Extra CMake args for all stages.
jobs = None, # Restrict a degree of parallelism if needed.
env = None, # Environmental variables for all steps.
**kwargs):
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
depends_on_projects = ['llvm', 'clang', 'clang-tools-extra', 'compiler-rt', 'lld']
if extra_configure_args is None:
cmake_args = list()
else:
cmake_args = list(extra_configure_args)
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
f = UnifiedTreeBuilder.getCmakeBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm",
obj_dir="build",
clean=clean,
extra_configure_args=cmake_args,
env=merged_env,
**kwargs) # Pass through all the extra arguments.
f.addStep(NinjaCommand(name="build-unified-tree",
haltOnFailure=True,
description=["Build", "unified", "tree"],
env=merged_env,
workdir=f.obj_dir,
**kwargs # Pass through all the extra arguments.
))
# Checkout the test-suite.
f.addStep(SVN(name='svn-test-suite',
mode='update', baseURL='http://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir='test-suite'))
# Run the ABI test.
abi_test_env = {
'PYTHONPATH' : WithProperties("%(workdir)s/" + f.llvm_srcdir + "/utils/lit:${PYTHONPATH}"),
'PATH' : WithProperties("%(workdir)s/" + f.obj_dir + "/bin:${PATH}"),
}
merged_env.update(abi_test_env)
abi_test_cmd = ["python", "linux-x86.py", "clang", "test", "-v"]
if jobs:
abi_test_cmd.append("-j" + str(jobs))
f.addStep(LitTestCommand(name='abi-test-suite',
command=abi_test_cmd,
description=["running", "ABI", "test-suite"],
descriptionDone=["ABI", "test-suite", "completed"],
workdir='test-suite/ABI-Testsuite',
env=merged_env))
return f