def get_configuration_build_steps(link, type, options=[]):
from buildbot.steps.slave import RemoveDirectory
from buildbot.steps.slave import MakeDirectory
from buildbot.process.properties import Interpolate
return [
MakeDirectory(name='create build directory',
description=['preparing build directory'],
descriptionDone=['create build directory'],
dir=Interpolate('%(prop:workdir)s/build/build'),
doStepIf=lambda step:
(bool(options) and
('osx' in step.build.getProperty('buildername'))),
hideStepIf=skipped_or_success),
get_cmake_step(link, type, options),
get_build_step(link, type, options),
RemoveDirectory(name='remove build directory',
description=['removing build directory'],
descriptionDone=['remove build directory'],
dir=Interpolate('%(prop:workdir)s/build/build'),
doStepIf=lambda step:
(bool(options) and
('osx' in step.build.getProperty('buildername'))),
hideStepIf=skipped_or_success)
]
def getLLVMBuildFactoryAndPrepareForSourcecodeSteps(
depends_on_projects = None,
llvm_srcdir = None,
obj_dir = None,
install_dir = None,
cleanBuildRequested = None,
env = None,
**kwargs):
def cleanBuildRequestedByProperty(step):
return step.build.getProperty("clean")
if cleanBuildRequested is None:
# We want a clean checkout only if requested by the property.
cleanBuildRequested = cleanBuildRequestedByProperty
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=cleanBuildRequested,
**kwargs) # Pass through all the extra arguments.
# Remove the source code for a clean checkout if requested by property.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(RemoveDirectory(name='clean-src-dir',
dir=f.monorepo_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequestedByProperty,
))
return f
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs = None):
gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist='+distro, '--git-arch='+arch]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name = job_name+'-clean',
dir = Interpolate('%(prop:workdir)s'),
hideStepIf = success,
)
)
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl = url,
branch = 'master',
alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev
mode = 'full' # clean out old versions
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo)
branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s_'+distro # release branch from bloom
deb_name = debian_pkg+'_%(prop:release_version)s'+distro
final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-checkout',
command = ['git', 'checkout', Interpolate(branch_name), '--force'],
hideStepIf = success
)
)
# Download script for building the source deb
f.addStep(
FileDownload(
name = job_name+'-grab-build-source-deb-script',
mastersrc = 'scripts/build_source_deb.py',
slavedest = Interpolate('%(prop:workdir)s/build_source_deb.py'),
mode = 0755,
hideStepIf = success
)
)
def get_clean_step():
from buildbot.steps.slave import RemoveDirectory
from buildbot.process.properties import Interpolate
return [
RemoveDirectory(name='clean slave',
description=['cleaning slave'],
descriptionDone=['clean slave'],
dir=Interpolate('%(prop:workdir)s'),
alwaysRun=True,
hideStepIf=skipped_or_success)
]
def add_artifact_pre_build_steps(job):
artifacts_dir = Interpolate(join('%(prop:builddir)s', 'artifacts'))
job.add_step(
SetProperty(property="artifactsdir",
value=artifacts_dir,
hideStepIf=True))
job.add_step(
RemoveDirectory(dir=Interpolate('%(prop:artifactsdir)s'),
hideStepIf=True))
job.add_step(
MakeDirectory(dir=Interpolate('%(prop:artifactsdir)s'),
hideStepIf=True))
def __init__(self,
source,
uncleanWarnings,
python="python",
reactor="epoll",
easy_install="easy_install",
dependencies=[]):
TwistedBaseFactory.__init__(self, python, source, uncleanWarnings)
self.addStep(RemoveDirectory(self.virtualenv))
setupCommands = [{
'name':
'create_virtualenv',
'description': ['creating', 'virtualenv'],
'descriptionDone': ['created', 'virtualenv'],
'command': ["virtualenv", "--never-download", self.virtualenv],
}]
for dependency in dependencies:
setupCommands.append({
'name': 'install_%s' % dependency,
'description': ['installing', dependency],
'descriptionDone': ['installed', dependency],
'command': [easy_install, dependency],
})
setupCommands.append({
'name': 'install_twisted',
'description': ['installing', 'twisted'],
'descriptionDone': ['installed', 'twisted'],
'command': [easy_install, "."]
})
for command in setupCommands:
self.addStep(
shell.ShellCommand(env={"PATH": self.virtualenvPath},
haltOnFailure=True,
**command))
self.addTrialStep(name=reactor,
reactor=reactor,
flunkOnFailure=True,
warnOnFailure=False,
workdir=self.virtualenv,
env={"PATH": self.virtualenvPath})
def getLLVMBuildFactoryAndSVNSteps(depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
install_dir=None,
cleanBuildRequested=None,
env=None,
**kwargs):
def cleanBuildRequestedByProperty(step):
return step.build.getProperty("clean")
# Set defaults
if not depends_on_projects:
depends_on_projects = ['llvm', 'clang']
if cleanBuildRequested is None:
# We want a clean checkout only if requested by the property.
cleanBuildRequested = cleanBuildRequestedByProperty
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir or "llvm",
obj_dir=obj_dir or "build",
install_dir=install_dir,
cleanBuildRequested=cleanBuildRequested,
**kwargs) # Pass through all the extra arguments.
# Do a clean checkout if requested by a build property.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(
RemoveDirectory(
name='clean-src-dir',
dir=f.llvm_srcdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequestedByProperty,
))
# Get the source code.
f.addSVNSteps()
return f
def rqg_win_factory(mtr_build_thread="130", config="Debug"):
if config == 'Debug':
do_debug_steps = True
do_release_steps = False
else:
do_debug_steps = False
do_release_steps = True
f = factory.BuildFactory()
# We shouldn't need it anymore since we are setting appverif in runall.pl now, but let it be here just in case
f.addStep(
ShellCommand(name="disable_app_verifier",
command=["dojob", "appverif", "/n", "mysqld.exe"],
doStepIf=do_release_steps))
# that's where pre-cloned trees (mariadb-server, rqg, mariadb-toolbox etc.) and local scripts are
f.addStep(
SetPropertyFromCommand(name="set_shared_dir",
property="sharedir",
command=["dojob", "echo E:\\buildbot"]))
# that's where we will build servers
f.addStep(
SetPropertyFromCommand(
name="set_bb_workdir",
property="bb_workdir",
command=["dojob",
WithProperties("echo D:\\%(buildername)s")]))
# that's where the main server (server under test) will be built
f.addStep(
SetPropertyFromCommand(
name="set_builddir",
property="builddir",
command=[
"dojob",
WithProperties("echo D:\\%(buildername)s\\build")
]))
# logdir is where vardirs are written
f.addStep(
SetPropertyFromCommand(
name="set_logdir",
property="logdir",
command=[
"dojob",
WithProperties(
"echo %(sharedir)s\\vardirs\\%(buildername)s\\%(branch)s-%(buildnumber)s"
)
]))
f.addStep(
ShellCommand(
name="close_open_handles",
command=[
"dojob",
WithProperties(
"cd /d %(bb_workdir)s && %(sharedir)s\\mariadb-tools\\buildbot\\unlock_handles.bat"
)
],
alwaysRun=True))
# f.addStep(ShellCommand(
# name= "kill_stale_mysqld",
# command=["dojob", WithProperties("taskkill /IM mysqld.exe /F || tasklist")],
# locks=[kill_mysqld_lock.access('exclusive')]
# ));
f.addStep(
ShellCommand(
name="kill_stale_mysqld",
command=[
"dojob",
WithProperties(
"PowerShell -Command \"Get-WmiObject -class Win32_Process | Where-Object { $_.Name -eq 'mysqld.exe' -and $_.Path -like '*\%(buildername)s\*'} | Select-Object\" && PowerShell -Command \"Get-WmiObject -class Win32_Process | Where-Object { $_.Name -eq 'mysqld.exe' -and $_.Path -like '*\%(buildername)s\*'} | Remove-WmiObject\" && PowerShell -Command \"Get-WmiObject -class Win32_Process | Where-Object { $_.Name -eq 'mysqld.exe' -and $_.Path -like '*\%(buildername)s\*'} | Select-Object\""
)
]))
f.addStep(
RemoveDirectory(name="remove_old_builds",
dir=WithProperties("%(bb_workdir)s")))
f.addStep(
RemoveDirectory(name="remove_old_logs",
dir=WithProperties("%(logdir)s")))
f.addStep(
ShellCommand(
name="create_dirs",
command=[
"dojob",
WithProperties(
"mkdir %(bb_workdir)s && mkdir %(bb_workdir)s\\build-last-release"
)
],
timeout=3600))
f.addStep(
ShellCommand(
name="pull_server",
command=[
"dojob",
WithProperties(
"git clone %(sharedir)s\\mariadb-server %(builddir)s && cd /d %(builddir)s && git remote set-url origin %(repository)s && git pull && git reset --hard %(revision)s"
)
],
timeout=3600))
f.addStep(
ShellCommand(
name="pull_rqg_and_tools",
command=[
"dojob",
WithProperties(
"cd /d %(sharedir)s\\rqg && git pull && cd /d %(sharedir)s\\mariadb-toolbox && git pull"
)
],
locks=[git_rqg_lock.access('exclusive')],
timeout=3600))
f.addStep(
SetPropertyFromCommand(
name="get_generator",
property="vs_generator",
command=[
"dojob",
WithProperties("cat %(sharedir)s\\vs_generator.txt")
]))
f.addStep(
ShellCommand(
name="version_info",
command=[
"dojob",
WithProperties(
"cd /d %(builddir)s && git log -1 && cd /d %(sharedir)s\\rqg && git log -1 && cd /d %(sharedir)s\\mariadb-toolbox && git log -1"
)
],
timeout=3600))
f.addStep(
Compile(
name="build",
command=[
"dojob",
WithProperties(
"cd /d %(builddir)s && cmake . -G %(vs_generator)s && cmake --build . --config "
+ config)
],
warningPattern=vsWarningPattern,
warningExtractor=Compile.warnExtractFromRegexpGroups))
# We shouldn't need it anymore since we are setting appverif in runall.pl now
# f.addStep(Test(
# name = "enable_app_verifier",
# doStepIf=do_release_steps,
# command=["dojob", "appverif", "/verify", "mysqld.exe"]
# ));
# storage tests are currently broken on 10.2 (MDEV-9705)
f.addStep(
getMTR(
doStepIf=do_release_steps,
name="storage_engine",
test_type="storage_engine",
test_info="Storage engine test suites",
timeout=3600,
command=[
"dojob",
WithProperties(
"cd /d %(builddir)s\\mysql-test && perl mysql-test-run.pl --verbose-restart --force --suite=storage_engine-,storage_engine/*- --max-test-fail=0 --parallel=4"
)
]))
f.addStep(
Test(
doStepIf=do_release_steps,
name="transform",
timeout=3600,
env={"MTR_BUILD_THREAD": mtr_build_thread},
command=[
"dojob",
WithProperties(
"cd /d %(sharedir)s\\rqg && perl combinations.pl --config=%(sharedir)s\\mariadb-toolbox\\configs\\buildbot-transform.cc --run-all-combinations-once --force --basedir=%(builddir)s --workdir=%(logdir)s\\optim-transform || perl %(sharedir)s\\mariadb-toolbox\\scripts\\result_summary.pl %(logdir)s\\optim-transform\\trial*"
)
]))
# We shouldn't need it anymore since we are setting appverif in runall.pl now
# f.addStep(ShellCommand(
# doStepIf=do_release_steps,
# name= "disable_app_verifier_again",
# command=["dojob", "appverif", "/n", "mysqld.exe"]
# alwaysRun=True
# ));
f.addStep(
Test(
doStepIf=do_debug_steps,
name="crash_tests",
timeout=3600,
env={"MTR_BUILD_THREAD": mtr_build_thread},
command=[
"dojob",
WithProperties(
"cd /d %(sharedir)s\\rqg && perl combinations.pl --config=%(sharedir)s\\mariadb-toolbox\\configs\\buildbot-no-comparison.cc --run-all-combinations-once --force --basedir=%(builddir)s --workdir=%(logdir)s\\optim-crash-tests || perl %(sharedir)s\\mariadb-toolbox\\scripts\\result_summary.pl %(logdir)s\\optim-crash-tests\\trial*"
)
]))
f.addStep(
ShellCommand(
name="get_previous_release",
command=[
"dojob",
WithProperties(
"perl %(sharedir)s\\mariadb-toolbox\\scripts\\last_release_tag.pl --source-tree=%(builddir)s --dest-tree=%(bb_workdir)s/build-last-release"
)
],
timeout=3600,
doStepIf=do_release_steps))
f.addStep(
Compile(
doStepIf=do_release_steps,
name="build_previous_release",
command=[
"dojob",
WithProperties(
"cd /d %(bb_workdir)s\\build-last-release && cmake . -G %(vs_generator)s && cmake --build . --config RelWithDebInfo"
)
],
timeout=3600,
warningPattern=vsWarningPattern,
warningExtractor=Compile.warnExtractFromRegexpGroups))
f.addStep(
Test(
doStepIf=do_release_steps,
name="comparison",
timeout=3600,
env={"MTR_BUILD_THREAD": mtr_build_thread},
command=[
"dojob",
WithProperties(
"cd /d %(sharedir)s\\rqg && perl combinations.pl --config=%(sharedir)s\\mariadb-toolbox\\configs\\buildbot-comparison.cc --run-all-combinations-once --force --basedir1=%(builddir)s --basedir2=%(bb_workdir)s\\build-last-release --workdir=%(logdir)s\\optim-comparison || perl %(sharedir)s\\mariadb-toolbox\\scripts\\result_summary.pl %(logdir)s\\optim-comparison\\trial*"
)
]))
# f.addStep(Test(
# name = "app_verifier",
# doStepIf=False,
# command=["dojob", WithProperties("dir %(logdir)s && appverif -export log -for mysqld.exe -with to=%(logdir)s\\appverif.xml && cat %(logdir)s\\appverif.xml")]
# ));
return f
def _addSteps4SystemCompiler(
f,
stage_idx = 0,
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
# This stage could use incremental build.
# Clean stage1, only if requested.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=clean
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'OFF'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
# Do not expect warning free build by the system toolchain.
('-DLLVM_ENABLE_WERROR=', 'OFF'),
('-DCMAKE_COLOR_MAKEFILE=', 'OFF'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Unix Makefiles'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
# Note: On this stage we do not care of warnings, as we build with
# a system toolchain and cannot control the environment.
# Warnings are likely, and we ignore them.
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
flunkOnWarnings=False,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build clang by the system compiler
f.addStep(ShellCommand(name="build-stage%s-compiler"% stage_num,
command=['make', WithProperties("-j%s" % jobs), "-k"],
haltOnFailure=True,
flunkOnWarnings=False,
description=["build stage%s compiler" % stage_num],
env=env,
workdir=obj_dir))
# Test stage1 compiler
f.addStep(ShellCommand(name="test-stage%s-compiler"% stage_num,
command=["make", WithProperties("-j%s" % jobs), "check-all"], # or "check-llvm", "check-clang"
haltOnFailure=True,
flunkOnWarnings=False,
description=["test stage%s compiler" % stage_num],
env=env,
workdir=obj_dir))
# Install stage1 compiler
f.addStep(ShellCommand(name="install-stage%s-compiler"% stage_num,
command=["make", WithProperties("-j%s" % jobs), "install"],
haltOnFailure=True,
flunkOnWarnings=False,
description=["install stage%s compiler" % stage_num],
env=env,
workdir=obj_dir))
def getClangWithLTOBuildFactory(
depends_on_projects = None,
clean = False,
jobs = None,
extra_configure_args = None,
compare_last_2_stages = True,
lto = None, # The string gets passed to -flto flag as is. Like -flto=thin.
env = None):
# Set defaults
if depends_on_projects:
depends_on_projects = list(depends_on_projects)
else:
# By default we link with LLD.
depends_on_projects = ['llvm', 'clang', 'lld']
if lto is None:
lto = 'ON'
if jobs is None:
jobs = "%(jobs)s"
if extra_configure_args is None:
extra_configure_args = []
else:
extra_configure_args = list(extra_configure_args)
# Make sure CMAKE_INSTALL_PREFIX and -G are not specified
# in the extra_configure_args. We set them internally as needed.
# TODO: assert extra_configure_args.
install_prefix_specified = any(a.startswith('-DCMAKE_INSTALL_PREFIX=') for a in extra_configure_args)
assert True, "Please do not explicitly specify the install prefix for multi-stage build."
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir="llvm.src",
stage_objdirs=[
"build/stage1",
"build/stage2",
"build/stage3",
"build/stage4",
],
stage_installdirs=[
"install/stage1",
"install/stage2",
"install/stage3",
"install/stage4",
],
staged_compiler_idx = 1)
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
# Do a clean checkout if requested.
f.addStep(RemoveDirectory(name='clean-src-dir',
dir=f.llvm_srcdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Get the source code.
f.addSVNSteps()
# Build with the system compiler first
_addSteps4SystemCompiler(f,
stage_idx=0,
clean=cleanBuildRequested,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Then build the compiler we would use for the bootstrap.
_addSteps4StagedCompiler(f,
stage_idx=1,
withLTOSupport=True,
jobs=jobs,
extra_configure_args=extra_configure_args,
env=merged_env)
# Build all the remaining stages with exactly the same configuration.
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LTO=', lto),
])
# If we build LLD, we would link with LLD.
# Otherwise we link with the system linker.
if 'lld' in depends_on_projects:
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DLLVM_ENABLE_LLD=', 'ON'),
])
# The rest are test stages, which depend on the staged compiler we are ultimately after.
s = f.staged_compiler_idx + 1
staged_install = f.stage_installdirs[f.staged_compiler_idx]
for i in range(s, len(f.stage_objdirs[s:]) + s):
configure_args = extra_configure_args[:]
configure_args.append(
WithProperties(
"-DCMAKE_AR=%(workdir)s/" + staged_install + "/bin/llvm-ar"
))
configure_args.append(
WithProperties(
"-DCMAKE_RANLIB=%(workdir)s/" + staged_install + "/bin/llvm-ranlib"
))
_addSteps4StagedCompiler(f,
stage_idx=i,
use_stage_idx=f.staged_compiler_idx,
jobs=jobs,
extra_configure_args=configure_args,
env=merged_env)
if compare_last_2_stages:
# Compare the compilers built on the last 2 stages if requested.
diff_command = [
"diff",
"-q",
f.stage_installdirs[-2] + "/bin/clang",
f.stage_installdirs[-1] + "/bin/clang",
]
f.addStep(
ShellCommand(
name="compare-compilers",
description=[
"compare",
"stage%d" % (len(f.stage_installdirs)-1),
"and",
"stage%d" % len(f.stage_installdirs),
"compilers",
],
haltOnFailure=True,
command=WithProperties(" ".join(diff_command)),
workdir=".",
env=merged_env
)
)
return f
def _addSteps4StagedCompiler(
f,
stage_idx = 1,
use_stage_idx = -1,
withLTOSupport = False,
jobs = None,
extra_configure_args = None,
env = None):
if use_stage_idx < 0:
use_stage_idx = stage_idx - 1
# Index is zero-based, so we want to use a human friendly number instead.
stage_num = stage_idx + 1
# Directories to use on this stage.
obj_dir = f.stage_objdirs[stage_idx]
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
install_dir = LLVMBuildFactory.pathRelativeToBuild(f.stage_installdirs[stage_idx], obj_dir)
staged_install = f.stage_installdirs[use_stage_idx]
# Always do a clean build for the staged compiler.
f.addStep(RemoveDirectory(name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
))
f.addStep(RemoveDirectory(name='clean-%s-dir' % f.stage_installdirs[stage_idx],
dir=f.stage_installdirs[stage_idx],
haltOnFailure=False,
flunkOnFailure=False,
))
# Reconcile the cmake options for this stage.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if withLTOSupport:
CmakeCommand.applyDefaultOptions(cmake_args, [
# LTO Plugin dependency:
('-DLLVM_BINUTILS_INCDIR=', '/opt/binutils/include'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
('-DCMAKE_INSTALL_PREFIX=', install_dir),
])
cmake_args.append(
WithProperties(
"-DCMAKE_CXX_COMPILER=%(workdir)s/" + staged_install + "/bin/clang++"
))
cmake_args.append(
WithProperties(
"-DCMAKE_C_COMPILER=%(workdir)s/" + staged_install + "/bin/clang"
))
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure-stage%s" % stage_num,
description=["stage%s cmake configure" % stage_num],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt")
))
if withLTOSupport:
# Build LTO plugin if requested.
f.addStep(NinjaCommand(name="build-stage%s-LLVMgold.so" % stage_num,
targets=['lib/LLVMgold.so'],
haltOnFailure=True,
description=["stage%s build LLVMgold.so" % stage_num],
env=env,
workdir=obj_dir,
))
# Build clang by the staged compiler
f.addStep(NinjaCommand(name="build-stage%s-compiler" % stage_num,
haltOnFailure=True,
description=["build stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Test just built compiler
f.addStep(NinjaCommand(name="test-stage%s-compiler"% stage_num,
targets=["check-all"],
haltOnFailure=True,
description=["test stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
# Install just built compiler
f.addStep(NinjaCommand(name="install-stage%s-compiler"% stage_num,
targets=["install"],
haltOnFailure=True,
description=["install stage%s compiler" % stage_num],
timeout=10800, # LTO could take time.
env=env,
workdir=obj_dir,
))
def getLLDWinBuildFactory(
clean = True,
# Default values for VS devenv and build configuration
vs = None, # What to run to configure Visual Studio utils.
target_arch = None, # Native.
extra_configure_args = None,
env = None):
# Set defaults
if vs is None:
vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015.
if extra_configure_args is None:
extra_configure_args = []
if env is None:
env = {}
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# If set up environment step is requested, do this now.
if vs:
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
# Always build with ninja.
cmake_options = ["-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmake_options.append('-DCMAKE_BUILD_TYPE=Release')
if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_WERROR=ON')
if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"')
cmake_options += extra_configure_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
warnOnWarnings=True,
options=cmake_options,
path="../%s" % f.llvm_srcdir,
env=env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld.
f.addStep(NinjaCommand(name='build lld',
haltOnFailure=True,
warnOnWarnings=True,
description='build lld',
workdir=f.llvm_objdir,
env=env))
# Test Lld
f.addStep(NinjaCommand(name='test lld',
targets=['lld-test'],
haltOnFailure=True,
warnOnWarnings=True,
description='test lld',
workdir=f.llvm_objdir,
env=env))
return f
def ros_debbuild(c,
job_name,
packages,
url,
distro,
arch,
rosdistro,
version,
machines,
othermirror,
keys,
trigger_pkgs=None):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:workdir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch='master',
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full' # clean out old versions
))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s_' + distro # release branch from bloom
deb_name = debian_pkg + '_%(prop:release_version)s' + distro
final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# Build the source deb
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-buildsource',
command=['git-buildpackage', '-S'] + gbp_args,
descriptionDone=['sourcedeb', package]))
# Upload sourcedeb to master (currently we are not actually syncing these with a public repo)
f.addStep(
FileUpload(
name=package + '-uploadsource',
slavesrc=Interpolate('%(prop:workdir)s/' + deb_name + '.dsc'),
masterdest=Interpolate('sourcedebs/' + deb_name + '.dsc'),
hideStepIf=success))
# Stamp the changelog, in a similar fashion to the ROS buildfarm
f.addStep(
SetPropertyFromCommand(command="date +%Y%m%d-%H%M-%z",
property="datestamp",
name=package + '-getstamp',
hideStepIf=success))
f.addStep(
ShellCommand(
haltOnFailure=True,
name=package + '-stampdeb',
command=[
'git-dch', '-a', '--ignore-branch', '--verbose', '-N',
Interpolate('%(prop:release_version)s-%(prop:datestamp)s' +
distro)
],
descriptionDone=[
'stamped changelog',
Interpolate('%(prop:release_version)s'),
Interpolate('%(prop:datestamp)s')
]))
# download hooks
f.addStep(
FileDownload(
name=package + '-grab-hooks',
mastersrc='hooks/D05deps',
slavedest=Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf=success,
mode=0777 # make this executable for the cowbuilder
))
# build the binary from the git working copy
f.addStep(
ShellCommand(
haltOnFailure=True,
name=package + '-buildbinary',
command=[
'git-buildpackage', '--git-pbuilder', '--git-export=WC',
Interpolate('--git-export-dir=%(prop:workdir)s')
] + gbp_args,
env={
'DIST':
distro,
'GIT_PBUILDER_OPTIONS':
Interpolate(
'--hookdir %(prop:workdir)s/hooks --override-config'),
'OTHERMIRROR':
othermirror
},
descriptionDone=['binarydeb', package]))
# Upload binarydeb to master
f.addStep(
FileUpload(name=package + '-uploadbinary',
slavesrc=Interpolate('%(prop:workdir)s/' + final_name),
masterdest=Interpolate('binarydebs/' + final_name),
hideStepIf=success))
# Add the binarydeb using reprepro updater script on master
f.addStep(
MasterShellCommand(name=package + 'includedeb',
command=[
'reprepro-include.bash', debian_pkg,
Interpolate(final_name), distro, arch
],
descriptionDone=['updated in apt', package]))
# Trigger if needed
if trigger_pkgs != None:
f.addStep(
Trigger(schedulerNames=[
t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' +
arch + '-debtrigger' for t in trigger_pkgs
],
waitForFinish=False,
alwaysRun=True))
# Create trigger
c['schedulers'].append(
triggerable.Triggerable(
name=job_name.replace('_', '-') + '-' + rosdistro + '-' + distro +
'-' + arch + '-debtrigger',
builderNames=[
job_name + '_' + rosdistro + '_' + distro + '_' + arch +
'_debbuild',
]))
# Add to builders
c['builders'].append(
BuilderConfig(name=job_name + '_' + rosdistro + '_' + distro + '_' +
arch + '_debbuild',
properties={'release_version': version},
slavenames=machines,
factory=f))
# return name of builder created
return job_name + '_' + rosdistro + '_' + distro + '_' + arch + '_debbuild'
def getToolchainBuildFactory(
clean=False,
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = LLVMBuildFactory(
depends_on_projects=[
"llvm",
"clang",
"clang-tools-extra",
"compiler-rt",
"libcxx",
"libcxxabi",
"libunwind",
"lld"
])
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(RemoveDirectory(name="clean-sdk",
dir=sdk_dir,
haltOnFailure=True))
f.addStep(ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean
# Clean up llvm sources.
f.addStep(RemoveDirectory(name="clean-llvm.src",
dir=src_dir,
haltOnFailure=True,
doStepIf=cleanCheckoutRequested))
# Get sources.
for project in f.depends_on_projects:
_, baseURL = svn_repos[project]
f.addStep(SVN(name="svn-%s" % project,
workdir=src_dir + "/" + project,
baseURL=baseURL))
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
# Clean up llvm build.
f.addStep(RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties(
"-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir
))
cmake_options.append(
WithProperties(
"-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir
))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(NinjaCommand(name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def getCUDATestsuiteBuildFactory(
externals, # Directory with CUDA, thrust and gcc versions for testing.
always_clean=True,
test=False,
useTwoStage=False,
cmake='cmake',
extra_cmake_args=None, # Extra CMake args for all stages.
extra_ts_cmake_args=None, # extra cmake args for testsuite.
jobs=None,
cuda_jobs=1, # number of simultaneous CUDA apps to run
env=None, # Environmental variables for all steps.
enable_thrust_tests=False,
split_thrust_tests=False, # Each thrust test is a separate executable.
run_thrust_tests=False,
enable_libcxx=True, # checkout/build libcxx to test with.
gpu_arch_list=None,
gpu_devices=None, # List of devices to make visible to CUDA
stage1_config='Release',
stage2_config='Release'):
if extra_cmake_args is None:
extra_cmake_args = []
if extra_ts_cmake_args is None:
extra_ts_cmake_args = []
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
source_dir = 'llvm' # Should match the one used in getClangCMakeBuildFactory.
stage1_build_dir = 'stage1' # Should match the one defined in getClangCMakeBuildFactory.
stage2_build_dir = 'stage2' # Should match the one defined in getClangCMakeBuildFactory.
install_dir = 'clang.install'
if useTwoStage:
clang_build_dir = stage2_build_dir
else:
clang_build_dir = stage1_build_dir
# Build clang.
f = ClangBuilder.getClangCMakeBuildFactory(
clean=always_clean,
test=test,
cmake=cmake,
extra_cmake_args=extra_cmake_args,
jobs=jobs,
env=merged_env,
useTwoStage=useTwoStage,
stage1_config=stage1_config,
stage2_config=stage2_config,
checkout_clang_tools_extra=False,
checkout_compiler_rt=False,
checkout_libcxx=enable_libcxx,
checkout_lld=False,
checkout_test_suite=True)
cuda_test_env = {
'PYTHONPATH':
WithProperties("%(workdir)s/" + source_dir +
"/utils/lit:${PYTHONPATH}"),
'DESTDIR':
WithProperties("%(workdir)s/" + install_dir),
'PATH':
WithProperties("%(workdir)s/" + install_dir +
"/usr/local/bin:${PATH}"),
}
merged_env.update(cuda_test_env)
ts_build_dir = 'test-suite-build'
f.addStep(
RemoveDirectory(name="Remove old clang install directory",
dir=install_dir))
# Install clang into directory pointed by $DESTDIR
f.addStep(
NinjaCommand(name='ninja install clang',
targets=["install"],
jobs=jobs,
haltOnFailure=True,
description=split("installing clang"),
descriptionDone=split("Clang installation is done."),
workdir=clang_build_dir,
env=merged_env))
# Completely remove test suite build dir.
f.addStep(
RemoveDirectory(name="Remove old test-suite build directory",
dir=ts_build_dir))
if extra_ts_cmake_args:
cmake_args = extra_ts_cmake_args[:]
else:
cmake_args = []
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
])
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cmake_args.append(
WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" + clang_build_dir +
"/bin/clang++"))
cmake_args.append(
WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + clang_build_dir +
"/bin/clang"))
cmake_args.append('-DTEST_SUITE_SUBDIRS=External'),
# Limit to External tests only.
if externals:
cmake_args.append('-DTEST_SUITE_EXTERNALS_DIR=' + externals)
if split_thrust_tests:
cmake_args.append('-DTHRUST_SPLIT_TESTS=1')
if gpu_arch_list:
cmake_args.append('-DCUDA_GPU_ARCH=' + ';'.join(gpu_arch_list))
if cuda_jobs:
cmake_args.append('-DCUDA_JOBS=%s' % cuda_jobs)
# Then do fresh cmake configuration.
f.addStep(
CmakeCommand(name='cmake test-suite',
description='cmake test-suite',
haltOnFailure=True,
options=cmake_args,
path="../test/test-suite",
workdir=ts_build_dir,
env=merged_env))
# Always build simple CUDA tests. They serve as compilation
# smoketests and will fail quickly if compiler has obvious issues
# compiling CUDA files.
f.addStep(
NinjaCommand(name='ninja build simple CUDA tests',
targets=["cuda-tests-simple"],
jobs=jobs,
haltOnFailure=True,
description=split("building simple CUDA tests"),
descriptionDone=split("simple CUDA tests built."),
workdir=ts_build_dir,
env=merged_env))
# Limit GPUs visible to CUDA.
if gpu_devices:
for gpu_id in gpu_devices:
# make ID a string as it may be either an integer or a UUID string.
gpu_id = str(gpu_id)
gpu_env = dict(merged_env)
gpu_env["CUDA_VISIBLE_DEVICES"] = gpu_id
f.addStep(
NinjaCommand(
name='run simple CUDA tests on gpu %s' % gpu_id,
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=split("Running simple CUDA tests on GPU %s" %
gpu_id),
descriptionDone=split("simple CUDA tests on GPU %s done." %
gpu_id),
workdir=ts_build_dir,
env=gpu_env))
else:
f.addStep(
NinjaCommand(
name='run simple CUDA tests',
targets=["check-cuda-simple"],
jobs=1, # lit will parallelize the jobs
haltOnFailure=True,
description=split("Running simple CUDA tests"),
descriptionDone=split("simple CUDA tests done."),
workdir=ts_build_dir,
env=merged_env))
# If we've enabled thrust tests, build them now.
# WARNING: This takes a lot of time to build.
if (enable_thrust_tests):
f.addStep(
NinjaCommand(name='ninja build thrust',
targets=["cuda-tests-thrust"],
jobs=jobs,
haltOnFailure=True,
description=split("building thrust tests"),
descriptionDone=split("thrust tests built."),
workdir=ts_build_dir,
env=merged_env))
# Run them. That also takes a while.
# TODO(tra) we may want to run more than one instance so one
# can be compiling tests while another is running them on GPU.
if run_thrust_tests:
f.addStep(
NinjaCommand(
name='run all CUDA tests',
targets=["check"],
jobs=1, # lit will parallelize the jobs.
haltOnFailure=True,
description=split("running all CUDA tests."),
descriptionDone=split("all cuda tests done."),
workdir=ts_build_dir,
env=merged_env))
return f
def ros_branch_build(c, job_name, packages, url, branch, distro, arch, rosdistro, machines, othermirror, keys):
gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist='+distro, '--git-arch='+arch]
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name = job_name+'-clean',
dir = Interpolate('%(prop:workdir)s'),
hideStepIf = success,
)
)
# Pulling the repo
f.addStep(
Git(
repourl = url,
branch = 'HEAD',
alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev
mode = 'full', # clean out old versions
getDescription={'tags': True}
)
)
# Check out the repository branch/commit/tag
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'checkout: ' + branch,
command = ['git', 'checkout', branch],
)
)
# get the short commit hash
f.addStep(
SetPropertyFromCommand(
command="git rev-parse --short HEAD", property="commit_hash",
name = 'commit-short-hash',
hideStepIf = success
)
)
# get the time stamp
f.addStep(
SetPropertyFromCommand(
command='date +%Y%M%d%H', property='date_stamp',
name = 'date-stamp',
hideStepIf = success
)
)
# Update the cowbuilder
f.addStep(
ShellCommand(
command = ['cowbuilder-update.py', distro, arch] + keys,
hideStepIf = success
)
)
# Generate the changelog for the package
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'catkin_generate_changelog',
command= ['catkin_generate_changelog', '-y'],
descriptionDone = ['catkin_generate_changelog']
)
)
# Add all files including untracked ones
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'add_changelogs',
command= ['git', 'add', '.'],
descriptionDone = ['add_changelogs']
)
)
# Commit the changelog after updating it
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'update_changelogs',
command= ['git', 'commit', '-m', '\"Updated changelogs\"'],
descriptionDone = ['update_changelogs']
)
)
# Prepare the release without pushing it
# Set very big number to avoid conflicts with available tags
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'catkin_prepare_release',
command= ['catkin_prepare_release', '--version', '100.0.0', '--no-push', '-y'],
descriptionDone = ['catkin_prepare_release']
)
)
#
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'git_bloom_generate_release',
command = ['git-bloom-generate', '-y', 'rosrelease', rosdistro],
)
)
f.addStep(
ShellCommand(
haltOnFailure = True,
name = 'git_bloom_generate_debian',
command = ['git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release', rosdistro],
)
)
# Get the tag number for the lastest commit
f.addStep(
SetPropertyFromCommand(
command="git describe --tags", property="release_version",
name = 'latest_tag',
)
)
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo)
branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s-0_'+distro
deb_name = debian_pkg+'_%(prop:release_version)s-0'+distro
final_name = debian_pkg+'_%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro+'_'+arch+'.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-checkout',
command = ['git', 'checkout', Interpolate(branch_name), '--force'],
hideStepIf = success
)
)
# Stamp the changelog
f.addStep(
ShellCommand(
haltOnFailure = True,
name = package+'-stampdeb',
command = ['gbp', 'dch', '-a', '--ignore-branch', '--verbose',
'-N', Interpolate('%(prop:release_version)s-%(prop:date_stamp)s-%(prop:commit_hash)s-'+distro)],
descriptionDone = ['stamped changelog', Interpolate('%(prop:release_version)s'),
Interpolate('%(prop:date_stamp)s'), Interpolate('%(prop:commit_hash)s')]
)
)
# download hooks
f.addStep(
FileDownload(
name = package+'-grab-hooks',
mastersrc = 'hooks/D05deps',
slavedest = Interpolate('%(prop:workdir)s/hooks/D05deps'),
hideStepIf = success,
mode = 0777 # make this executable for the cowbuilder
)
)
# Download script for building the binary deb
f.addStep(
FileDownload(
name = job_name+'-grab-build-binary-deb-script',
mastersrc = 'scripts/build_binary_deb.py',
slavedest = Interpolate('%(prop:workdir)s/build_binary_deb.py'),
mode = 0755,
hideStepIf = success
)
)
def getLLDBWindowsCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=r'C:/src/python',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[],
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
if config.lower() == 'release':
python_lib = 'python27.lib'
python_exe = 'python.exe'
elif config.lower() == 'debug':
python_lib = 'python27_d.lib'
python_exe = 'python_d.exe'
python_lib = os.path.join(python_source_dir, 'PCbuild', python_lib)
python_exe = os.path.join(python_source_dir, 'PCbuild', python_exe)
python_include = os.path.join(python_source_dir, 'Include')
# Use batch files instead of ShellCommand directly, Windows quoting is
# borked. FIXME: See buildbot ticket #595 and buildbot ticket #377.
f.addStep(
batch_file_download.BatchFileDownload(
name='cmakegen',
command=[
cmake,
"-G",
"Ninja",
"../llvm",
"-DCMAKE_BUILD_TYPE=" + config,
# Need to use our custom built version of python
'-DPYTHON_LIBRARY=' + python_lib,
'-DPYTHON_INCLUDE_DIR=' + python_include,
'-DPYTHON_EXECUTABLE=' + python_exe,
"-DCMAKE_INSTALL_PREFIX=../install"
] + extra_cmake_args,
workdir=build_dir))
f.addStep(
ShellCommand(name='cmake',
command=['cmakegen.bat'],
haltOnFailure=True,
description='cmake gen',
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
# Upload the pdb zip file.
FileUpload(slavesrc=pdb_filename, masterdest=pdb_upload_filename,
mode=0o664, haltOnFailure=False),
] + publish_exe_steps
# Now make the factories.
sdk_factory = BuildFactory()
for step in checkout_steps + whl_steps + build_steps + publish_sdk_steps:
sdk_factory.addStep(step)
runtime_factory = BuildFactory()
for step in checkout_steps + build_steps + publish_exe_steps:
runtime_factory.addStep(step)
rtdist_factory = BuildFactory()
rtdist_factory.addStep(RemoveDirectory(dir="built/slave"))
for step in checkout_steps + build_steps + publish_rtdist_steps:
rtdist_factory.addStep(step)
def windows_builder(buildtype, arch):
if buildtype == "rtdist":
factory = rtdist_factory
elif buildtype == "runtime":
factory = runtime_factory
else:
factory = sdk_factory
if arch == "amd64":
platform = "win_amd64"
else:
def mkbuildsfactory():
f = factory.BuildFactory()
f.addSteps([
gitStep,
VirtualenvSetup(name='virtualenv setup',
no_site_packages=True,
virtualenv_packages=[
# required to build www packages (#2877)
'--editable=master', '--editable=pkg', 'mock',
# required to make wheels
'wheel',
# ..and this is required to actually build them correctly (#2883)
'setuptools'
],
virtualenv_dir='../sandbox',
haltOnFailure=True),
RemoveDirectory(dir='build/uploads'),
MakeDirectory(dir='build/uploads'),
])
for name, workdir, command in [
('buildbot', 'build/master', 'sdist'),
('buildbot-slave', 'build/slave', 'sdist'),
('buildbot-pkg', 'build/pkg', 'sdist'),
('buildbot-www', 'build/www/base', 'bdist_wheel'),
('buildbot-console-view', 'build/www/console_view', 'bdist_wheel'),
('buildbot-waterfall-view', 'build/www/waterfall_view', 'bdist_wheel'),
('buildbot-codeparameter', 'build/www/codeparameter', 'bdist_wheel'),
]:
levels = workdir.count('/') + 1
sandbox = "../" * levels + "sandbox"
extension = {'sdist': '.tar.gz', 'bdist_wheel': '-py2-none-any.whl'}[command]
pkgname = name
if command == 'bdist_wheel':
pkgname = pkgname.replace('-', '_') # wheels always use _
f.addSteps([
# the 'buildbot-www' package requires that the sandbox be *active* so that it
# can find 'buildbot'
ShellCommand(command="rm -rf dist/*; . %s/bin/activate; python setup.py %s"
% (sandbox, command), workdir=workdir,
name=name, flunkOnFailure=True, haltOnFailure=False,
env={'BUILDBOT_VERSION': '1latest'}), # wheels require a digit
ShellCommand(command="""
mv %(workdir)s/dist/%(pkgname)s-1latest%(extension)s build/uploads
""" % dict(pkgname=pkgname, extension=extension, workdir=workdir),
name=name + " mv", flunkOnFailure=True, haltOnFailure=False,
workdir='.'),
])
# now upload all of those. SFTP is annoying to script.
script = textwrap.dedent("""\
# get the version from git, since 'buildbot.version' only has the x.y.z prefix
VERSION=$(git describe --tags --always)
version=$(mktemp)
echo $VERSION >${version}
readme=$(mktemp)
cat <<EOF >${readme}
This directory contains the latest versions of Buildbot, packaged as they would
be for a release.
These are most useful for developers wishing to use a working web UI without installing Node:
pip intall http:s//ftp.buildbot.net/latest/buildbot-www-1latest-py2-none-any.whl
The packages here are for ${VERSION}.
EOF
batchfile=$(mktemp)
cat <<EOF >${batchfile}
cd pub/latest
put uploads/*
put ${readme} README.txt
chmod 644 README.txt
put ${version} VERSION.txt
chmod 644 VERSION.txt
EOF
sftp \
-b ${batchfile} \
-oPort=2200 \
-oIdentityFile=~/.ssh/ftp.key \
[email protected]
rv=$?
rm ${batchfile} ${readme} ${version}
exit $rv""")
f.addStep(ShellCommand(command=script, flunkOnFailure=True))
return f
def addCmakeSteps(f,
cleanBuildRequested,
obj_dir,
install_dir=None,
extra_configure_args=None,
env=None,
stage_name=None,
**kwargs):
# Consume is_legacy_mode if given.
# TODO: Remove this once legacy mode gets dropped.
kwargs.pop('is_legacy_mode', None)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if obj_dir is None:
obj_dir = f.obj_dir
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
if not f.is_legacy_mode:
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
install_dir, obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % install_dir,
dir=install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
if stage_name:
step_name = "cmake-configure-%s" % stage_name
else:
stage_name = ""
step_name = "cmake-configure"
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
f.addStep(
CmakeCommand(
name=step_name,
haltOnFailure=True,
description=["Cmake", "configure", stage_name],
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def getLLDBuildFactory(
clean = True,
jobs = None,
extra_configure_args = None,
env = None):
# Set defaults
if jobs is None:
jobs = "%(jobs)s"
if extra_configure_args is None:
extra_configure_args = []
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'CC' : "clang",
'CXX' : "clang++",
'TERM' : 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# Create configuration files with cmake
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=extra_configure_args,
path="../%s" % f.llvm_srcdir,
env=merged_env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld
f.addStep(ShellCommand(name="build_Lld",
command=['nice', '-n', '10',
'make', WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["build lld"],
env=merged_env,
workdir=f.llvm_objdir))
# Test Lld
f.addStep(ShellCommand(name="test_lld",
command=["make", "lld-test"],
haltOnFailure=True,
description=["test lld"],
env=merged_env,
workdir=f.llvm_objdir))
return f
def getPollyBuildFactory(
clean=False,
install=False,
make='make',
jobs=None,
checkAll=False,
env=None,
extraCmakeArgs=None,
testsuite=False,extraTestsuiteCmakeArgs=None,
**kwargs):
if extraCmakeArgs is None:
extraCmakeArgs=[]
if extraTestsuiteCmakeArgs is None:
extraTestsuiteCmakeArgs = []
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llvm_instdir = "llvm.inst"
testsuite_srcdir = "test-suite.src"
testsuite_builddir = "test-suite.build"
jobs_cmd = []
if jobs is not None:
jobs_cmd = ["-j"+str(jobs)]
build_cmd = [make] + jobs_cmd
install_cmd = [make, 'install'] + jobs_cmd
check_all_cmd = [make, 'check-all'] + jobs_cmd
check_polly_cmd = [make, 'check-polly'] + jobs_cmd
cmake_install = []
if install:
cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir]
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything.
depends_on_projects = ['llvm','clang','polly']
if testsuite:
# XRay tests in test-suite require compiler-rt
depends_on_projects += ['compiler-rt']
cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False)
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
install_dir=llvm_instdir,
cleanBuildRequested=cleanBuildRequested,
**kwargs) # Pass through all the extra arguments.
f.addStep(RemoveDirectory(name='clean-src-dir',
dir=f.monorepo_dir,
warnOnFailure=True,
doStepIf=cleanBuildRequestedByProperty))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Clean build dir
f.addStep(RemoveDirectory(name='clean-build-dir',
dir=llvm_objdir,
warnOnFailure=True,
doStepIf=cleanBuildRequested))
# Create configuration files with cmake
cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir,
"-DCMAKE_COLOR_MAKEFILE=OFF",
"-DPOLLY_TEST_DISABLE_BAR=ON",
"-DPOLLY_ENABLE_GPGPU_CODEGEN=ON",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_POLLY_LINK_INTO_TOOLS=ON",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
] + cmake_install + extraCmakeArgs
f.addStep(ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir,
env=merged_env))
# Build
f.addStep(WarningCountingShellCommand(name="build",
command=build_cmd,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
clangexe = "%(workdir)s/" + llvm_objdir + "/bin/clang"
clangxxexe = "%(workdir)s/" + llvm_objdir + "/bin/clang++"
litexe = "%(workdir)s/" + llvm_objdir + "/bin/llvm-lit"
sizeexe = "%(workdir)s/" + llvm_objdir + "/bin/llvm-size"
# Clean install dir
if install:
f.addStep(RemoveDirectory(name='clean-install-dir',
dir=llvm_instdir,
haltOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(ShellCommand(name="install",
command=install_cmd,
haltOnFailure=True,
description=["install"],
workdir=llvm_objdir,
env=merged_env))
# If installing, use the installed version of clang.
clangexe = "%(workdir)s/" + llvm_instdir + "/bin/clang"
clangxxexe = "%(workdir)s/" + llvm_instdir + "/bin/clang++"
sizeexe = "%(workdir)s/" + llvm_instdir + "/bin/llvm-size"
# Test
if checkAll:
f.addStep(LitTestCommand(name="check_all",
command=check_all_cmd,
haltOnFailure=False,
description=["check all"],
workdir=llvm_objdir,
env=merged_env))
else:
f.addStep(LitTestCommand(name="check_polly",
command=check_polly_cmd,
haltOnFailure=False,
description=["check polly"],
workdir=llvm_objdir,
env=merged_env))
if testsuite:
f.addStep(RemoveDirectory(name='test-suite_clean-src-dir',
dir=testsuite_srcdir,
haltOnFailure=False,
warnOnFailure=True,
doStepIf=cleanBuildRequestedByProperty))
f.addGetSourcecodeForProject(
project='test-suite',
src_dir=testsuite_srcdir,
alwaysUseLatest=True)
f.addStep(RemoveDirectory(name='test-suite_clean-build-dir',
dir=testsuite_builddir,
haltOnFailure=False,
warnOnFailure=True))
# -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument.
f.addStep(ShellCommand(name='test-suite_cmake-configure',
description=["Test-Suite: cmake"],
command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir,
"-DCMAKE_BUILD_TYPE=Release",
"-DTEST_SUITE_COLLECT_STATS=ON",
"-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly",
"-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly",
"-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json",
WithProperties("-DCMAKE_C_COMPILER=" + clangexe),
WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe),
WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe),
WithProperties("-DTEST_SUITE_LIT=" + litexe),
] + extraTestsuiteCmakeArgs,
haltOnFailure=True,
workdir='.',
env=merged_env))
f.addStep(WarningCountingShellCommand(name='test-suite_build',
description=["Test-Suite: build"],
# Continue building; programs that don't compile will fail with NOEXE.
command=[make, 'all', '-k0'] + jobs_cmd,
haltOnFailure=False,
flunkOnFailure=True,
workdir=testsuite_builddir,
env=merged_env))
f.addStep(LitTestCommand(name='test-suite_run',
description=['Test-Suite: run'],
command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'],
haltOnFailure=True,
workdir=testsuite_builddir,
logfiles={
'test.log' : 'test.log',
'report.json': 'report.json'
},
env=merged_env))
return f
def getLLDBCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
if vs:
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
# get full path to build directory
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=build_dir))
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
cmake_cmd = [
"cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install"
]
if python_source_dir:
cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_cmd += extra_cmake_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
command=WithProperties(" ".join(cmake_cmd)),
haltOnFailure=True,
warnOnWarnings=True,
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=2400,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getCmakeBuildFactory(depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
install_dir=None,
clean=False,
extra_configure_args=None,
env=None,
**kwargs):
# Set defaults
if not depends_on_projects:
depends_on_projects = ['llvm', 'clang']
if extra_configure_args is None:
extra_configure_args = []
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
if not obj_dir:
obj_dir = "build"
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
install_dir, obj_dir)
CmakeCommand.applyRequiredOptions(extra_configure_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir or "llvm.src",
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=cleanBuildRequested,
**kwargs # Pass through all the extra arguments.
)
# Directories to use on this stage.
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
# Do a clean checkout if requested.
f.addStep(
RemoveDirectory(
name='clean-src-dir',
dir=f.llvm_srcdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Get the source code.
f.addSVNSteps()
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % f.obj_dir,
dir=f.obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
if f.install_dir:
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % f.install_dir,
dir=f.install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
f.addStep(
CmakeCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_args,
path=src_dir,
env=env,
workdir=obj_dir,
doStepIf=FileDoesNotExist("CMakeCache.txt"),
**kwargs # Pass through all the extra arguments.
))
return f
def addCmakeSteps(f,
cleanBuildRequested,
obj_dir,
install_dir=None,
extra_configure_args=None,
env=None,
stage_name=None,
**kwargs):
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
# This is an incremental build, unless otherwise has been requested.
# Remove obj and install dirs for a clean build.
# TODO: Some Windows slaves do not handle RemoveDirectory command well.
# So, consider running "rmdir /S /Q <dir>" if the build runs on Windows.
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % obj_dir,
dir=obj_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
install_dir, obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
f.addStep(
RemoveDirectory(
name='clean-%s-dir' % install_dir,
dir=install_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested,
))
# Reconcile the cmake options for this build.
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_args, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCLANG_BUILD_EXAMPLES=', 'OFF'),
('-DLLVM_BUILD_TESTS=', 'ON'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
('-DLLVM_LIT_ARGS=', '"-v"'),
])
# Create configuration files with cmake, unless this has been already done
# for an incremental build.
if stage_name:
step_name = "cmake-configure-%s" % stage_name
else:
stage_name = ""
step_name = "cmake-configure"
src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir, obj_dir)
f.addStep(
CmakeCommand(
name=step_name,
description=["Cmake", "configure", stage_name],
options=cmake_args,
path=src_dir,
haltOnFailure=kwargs.get('haltOnFailure', True),
env=env,
workdir=obj_dir,
**kwargs # Pass through all the extra arguments.
))
def getSanitizerBuildFactoryII(
clean=False,
sanity_check=True,
sanitizers=[
'sanitizer', 'asan', 'lsan', 'msan', 'tsan', 'ubsan', 'dfsan'
],
common_cmake_options=None, # FIXME: For backward compatibility. Will be removed.
extra_configure_args=[],
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
env=None,
jobs="%(jobs)s",
timeout=1200):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb', # Make sure Clang doesn't use color escape sequences.
}
if env:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = BuildFactory()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + llvm_objdir,
dir=llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
# Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind
f.addStep(
SVN(name='svn-llvm',
mode='update',
description='svn-llvm',
descriptionDone='svn-llvm',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
description='svn-clang',
descriptionDone='svn-clang',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
description='svn-compiler-rt',
descriptionDone='svn--compiler-rt',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(
SVN(name='svn-libcxx',
mode='update',
description='svn-libcxx',
descriptionDone='svn-libcxx',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir='%s/projects/libcxx' % llvm_srcdir))
f.addStep(
SVN(name='svn-libcxxabi',
mode='update',
description='svn-libcxxabi',
descriptionDone='svn-libcxxabi',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir='%s/projects/libcxxabi' % llvm_srcdir))
f.addStep(
SVN(name='svn-libunwind',
mode='update',
description='svn-libunwind',
descriptionDone='svn-libunwind',
baseURL='http://llvm.org/svn/llvm-project/libunwind/',
defaultBranch='trunk',
workdir='%s/projects/libunwind' % llvm_srcdir))
# Run annotated command for sanitizer.
if sanity_check:
f.addStep(
AnnotatedCommand(
name="lint check",
description="lint check",
timeout=timeout,
haltOnFailure=False, #True,
warnOnWarnings=True,
command=["./check_lint.sh"],
workdir="%s/projects/compiler-rt/lib/sanitizer_common/scripts"
% llvm_srcdir,
env=merged_env))
# Always build with ninja.
cmakeCommand = ["cmake", "-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(
a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_BUILD_TYPE=Release')
if not any(
a.startswith('-DLLVM_ENABLE_WERROR=')
for a in extra_configure_args):
cmakeCommand.append('-DLLVM_ENABLE_WERROR=OFF')
if not any(
a.startswith('-DLLVM_ENABLE_ASSERTIONS=')
for a in extra_configure_args):
cmakeCommand.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(
a.startswith('-DCMAKE_C_COMPILER') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_C_COMPILER=clang')
if not any(
a.startswith('-DCMAKE_CXX_COMPILER')
for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_CXX_COMPILER=clang++')
if not any(
a.startswith('-DCMAKE_CXX_FLAGS') for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_CXX_FLAGS=\"-std=c++11 -stdlib=libc++\"')
if not any(
a.startswith('-DCMAKE_EXE_LINKER_FLAGS')
for a in extra_configure_args):
cmakeCommand.append('-DCMAKE_EXE_LINKER_FLAGS=-lcxxrt')
if not any(
a.startswith('-DLIBCXXABI_USE_LLVM_UNWINDER=')
for a in extra_configure_args):
cmakeCommand.append('-DLIBCXXABI_USE_LLVM_UNWINDER=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmakeCommand.append('-DLLVM_LIT_ARGS=\"-v\"')
cmakeCommand += extra_configure_args + ["../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=False, #True,
warnOnWarnings=True,
command=WithProperties(" ".join(cmakeCommand)),
env=merged_env,
workdir=llvm_objdir,
doStepIf=FileDoesNotExist("./%s/CMakeCache.txt" % llvm_objdir)))
# Build everything.
f.addStep(
NinjaCommand(
name='build',
haltOnFailure=False, #True,
warnOnWarnings=True,
description=['building', 'with', 'ninja'],
descriptionDone=['built', 'with', 'ninja'],
workdir=llvm_objdir,
env=merged_env))
# Run tests for each of the requested sanitizers.
if sanitizers:
for s in sanitizers:
f.addStep(
NinjaCommand(
name='test %s' % s,
targets=['check-%s' % s],
haltOnFailure=False, #True,
description=['testing', '%s' % s],
descriptionDone=['test', '%s' % s],
workdir=llvm_objdir,
env=merged_env))
return f
def getLLDBCMakeBuildFactory(
clean=False,
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
testTimeout=2400,
install=False):
############# PREPARING
# Global configurations
build_dir = 'build'
f = LLVMBuildFactory(is_legacy_mode=False,
depends_on_projects=["llvm", "clang", "lldb", "lld"],
obj_dir=build_dir)
# Determine Slave Environment and Set MSVC environment.
if vs:
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f.addGetSourcecodeSteps()
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
############# CLEANING
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
f.obj_dir)
cmake_options = [
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
]
if python_source_dir:
cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_options += extra_cmake_args
f.addStep(
CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_options,
path=rel_src_dir,
env=Property('slave_env'),
workdir=build_dir))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=testTimeout,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Kill any stale symbolizer processes for the last run. If there are any
# stale processes, the build will fail during linking. This can happen to
# any process, but it is most likely to happen to llvm-symbolizer if its
# pipe isn't closed.
taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
f.addStep(
ShellCommand(name='taskkill',
description='kill stale processes',
command=['cmd', '/c', taskkill_cmd],
haltOnFailure=False))
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f, 'llvm')
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(
ShellCommand(
name='cmake',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON"
] + extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizer runtimes.
f.addStep(
NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Build Clang and LLD next so that most compilation errors occur in a build
# step.
f.addStep(
NinjaCommand(name='build clang lld',
targets=['clang', 'lld'],
haltOnFailure=True,
description='ninja clang lld',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
for target in [
'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi'
]:
f.addStep(
NinjaCommand(name='run %s' % target,
targets=[target],
haltOnFailure=False,
description='ninja %s' % target,
workdir=build_dir,
env=Property('slave_env')))
return f
def ros_branch_build(c,
job_name,
packages,
url,
branch,
distro,
arch,
rosdistro,
machines,
othermirror,
keys,
trigger_pkgs=None):
gbp_args = [
'-uc', '-us', '--git-ignore-branch', '--git-ignore-new',
'--git-verbose', '--git-dist=' + distro, '--git-arch=' + arch
]
with open(os.path.dirname(os.path.realpath(__file__)) +
"/spec.yaml") as file:
spec_list = yaml.full_load(file)
f = BuildFactory()
# Remove the build directory.
f.addStep(
RemoveDirectory(
name=job_name + '-clean',
dir=Interpolate('%(prop:workdir)s'),
hideStepIf=success,
))
# Check out the repository master branch, since releases are tagged and not branched
f.addStep(
Git(
repourl=url,
branch=branch,
alwaysUseLatest=
True, # this avoids broken builds when schedulers send wrong tag/rev
mode='full', # clean out old versions
getDescription={'tags': True}))
# Update the cowbuilder
f.addStep(
ShellCommand(command=['cowbuilder-update.py', distro, arch] + keys,
hideStepIf=success))
# Generate the changelog for the package
f.addStep(
ShellCommand(haltOnFailure=True,
name='catkin_generate_changelog',
command=['catkin_generate_changelog', '-y'],
descriptionDone=['catkin_generate_changelog']))
# Add all files including untracked ones
f.addStep(
ShellCommand(haltOnFailure=True,
name='add_changelogs',
command=['git', 'add', '.'],
descriptionDone=['add_changelogs']))
# Commit the changelog after updating it
f.addStep(
ShellCommand(haltOnFailure=True,
name='update_changelogs',
command=['git', 'commit', '-m', '\"Updated changelogs\"'],
descriptionDone=['update_changelogs']))
# Prepare the release without pushing it
f.addStep(
ShellCommand(haltOnFailure=True,
name='catkin_prepare_release',
command=[
'catkin_prepare_release', '--bump', 'minor',
'--no-push', '-y'
],
descriptionDone=['catkin_prepare_release']))
#
f.addStep(
ShellCommand(
haltOnFailure=True,
name='git_bloom_generate_release',
command=['git-bloom-generate', '-y', 'rosrelease', rosdistro],
))
f.addStep(
ShellCommand(
haltOnFailure=True,
name='git_bloom_generate_debian',
command=[
'git-bloom-generate', '-y', 'rosdebian', '-a', '-p', 'release',
rosdistro
],
))
# Get the tag number for the lastest commit
f.addStep(
SetPropertyFromCommand(
command="git describe --tags",
property="release_version",
name='latest_tag',
))
# Need to build each package in order
for package in packages:
debian_pkg = 'ros-' + rosdistro + '-' + package.replace(
'_', '-') # debian package name (ros-groovy-foo)
branch_name = 'debian/' + debian_pkg + '_%(prop:release_version)s-0_' + distro
deb_name = debian_pkg + '_%(prop:release_version)s-0' + distro
final_name = debian_pkg + '_%(prop:release_version)s-%(prop:datestamp)s' + distro + '_' + arch + '.deb'
# Check out the proper tag. Use --force to delete changes from previous deb stamping
f.addStep(
ShellCommand(haltOnFailure=True,
name=package + '-checkout',
command=[
'git', 'checkout',
Interpolate(branch_name), '--force'
],
hideStepIf=success))
# A hack for generating the debian folder so we could build the lastest commit of the specified branch
# f.addStep(
# ShellCommand(
# haltOnFailure = True,
# name = package+'-bloom_generate',
# command= ['bloom-generate', 'rosdebian'],
# descriptionDone = ['bloom_generate', package]
# )
# )
# Download script for building the source deb
f.addStep(
FileDownload(
name=job_name + '-grab-build-source-deb-script',
mastersrc='scripts/build_source_deb.py',
slavedest=Interpolate('%(prop:workdir)s/build_source_deb.py'),
mode=0755,
hideStepIf=success))
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f,'llvm')
# Global configurations
build_dir='build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean '+build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
f.addStep(ShellCommand(name='cmake',
command=[cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE="+config,
"-DLLVM_ENABLE_ASSERTIONS=ON"]
+ extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizers runtime.
f.addStep(NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
test_targets = ['check-asan','check-asan-dynamic','check-sanitizer', 'check-cfi']
f.addStep(NinjaCommand(name='run tests',
targets=test_targets,
haltOnFailure=True,
description='ninja test',
workdir=build_dir,
env=Property('slave_env')))
return f
