def _step_AdditionalProperties(self):
self.addStep(
SetProperty(name="modifying configures",
doStepIf=lambda step:
(step.getProperty("slavename") in self.wineslaves),
property="gcc_config_args",
command=[
"echo",
Property("gcc_config_args", default=""),
"--build=i686-w64-mingw32"
]))
self.addStep(
SetProperty(name="modifying configures",
doStepIf=lambda step:
(step.getProperty("slavename") in self.wineslaves),
property="binutils_config_args",
command=[
"echo",
Property("binutils_config_args", default=""),
"--build=i686-w64-mingw32"
]))
self.addStep(
SetProperty(name="modifying configures",
doStepIf=lambda step:
(step.getProperty("slavename") in self.wineslaves),
property="mingw_config_args",
command=[
"echo",
Property("mingw_config_args", default=""),
"--build=i686-w64-mingw32"
]))
def getAnnotatedBuildFactory(script,
clean=False,
depends_on_projects=None,
env=None,
timeout=1200):
"""
Returns a new build factory that uses AnnotatedCommand, which
allows the build to be run by version-controlled scripts that do
not require a buildmaster restart to update.
"""
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm.src')
if clean:
f.addStep(SetProperty(property='clean', command='echo 1'))
# We normally use the clean property to indicate that we want a
# clean build, but AnnotatedCommand uses the clobber property
# instead. Therefore, set clobber if clean is set to a truthy
# value. This will cause AnnotatedCommand to set
# BUILDBOT_CLOBBER=1 in the environment, which is how we
# communicate to the script that we need a clean build.
f.addStep(
SetProperty(
property='clobber',
command='echo 1',
doStepIf=lambda step: step.build.getProperty('clean', False)))
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
scripts_dir = "annotated"
f.addStep(
SVN(name='update-annotate-scripts',
mode='update',
svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
'zorg/buildbot/builders/annotated',
workdir=scripts_dir,
alwaysUseLatest=True))
# Explicitly use '/' as separator, because it works on *nix and Windows.
script_path = "../%s/%s" % (scripts_dir, script)
f.addStep(
AnnotatedCommand(name="annotate",
description="annotate",
timeout=timeout,
haltOnFailure=True,
command=WithProperties(
"python %(script)s --jobs=%(jobs:-)s",
script=lambda _: script_path),
env=merged_env))
return f
def __init__(self, python, package, **kw):
SetProperty.__init__(
self,
name="check-%s-version" % package,
description="checking %s version" % package,
command=[python, '-c', self.src % dict(package=package)],
extract_fn=self._extractVersion, **kw)
self.package = package
def __init__(self, python, package, **kw):
SetProperty.__init__(
self,
name="check-%s-version" % package,
description="checking %s version" % package,
command=[python, '-c', self.src % dict(package=package)],
extract_fn=self._extractVersion, **kw)
self.package = package
def __init__(self, **kwargs):
factory.BuildFactory.__init__(self, **kwargs)
self.addStep(
SetProperty(
property="optionTargetOs",
doStepIf=lambda step: step.build.getProperty("target-os"),
command=["echo", "--os"]))
self.addStep(
SetProperty(property="optionPath",
doStepIf=lambda step: step.build.getProperty("path"),
command=["echo", "--path"]))
self.addStep(
SetProperty(
property="optionDatestamp",
doStepIf=lambda step: step.build.getProperty("datestamp"),
command=["echo", "--datestamp"]))
# this assumes the following properties have been set:
# masterdir - the path to the master configuration (has tarballs + scripts + passwords)
# filename - the name of the file to upload (relative to masterdir)
# destname - how to name the file on the server
self.addStep(
ShellCommand(name="sfupload",
description=["uploading"],
descriptionDone=["upload"],
doStepIf=lambda step:
(step.build.getProperty("is_nightly")),
workdir=Property("masterdir"),
command=[
"scripts/upload.py",
Property("filename"),
Property("destname"),
Property("optionTargetOs", default=""),
Property("target-os", default=""),
Property("optionPath", default=""),
Property("path", default=""),
Property("optionDatestamp", default=""),
Property("datestamp", default="")
],
haltOnFailure=True))
self.addStep(
ShellCommand(name="uploadComplete",
description=["removing", "temporary", "file"],
descriptionDone=["temporary", "file", "removed"],
doStepIf=lambda step:
(step.build.getProperty("is_nightly")),
workdir=Property("masterdir"),
command=["rm", "-fv",
Property("filename")]))
def __init__(self,
condprop=None,
condinvert=False,
condvalue=None,
conddefault=None,
**kwargs):
SetProperty.__init__(self, **kwargs)
self.addFactoryArguments(condprop=condprop,
condinvert=condinvert,
condvalue=condvalue,
conddefault=conddefault)
self.condProp = condprop
self.condInvert = condinvert
self.condValue = condvalue
self.condDefault = conddefault
def setProperty(f, new_property, new_value):
f.addStep(SetProperty(name = 'set.' + new_property,
command=['echo', new_value],
property=new_property,
description=['set property', new_property],
workdir='.'))
return f
def getUserDir(f):
f.addStep(SetProperty(command=['sh', '-c', 'cd ~;pwd'],
haltOnFailure=True,
property='user_dir',
description=['set property', 'user_dir'],
workdir='.'))
return f
def getBuildDir(f):
f.addStep(SetProperty(name='get.build.dir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.'))
return f
def update_repo(self, repo, branch='default', clobber=False):
workdir = repo.split("/")[-1]
repourl = 'http://%s/%s' % (self.hgHost, repo)
if clobber:
self.addStep(ShellCommand(
name='%s_clobber' % workdir,
command=['rm', '-rf', workdir],
workdir='.',
))
self.addStep(ShellCommand(
name='%s_update' % workdir,
command=['bash', '-c',
'if test -d %(workdir)s; then hg -R %(workdir)s pull; \
else hg clone --noupdate %(repourl)s %(workdir)s; fi && \
hg -R %(workdir)s up -C --rev %(branch)s' % locals()],
timeout=3*60,
descriptionDone="%s source" % workdir,
workdir='.',
))
self.addStep(SetProperty(
name='set_%s_revision' % workdir,
command=['hg', 'identify', '-i'],
property='%s_revision' % workdir,
workdir=workdir,
))
def _step_AdditionalProperties(self):
self.addStep(
SetProperty(
property="host-extra",
command=[
"bash", "-c",
"""cygcheck -c -d cygwin | perl -ne 'm/^cygwin\s+(\S+)/ and print "-$1"'"""
]))
def getTestConfig(f):
def getRemoteCfg(rc, stdout, stderr):
return json.loads(stdout)
f.addStep(SetProperty(name="get test config",
command="cat test_cfg.json",
extract_fn=getRemoteCfg,
description="get remote target",
workdir="scripts"))
return f
def getSanitizerBuildFactory(clean=False,
depends_on_projects=None,
env=None,
timeout=1200,
is_legacy_mode=False):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM": "dumb", # Make sure Clang doesn't use color escape sequences.
}
# Clobber bot if we need a clean build.
if clean:
merged_env["BUILDBOT_CLOBBER"] = "1"
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
if depends_on_projects is None:
depends_on_projects = [
"llvm", "clang", "compiler-rt", "libcxx", "libcxxabi", "libunwind",
"lld"
]
# Explicitly use '/' as separator, because it works on *nix and Windows.
sanitizer_script_dir = "sanitizer_buildbot/sanitizers"
sanitizer_script = "../%s/zorg/buildbot/builders/sanitizers/%s" % (
sanitizer_script_dir, "buildbot_selector.py")
f = LLVMBuildFactory(is_legacy_mode=is_legacy_mode,
clean=clean,
depends_on_projects=depends_on_projects,
llvm_srcdir=sanitizer_script_dir)
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Get sanitizer buildbot scripts.
f.addGetSourcecodeForProject(name='update-annotate-scripts',
project='zorg',
src_dir=sanitizer_script_dir,
alwaysUseLatest=True)
# Run annotated command for sanitizer.
f.addStep(
AnnotatedCommand(name="annotate",
description="annotate",
timeout=timeout,
haltOnFailure=True,
command="python " + sanitizer_script,
env=merged_env))
return f
def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,
gcs_url_property=None, use_pixz_compression=False,
xz_compression_factor=6):
"""
Add steps to upload to the Google Cloud Storage bucket.
f - The BuildFactory to modify.
package_name - The name of this package for the descriptions (e.g.
'stage 1')
install_prefix - The directory the build has been installed to.
gcs_directory - The subdirectory of the bucket root to upload to. This
should match the builder name.
env - The environment to use. Set BOTO_CONFIG to use a configuration file
in a non-standard location, and BUCKET to use a different GCS bucket.
gcs_url_property - Property to assign the GCS url to.
"""
gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/'
'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz')
time_fmt = '%Y-%m-%d_%H-%M-%S'
output_file_name = '../install.tar.xz'
gcs_url = \
WithProperties(
gcs_url_fmt,
gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'),
gcs_directory=lambda _: gcs_directory,
now=lambda _: datetime.utcnow().strftime(time_fmt))
if gcs_url_property:
f.addStep(SetProperty(
name="record GCS url for " + package_name,
command=['echo', gcs_url],
property=gcs_url_property))
if use_pixz_compression:
# tweak the xz compression level to generate packages faster
tar_command = ['tar', '-Ipixz', '-cvf', output_file_name, '.']
else:
xz_command = 'xz -{0}'.format(xz_compression_factor)
tar_command = ['tar', '-I', xz_command, '-cvf', output_file_name, '.']
f.addStep(ShellCommand(name='package ' + package_name,
command=tar_command,
description='packaging ' + package_name + '...',
workdir=install_prefix,
env=env))
f.addStep(ShellCommand(
name='upload ' + package_name + ' to storage bucket',
command=['gsutil', 'cp', '../install.tar.xz', gcs_url],
description=('uploading ' + package_name +
'to storage bucket ...'),
workdir=install_prefix,
env=env))
def start(self):
if self.condProp is not None:
value = self.condDefault
matched = False
if self.build.getProperties().has_key(self.condProp):
value = self.build.getProperty(self.condProp)
elif self.condDefault is None:
# no value and no default, assume we want to execute
matched = True
if not matched:
if self.condValue is None:
matched = bool(value)
else:
matched = (value == self.condValue)
if self.condInvert:
matched = not matched
if not matched:
return SKIPPED
SetProperty.start(self)
def __init__(self, repo, *testnames):
BuildProcedure.__init__(self, 'elisp')
self.addSteps(
Git(repourl='git://github.com/%s.git' % repo),
SetPropertiesFromEnv(variables=['EMACS']),
SetProperty(
command=[
Emacs(), '--batch', '--eval',
'(princ (make-temp-file "home" t ".bbot"))'
],
extract_fn=lambda rc, stdout, stderr: dict(FakeHome=stdout)))
for t in testnames or ['test/test']:
self.addStep(EmacsTest(load=t + '.el'))
def testGoodStep(self):
f = BuildFactory()
f.addStep(SetProperty(command=["echo", "value"], property="propname"))
ss = SourceStamp()
req = FakeBuildRequest("Testing", ss, None)
b = f.newBuild([req])
b.build_status = FakeBuildStatus()
b.slavebuilder = FakeSlaveBuilder()
# This shouldn't raise an exception
b.setupBuild(None)
def __init__(self, hgHost, **kwargs):
self.parent_class = BuildFactory
self.parent_class.__init__(self, **kwargs)
self.hgHost = hgHost
self.addStep(SetProperty(
name='set_topdir',
command=['pwd'],
property='topdir',
workdir='.',
))
self.addStep(ShellCommand(
name='rm_pyc',
command=['find', '.', '-name', '*.pyc', '-exec', 'rm', '-fv', '{}',
';'],
workdir=".",
))
def testCommand(self):
f = BuildFactory()
f.addStep(SetProperty(command=["echo", "value"], property="propname"))
f.addStep(
ShellCommand(
command=["echo", WithProperties("%(propname)s")]))
ss = SourceStamp()
req = FakeBuildRequest("Testing", {ss.repository: ss}, None)
b = f.newBuild([req])
b.build_status = FakeBuildStatus()
b.slavebuilder = FakeSlaveBuilder()
# This shouldn't raise an exception
b.setupBuild(None)
def xmippBundleFactory(groupId):
xmippTestSteps = util.BuildFactory()
xmippTestSteps.workdir = settings.SDEVEL_XMIPP_HOME
env = {
"SCIPION_HOME": util.Property("SCIPION_HOME"),
"SCIPION_LOCAL_CONFIG": util.Property("SCIPION_LOCAL_CONFIG"),
"LD_LIBRARY_PATH": LD_LIBRARY_PATH,
"EM_ROOT": settings.EM_ROOT
}
xmippTestSteps.addStep(
SetProperty(command=["bash", "-c", "source build/xmipp.bashrc; env"],
extract_fn=glob2list,
env=env))
xmippTestSteps.addStep(
GenerateStagesCommand(
command=["./xmipp", "test", "--show"],
name="Generate test stages for Xmipp programs",
description="Generating test stages for Xmipp programs",
descriptionDone="Generate test stages for Xmipp programs",
haltOnFailure=False,
pattern='./xmipp test (.*)',
rootName=settings.XMIPP_CMD,
timeout=settings.timeOutExecute,
blacklist=settings.SCIPION_TESTS_BLACKLIST,
env=util.Property('env')))
xmippTestSteps.addStep(
GenerateStagesCommand(
command=["./xmipp", "test", "--show"],
name="Generate test stages for Xmipp functions",
description="Generating test stages for Xmipp functions",
descriptionDone="Generate test stages for Xmipp functions",
haltOnFailure=False,
timeout=settings.timeOutExecute,
pattern='xmipp_test_(.*)',
rootName=settings.XMIPP_CMD,
blacklist=settings.SCIPION_TESTS_BLACKLIST,
env=util.Property('env')))
return xmippTestSteps
def getLLDBCMakeBuildFactory(
clean=False,
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
testTimeout=2400,
install=False):
############# PREPARING
# Global configurations
build_dir = 'build'
f = LLVMBuildFactory(is_legacy_mode=False,
depends_on_projects=["llvm", "clang", "lldb", "lld"],
obj_dir=build_dir)
# Determine Slave Environment and Set MSVC environment.
if vs:
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f.addGetSourcecodeSteps()
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
############# CLEANING
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
f.obj_dir)
cmake_options = [
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
]
if python_source_dir:
cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_options += extra_cmake_args
f.addStep(
CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_options,
path=rel_src_dir,
env=Property('slave_env'),
workdir=build_dir))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=testTimeout,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getLLDBUbuntuCMakeBuildFactory(build_compiler,
build_type,
local_test_archs=None,
local_test_compilers=None,
remote_configs=None,
jobs='%(jobs)s',
env=None):
"""Generate factory steps for ubuntu cmake builder
Arguments:
build_compiler -- string of compile name, example 'clang',
the compiler will be used to build binaries for host platform
build_type -- 'Debug' or 'Release',
used to define build type for host platform as well as remote platform if any
local_test_archs -- list of architectures, example ['i386','x86_64'],
defines architectures to run local tests against, if None, local tests won't be executed
local_test_compiler -- list of compilers, example ['clang','gcc4.8.2'],
definds compilers to run local tests with, if None, local tests won't be executed
remote_configs -- list of RemoteConfig objects, example [RemoteConfig(...)], if None, remote tests won't be executed
jobs -- number of threads for compilation step, example 40
default value is jobs number defined during slave creation
env -- environment variables passed to shell commands
"""
if env is None:
env = {}
llvm_srcdir = "llvm"
llvm_builddir = "build"
bindir = '%(builddir)s/' + llvm_builddir + '/bin'
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Determine the binary directory of *-tblgen.
f.addStep(
SetProperty(name="get tblgen dir",
command=["echo", WithProperties(bindir)],
property="tblgen_bindir",
description="set tblgen dir",
workdir="."))
# Get source code
f = getLLDBSource(f, llvm_srcdir)
# Clean Build Folder
f.addStep(
ShellCommand(name="clean",
command="rm -rf *",
description="clear build folder",
env=env,
workdir='%s' % llvm_builddir))
f = getLLDBCmakeAndCompileSteps(f, build_compiler, build_type, [], bindir,
'linux', 'x86_64', env)
# TODO: it will be good to check that architectures listed in test_archs are compatible with host architecture
# For now, the caller of this function should make sure that each target architecture is supported by builder machine
# Add local test steps
f = getLLDBTestSteps(f, bindir, local_test_archs, local_test_compilers)
# Remote test steps
if remote_configs is not None:
for config in remote_configs:
f = getLLDBRemoteTestSteps(f, bindir, build_type, config, env)
# archive test traces
f = archiveLLDBTestTraces(f, "build/lldb-test-traces-*")
return f
def getLLDBRemoteTestSteps(f, bindir, build_type, remote_config, env):
if None in (remote_config.test_archs, remote_config.test_compilers):
return f
# only supports linux and android as remote target at this time
if remote_config.platform not in ('linux', 'android'):
return f
llvm_srcdir = "llvm"
llvm_builddir = "build"
stepDesc = remote_config.platform + "-" + remote_config.host_arch
# get hostname
slave_hostname = None
f.addStep(
SetProperty(name="get hostname",
command=["hostname"],
property="slave_hostname",
description="set slave hostname",
workdir="."))
# get configuration of remote target
# config file should be placed under builddir on builder machine
# file name: remote_cfg.json
# content: json format with keys [remote_platform]-[remote_arch]
# the value for each key defines "remote_host", "remote_port", "remote_dir", "toolchain", "deviceId"
# example: {"android-i386": {"remote_host":"localhost",
# "remote_port":"5430",
# "remote_dir":"/data/local/tmp/lldb",
# "toolchain_build":"/home/lldb_build/Toolchains/i386-android-toolchain-21",
# "toolchain_test":"/home/lldb_build/Toolchains/i386-android-toolchain-16",
# "deviceid":"XXXXXXX"},
def getRemoteCfg(rc, stdout, stderr):
return json.loads(stdout)[stepDesc]
f.addStep(
SetProperty(name="get remote target " + stepDesc,
command="cat remote_cfg.json",
extract_fn=getRemoteCfg,
description="get remote target",
workdir="."))
# rsync
if remote_config.platform is 'linux':
shellcmd = ['ssh', WithProperties('%(remote_host)s')]
hostname = '%(slave_hostname)s'
launchcmd = shellcmd + ['screen', '-d', '-m']
terminatecmd = shellcmd + ['pkill', 'lldb-server']
cleandircmd = WithProperties(
'ssh %(remote_host)s rm -r %(remote_dir)s/*')
f.addStep(
ShellCommand(name="rsync lldb-server",
command=[
'rsync', '-havL', 'bin/lldb-server',
WithProperties('%(remote_host)s:%(remote_dir)s')
],
description="rsync lldb-server " + stepDesc,
haltOnFailure=True,
env=env,
workdir='%s' % llvm_builddir))
f.addStep(
ShellCommand(name="rsync python2.7",
command=[
'rsync', '-havL', 'lib/python2.7',
WithProperties('%(remote_host)s:%(remote_dir)s')
],
description="rsync python2.7 " + stepDesc,
haltOnFailure=True,
env=env,
workdir='%s' % llvm_builddir))
elif remote_config.platform is 'android':
shellcmd = ['adb', '-s', WithProperties('%(deviceid)s'), 'shell']
hostname = '127.0.0.1'
launchcmd = ['screen', '-d', '-m'] + shellcmd + [
WithProperties("TMPDIR=%(remote_dir)s/tmp")
]
terminatecmd = 'ps | grep lldb-server | awk \'{print $2}\' | xargs'
terminatecmd = WithProperties('adb -s %(deviceid)s shell ' +
terminatecmd +
' adb -s %(deviceid)s shell kill')
cleandircmd = WithProperties(
'adb -s %(deviceid)s shell rm -rf %(remote_dir)s/*')
# compile lldb-server for target platform
f = getLLDBCmakeAndCompileSteps(f, 'gcc', build_type, ['lldb-server'],
bindir, remote_config.platform,
remote_config.host_arch, env)
f.addStep(
ShellCommand(name="adb push lldb-server " + stepDesc,
command=[
'adb', '-s',
WithProperties('%(deviceid)s'), 'push',
remote_config.platform + '-' +
remote_config.host_arch + '/bin/lldb-server',
WithProperties('%(remote_dir)s/')
],
description="lldb-server",
env=env,
haltOnFailure=True,
workdir='%s' % llvm_builddir))
f.addStep(
ShellCommand(name="Build fingerprint " + stepDesc,
command=[
'adb', '-s',
WithProperties('%(deviceid)s'), 'shell',
'getprop', 'ro.build.fingerprint'
],
description="get build fingerprint",
env=env,
haltOnFailure=False,
workdir='%s' % llvm_builddir))
# launch lldb-server
f.addStep(
ShellCommand(
name="launch lldb-server " + stepDesc,
command=launchcmd + [
WithProperties('%(remote_dir)s/lldb-server'), 'platform',
'--listen',
WithProperties(hostname + ':%(remote_port)s'), '--server'
],
description="launch lldb-server on remote host",
env=env,
haltOnFailure=True,
workdir='%s' % llvm_builddir))
# test steps
f = getLLDBTestSteps(f, bindir, remote_config.test_archs,
remote_config.test_compilers, remote_config.platform,
'%(remote_host)s', '%(remote_port)s',
'%(remote_dir)s', env)
# terminate lldb-server on remote host
f.addStep(
ShellCommand(name="terminate lldb-server " + stepDesc,
command=terminatecmd,
description="terminate lldb-server",
env=env,
workdir='%s' % llvm_builddir))
# clean remote test directory
f.addStep(
ShellCommand(name="clean remote dir " + stepDesc,
command=cleandircmd,
description="clean remote dir",
env=env))
return f
def getLLDBBuildFactory(triple,
useTwoStage=False,
make='make',
jobs='%(jobs)s',
extra_configure_args=[],
env={},
*args,
**kwargs):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Find out what version of llvm and clang are needed to build this version
# of lldb. Right now we will assume they use the same version.
# XXX - could this be done directly on the master instead of the slave?
f.addStep(
SetProperty(
command=
'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2',
property='llvmrev'))
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout', '--force']
svn_co += ['--revision', WithProperties('%(llvmrev)s')]
# build llvm svn checkout command
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
llvm_srcdir]
# build clang svn checkout command
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
'%s/tools/clang' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
always_purge=True,
workdir='%s/tools/lldb' % llvm_srcdir))
# Run configure
config_args = [
WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
"--disable-bindings",
"--without-llvmgcc",
"--without-llvmgxx",
]
if triple:
config_args += ['--build=%s' % triple]
config_args += extra_configure_args
f.addStep(
Configure(name='configure',
command=config_args,
env=env,
workdir=llvm_objdir))
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
env=env,
haltOnFailure=True,
workdir=llvm_objdir))
# Test.
f.addStep(
LitTestCommand(name="test lldb",
command=['nice', '-n', '10', make],
description="test lldb",
env=env,
workdir='%s/tools/lldb/test' % llvm_objdir))
return f
def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
checks=None,
install_dir=None,
clean=False,
extra_configure_args=None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env=None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs = r"""%VS140COMNTOOLS%"""
f = getLLVMBuildFactoryAndSVNSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def getLLDBCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
if vs:
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
# get full path to build directory
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=build_dir))
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
cmake_cmd = [
"cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install"
]
if python_source_dir:
cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_cmd += extra_cmake_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
command=WithProperties(" ".join(cmake_cmd)),
haltOnFailure=True,
warnOnWarnings=True,
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=2400,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getText(self, cmd, results):
if 'version' in self.property_changes:
return [ "%s version %s" % (self.package, self.property_changes['version']) ]
else:
SetProperty.getText(self, cmd, results)
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=None,
testsuite_flags=None,
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env=None,
extra_cmake_args=None,
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
checkout_flang=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=None):
############# PREPARING
if nt_flags is None:
nt_flags = []
if testsuite_flags is None:
testsuite_flags = []
if env is None:
env = {}
if extra_cmake_args is None:
extra_cmake_args = []
if trigger_after_stage1 is None:
trigger_after_stage1 = []
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
if checkout_flang:
depends_on_projects.append('flang')
depends_on_projects.append('mlir')
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
# Checkout the latest code for LNT
# and the test-suite separately. Le's do this first,
# so we wouldn't poison got_revision property.
if runTestSuite or checkout_test_suite:
f.addGetSourcecodeForProject(project='lnt',
src_dir='test/lnt',
alwaysUseLatest=True)
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test/test-suite',
alwaysUseLatest=True)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs and vs != "manual":
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
CmakeCommand.applyRequiredOptions(extra_cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", rel_src_dir,
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
LitTestCommand(name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir,
stage2_build)
cmake_cmd2 = [
cmake, "-G", "Ninja", rel_src_dir,
WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install +
"/bin/" + cc),
WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" +
stage1_install + "/bin/" + cxx),
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
LitTestCommand(name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
LitTestCommand(name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Kill any stale symbolizer processes for the last run. If there are any
# stale processes, the build will fail during linking. This can happen to
# any process, but it is most likely to happen to llvm-symbolizer if its
# pipe isn't closed.
taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
f.addStep(
ShellCommand(name='taskkill',
description='kill stale processes',
command=['cmd', '/c', taskkill_cmd],
haltOnFailure=False))
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f, 'llvm')
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(
ShellCommand(
name='cmake',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON"
] + extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizer runtimes.
f.addStep(
NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Build Clang and LLD next so that most compilation errors occur in a build
# step.
f.addStep(
NinjaCommand(name='build clang lld',
targets=['clang', 'lld'],
haltOnFailure=True,
description='ninja clang lld',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
for target in [
'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi'
]:
f.addStep(
NinjaCommand(name='run %s' % target,
targets=[target],
haltOnFailure=False,
description='ninja %s' % target,
workdir=build_dir,
env=Property('slave_env')))
return f
def getLLDBxcodebuildFactory(use_cc=None,
build_type='Debug',
remote_configs=None,
env=None):
if env is None:
env = {}
f = buildbot.process.factory.BuildFactory()
f.addStep(
SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.'))
lldb_srcdir = 'lldb'
OBJROOT = '%(builddir)s/' + lldb_srcdir + '/build'
f.addStep(
SetProperty(name='get_bindir',
command=[
'echo',
WithProperties('%(builddir)s/' + lldb_srcdir +
'/build/' + build_type)
],
property='lldb_bindir',
description='set bin dir',
workdir='.'))
# cleaning out the build directory is vital for codesigning.
f.addStep(
ShellCommand(name='clean.lldb-buid',
command=['rm', '-rf',
WithProperties(OBJROOT)],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(name='clean.llvm-buid',
command=['rm', '-rf',
'%s/llvm-build' % lldb_srcdir],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(name='clean.test trace',
command='rm -rf build/*',
haltOnFailure=False,
flunkOnFailure=False,
workdir='.'))
# Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency
f.addStep(
ShellCommand(name='remove symbolic link lldb',
command=['rm', lldb_srcdir + '/llvm/tools/lldb'],
haltOnFailure=False,
flunkOnFailure=False,
workdir=WithProperties('%(builddir)s')))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
workdir=lldb_srcdir))
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir='%s/llvm' % lldb_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/llvm/tools/clang' % lldb_srcdir))
# setup keychain for codesign
# In order for the codesigning to work inside of buildbot, security must be
# called to unlock the keychain, which requires a password.
# I've set up a special keychain for this purpose, so as to not compromise
# the login password of the buildslave.
# This means I have to set the special keychain as the default and unlock it
# prior to building the sources.
f.addStep(
ShellCommand(name='check.keychain',
command=['security', 'default-keychain'],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(
name='find.certificate',
command=['security', 'find-certificate', '-c', 'lldb_codesign'],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
# Building the sources
#
buildcmd = ' '.join([
'xcrun', 'xcodebuild', '-target', 'desktop', '-configuration',
build_type, 'SYMROOT=' + OBJROOT, 'OBJROOT=' + OBJROOT
])
f.addStep(
ShellCommand(name='lldb-build',
command=WithProperties(buildcmd + " || " + buildcmd),
haltOnFailure=True,
workdir=lldb_srcdir))
# Testing
#
if not use_cc:
use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/'
use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang'
f.addStep(
SetProperty(name='set.cc',
command=['xcrun', '-find', 'clang'],
property='use_cc',
description='set cc',
workdir=lldb_srcdir))
else:
f.addStep(
SetProperty(name='set.cc',
command=['echo', use_cc],
property='use_cc',
description='set cc',
workdir=lldb_srcdir))
DOTEST_OPTS = ' '.join([
'--executable', '%(lldb_bindir)s/lldb', '--filecheck',
'%(lldb_bindir)s/FileCheck', '--framework',
'%(lldb_bindir)s/LLDB.framework', '-A', 'x86_64', '-C', 'clang', '-s',
'../../build/lldb-test-traces'
])
f.addStep(
LitTestCommand(
name='lldb-test',
command=['./dosep.py', '--options',
WithProperties(DOTEST_OPTS)],
haltOnFailure=False,
workdir='%s/test' % lldb_srcdir,
env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')}))
# Remote test steps
if remote_configs is not None:
# Source structure to use cmake command
f.addStep(
SetProperty(
name='get tblgen bindir',
command=[
'echo',
WithProperties(
'%(builddir)s/' + lldb_srcdir +
'/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin'
)
],
property='tblgen_bindir',
description='set tblgen binaries dir',
workdir='.'))
f = getSymbLinkSteps(f, lldb_srcdir)
for config in remote_configs:
f = getLLDBRemoteTestSteps(
f,
'%(lldb_bindir)s',
build_type,
config,
env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')})
# Compress and upload test log
f = archiveLLDBTestTraces(f, "build/lldb-test-traces*")
# Results go in a directory coded named according to the date and time of the test run, e.g.:
#
# 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log
#
# Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error. return f
return f
def getLLDBScriptCommandsFactory(
downloadBinary=True,
buildAndroid=False,
runTest=True,
scriptExt='.sh',
extra_cmake_args=None,
depends_on_projects=None,
):
if scriptExt is '.bat':
pathSep = '.\\'
else:
pathSep = './'
if extra_cmake_args is None:
extra_cmake_args = []
if depends_on_projects is None:
f = buildbot.process.factory.BuildFactory()
else:
f = LLVMBuildFactory(depends_on_projects=depends_on_projects)
# Update scripts
getShellCommandStep(f,
name='update scripts',
command=['updateScripts' + scriptExt])
# Acquire lock
if downloadBinary:
getShellCommandStep(
f,
name='acquire lock',
command=[pathSep + 'acquireLock' + scriptExt, 'totBuild'],
description='get')
# Checkout source code
getShellCommandStep(f,
name='checkout source code',
command=[
pathSep + 'checkoutSource' + scriptExt,
WithProperties('%(revision)s')
])
# Set source revision
f.addStep(
SetProperty(name="set revision",
command=[pathSep + 'getRevision' + scriptExt],
property="got_revision",
workdir="scripts"))
# Configure
getShellCommandStep(f,
name='cmake local',
command=[pathSep + 'cmake' + scriptExt] +
extra_cmake_args)
# Build
getShellCommandStep(f,
name='ninja build local',
command=[pathSep + 'buildLocal' + scriptExt])
if buildAndroid:
getShellCommandStep(f,
name='build android',
command=[pathSep + 'buildAndroid' + scriptExt])
# Get lldb-server binaries
if downloadBinary:
getShellCommandStep(f,
name='get lldb-server binaries',
command=[
pathSep + 'downloadBinaries' + scriptExt,
WithProperties('%(got_revision)s')
])
# Test
if runTest:
f.addStep(
LitTestCommand(name="run unit tests",
command=[pathSep + 'testUnit' + scriptExt],
description=["testing"],
descriptionDone=["unit test"],
workdir='scripts'))
getTestSteps(f, scriptExt, pathSep)
# upload test traces
getShellCommandStep(f,
name='upload test traces',
command=[
pathSep + 'uploadTestTrace' + scriptExt,
WithProperties('%(buildnumber)s'),
WithProperties('%(buildername)s')
],
flunkOnFailure=False)
# Upload lldb-server binaries and trigger android builders
if buildAndroid:
getShellCommandStep(f,
name='upload lldb-server binaries',
command=[pathSep + 'uploadBinaries' + scriptExt])
f.addStep(
trigger.Trigger(schedulerNames=['lldb_android_scheduler'],
updateSourceStamp=False,
waitForFinish=False))
# Release lock
if downloadBinary:
getShellCommandStep(
f,
name='release lock',
command=[pathSep + 'releaseLock' + scriptExt, 'totBuild'],
description='release',
alwaysRun=True)
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=[],
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env={},
extra_cmake_args=[],
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Triggers
trigger_after_stage1=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
addSVNUpdateSteps(f,
checkout_clang_tools_extra=checkout_clang_tools_extra,
checkout_compiler_rt=checkout_compiler_rt,
checkout_test_suite=runTestSuite)
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
check_build_cmd = [
"sh", "-c", "test -e build.ninja && echo OK || echo Missing"
]
if vs:
check_build_cmd = [
"cmd", "/c",
"if exist build.ninja (echo OK) " + " else (echo Missing & exit 1)"
]
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
if clean:
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
env=env))
else:
f.addStep(
SetProperty(name="check ninja files 1",
workdir=stage1_build,
command=check_build_cmd,
flunkOnFailure=False,
property="exists_ninja_1"))
############# STAGE 1
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=lambda step: step.build.getProperty("exists_ninja_1"
) != "OK",
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url')
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.',
env=env))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
cmake_cmd2 = [
'env',
WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
cxx), cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
doStepIf=lambda step: step.build.getProperty(
"exists_ninja_2") != "OK",
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags to get the same effect)
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
if testerName:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
