def getCmakeWithNinjaWithMSVCBuildFactory(
depends_on_projects=None,
llvm_srcdir=None,
obj_dir=None,
checks=None,
install_dir=None,
clean=False,
extra_configure_args=None,
# VS tools environment variable if using MSVC. For example,
# %VS140COMNTOOLS% selects the 2015 toolchain.
vs=None,
target_arch=None,
env=None,
**kwargs):
assert not env, "Can't have custom builder env vars with MSVC build"
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if checks is None:
checks = ['check-all']
# Set up VS environment, if appropriate.
if not vs:
# We build by Visual Studio 2015, unless otherwise is requested.
vs = r"""%VS140COMNTOOLS%"""
f = getLLVMBuildFactoryAndSourcecodeSteps(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
**kwargs) # Pass through all the extra arguments.
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
env = Property('slave_env')
# Some options are required for this build no matter what.
CmakeCommand.applyRequiredOptions(cmake_args, [
('-G', 'Ninja'),
])
cleanBuildRequested = lambda step: step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj")) or clean
addCmakeSteps(f,
cleanBuildRequested=cleanBuildRequested,
obj_dir=f.obj_dir,
install_dir=f.install_dir,
extra_configure_args=cmake_args,
env=env,
**kwargs)
addNinjaSteps(f,
obj_dir=obj_dir,
checks=checks,
install_dir=f.install_dir,
env=env,
**kwargs)
return f
def __init__(self, **kwargs):
factory.BuildFactory.__init__(self, **kwargs)
# set properties about this builder
self.addStep(
SetProperty(property="masterdir", command=["echo",
os.getcwd()]))
self.addStep(
SetProperty(property="basedir",
command=["bash", "-c", "builtin pwd"]))
self.addStep(
SetProperty(property="gmp_version",
command=["echo",
gConfig.get("libraries", "gmp")],
doStepIf=lambda step:
(not step.build.hasProperty("gmp_version"))))
self.addStep(
SetProperty(property="mpfr_version",
command=["echo",
gConfig.get("libraries", "mpfr")],
doStepIf=lambda step:
(not step.build.hasProperty("mpfr_version"))))
self.addStep(
SetProperty(property="mpc_version",
command=["echo",
gConfig.get("libraries", "mpc")],
doStepIf=lambda step:
(not step.build.hasProperty("mpc_version"))))
self.addStep(
SetProperty(property="binutils_branch",
command=["echo", "trunk"],
doStepIf=lambda step:
(not step.build.hasProperty("binutils_branch"))))
self.addStep(
SetProperty(property="gcc_branch",
command=["echo", "trunk"],
doStepIf=lambda step:
(not step.build.hasProperty("gcc_branch"))))
self.addStep(
SetProperty(property="mingw_branch",
command=["echo", "trunk"],
doStepIf=lambda step:
(not step.build.hasProperty("mingw_branch"))))
self.addStep(
SetProperty(property="filename",
command=[
"echo",
Property("src_archive",
default="mingw-w64-src.tar.bz2")
]))
self.addStep(
SetProperty(
property="srcname_format",
command=["echo", "mingw-w64-src%(datestamp:-)s.tar.bz2"],
doStepIf=lambda step:
(not step.build.hasProperty("srcname_format"))))
#self.addStep(M64NightlyRev)
if self.clobber:
self.addStep(
ShellCommand(name="clobber",
command=[
"rm", "-rfv", "build", "src",
Property("filename")
],
haltOnFailure=False,
description=["clobber all"],
descriptionDone=["clobbered"]))
self.addStep(
ShellCommand(
name="makefile-checkout",
description=["makefile", "checkout"],
descriptionDone=["checked out", "makefile"],
command=[
"curl", "-o", "mingw-makefile",
"https://svn.code.sf.net/p/mingw-w64/code/experimental/buildsystem/makebuildroot.mk"
],
haltOnFailure=True))
# self.addStep(SVN(mode="export",
# svnurl="https://mingw-w64.svn.sourceforge.net/svnroot/mingw-w64/experimental/buildsystem",
# extra_args=["--trust-server-cert"]))
self.addStep(
FileUpload(masterdest="mingw-makefile",
slavesrc="mingw-makefile",
maxsize=102400,
mode=0600))
self.addStep(
ShellCommand(
name="patch-pull",
command=["make", "-f", "mingw-makefile", "patch-pull"],
description=["patches", "pull"],
descriptionDone=["pulled", "patches"]))
# download binutils
self.addStep(
Compile(name="binutils-pull",
description=["binutils", "pull"],
descriptionDone=["pulled", "binutils"],
command=["make", "-f", "mingw-makefile", "binutils-pull"],
env={
"BINUTILS_REVISION":
Property("binutils_revision", default="head"),
"BINUTILS_BRANCH":
Property("binutils_branch")
}))
self.addStep(
ShellCommand(
name="binutils-patch",
description=["patch", "binutils"],
descriptionDone=["binutils", "patched"],
doStepIf=lambda step: (step.getProperty("scheduler") != "try"),
workdir="build/src/binutils",
command=[
"bash", "-c",
"""if [ -n "$( ls ../patches/binutils/*.patch )" ] ; then
for i in ../patches/binutils/*.patch ; do
patch -p1 -f -i "$i" ;
done ;
fi""".replace("\n", " ")
]))
# download gcc
self.addStep(
Compile(name="gcc-pull",
description=["gcc", "pull"],
descriptionDone=["pulled", "gcc"],
command=["make", "-f", "mingw-makefile", "gcc-pull"],
env={
"GCC_REVISION": Property("gcc_revision",
default="head"),
"GCC_BRANCH": Property("gcc_branch")
}))
self.addStep(
ShellCommand(
name="gcc-patch",
description=["patch", "gcc"],
descriptionDone=["gcc", "patched"],
doStepIf=lambda step: (step.getProperty("scheduler") != "try"),
workdir="build/src/gcc/src",
command=[
"bash", "-c",
"""if [ -n "$( ls ../../patches/gcc/*.patch )" ] ; then
for i in ../../patches/gcc/*.patch ; do
patch -p1 -f -i "$i" ;
done ;
fi""".replace("\n", " ")
]))
# download gmp
self.addStep(
Compile(name="gmp-download",
description=["gmp", "download"],
descriptionDone=["downloaded", "gmp"],
command=["make", "-f", "mingw-makefile", "gmp-download"],
env={"GMP_VERSION": Property("gmp_version")}))
self.addStep(
Compile(name="gmp-extract",
description=["gmp", "extract"],
descriptionDone=["extracted", "gmp"],
command=["make", "-f", "mingw-makefile", "gmp-extract"],
env={"GMP_VERSION": Property("gmp_version")}))
# Fix gmp (fails to find m4 for flex)
self.addStep(
ShellCommand(
name="gmp-patch",
workdir="build/src/gcc/src/gmp",
description=["patch", "gmp"],
command=[
"bash", "-c",
"""if [ -n "$( ls ../../../patches/gmp/*.patch )" ] ; then
for i in ../../../patches/gmp/*.patch ; do
patch -p1 -f -i "$i" ;
done ;
fi""".replace("\n", " ")
]))
# download mpfr
self.addStep(
Compile(name="mpfr-download",
description=["mpfr", "download"],
descriptionDone=["downloaded", "mpfr"],
command=["make", "-f", "mingw-makefile", "mpfr-download"],
env={"MPFR_VERSION": Property("mpfr_version")}))
self.addStep(
Compile(name="mpfr-extract",
description=["mpfr", "extract"],
descriptionDone=["extracted", "mpfr"],
command=["make", "-f", "mingw-makefile", "mpfr-extract"],
env={"MPFR_VERSION": Property("mpfr_version")}))
self.addStep(
ShellCommand(
name="mpfr-patch",
description=["patch", "mpfr"],
descriptionDone=["patched", "mpfr"],
doStepIf=lambda step: (step.getProperty("scheduler") != "try"),
workdir="build/src/gcc/src/mpfr",
command=[
"bash", "-c",
"""if [ -n "$( ls ../../../patches/mpfr/*.patch )" ] ; then
for i in ../../../patches/mpfr/*.patch ; do
patch -p1 -f -i "$i" ;
done ;
fi""".replace("\n", " ")
]))
# download mpc
self.addStep(
Compile(name="mpc-download",
description=["mpc", "download"],
descriptionDone=["downloaded", "mpc"],
command=["make", "-f", "mingw-makefile", "mpc-download"],
env={"MPC_VERSION": Property("mpc_version")}))
self.addStep(
Compile(name="mpc-extract",
description=["mpc", "extract"],
descriptionDone=["extracted", "mpc"],
command=["make", "-f", "mingw-makefile", "mpc-extract"],
env={"MPC_VERSION": Property("mpc_version")}))
self.addStep(
ShellCommand(
name="mpc-patch",
description=["patch", "mpc"],
descriptionDone=["patched", "mpc"],
doStepIf=lambda step: (step.getProperty("scheduler") != "try"),
workdir="build/src/gcc/src/mpc",
command=[
"bash", "-c",
"""if [ -n "$( ls ../../../patches/mpc/*.patch )" ] ; then
for i in ../../../patches/mpc/*.patch ; do
patch -p1 -f -i "$i" ;
done ;
fi""".replace("\n", " ")
]))
# download mingw-w64 crt and headers
self.addStep(
Compile(name="mingw-pull",
description=["mingw", "pull"],
descriptionDone=["pulled", "mingw"],
command=["make", "-f", "mingw-makefile", "mingw-pull"],
env={
"MINGW_REVISION":
Property("mingw_revision", default="head"),
"MINGW_BRANCH":
Property("mingw_branch")
}))
self.addStep(
ShellCommand(
name="mingw-patch",
description=["patch", "mingw"],
descriptionDone=["patched", "mingw"],
workdir="build/src/mingw",
doStepIf=lambda step: (step.getProperty("scheduler") != "try"),
command=[
"bash", "-c",
"""if [ -n "$( ls ../patches/mingw/*.patch )" ] ; then
for i in ../patches/mingw/*.patch ; do
patch -p1 -f -i "$i" ;
done ;
fi""".replace("\n", " ")
]))
# update the build stamp
self.addStep(
SubversionRevProperty(name="gcc-svnrev",
workdir="build/src/gcc/src",
prop_prefix="gcc_",
config_dir=WithProperties("%(basedir:-.)s")))
self.addStep(
SubversionRevProperty(name="mingw-svnrev",
workdir="build/src/mingw",
prop_prefix="mingw_",
config_dir=WithProperties("%(basedir:-.)s")))
self.addStep(
SetProperty(property="datestamp",
command=["date", "-u", "+_%Y%m%d"],
doStepIf=lambda step:
(not step.getProperty("datestamp") == "")))
self.addStep(
ShellCommand(
name="mingw-datestamp",
workdir="build/src/mingw/mingw-w64-crt",
description=["writing", "buildstamp"],
descriptionDone=["buildstamp", "written"],
command=[
"bash", "-c",
WithProperties(
"""echo -e '/* generated by buildbot */\n"""
"""#define __MINGW_W64_REV "%(mingw_revision)s"\n"""
"""#define __MINGW_W64_REV_STAMP "%(mingw_datestamp)s"\n'"""
""" > revstamp.h """)
]))
# Set the gcc version strings if this is a formal release
self.addStep(
ShellCommand(
name="release-information",
workdir="build/src/gcc/src/gcc",
description=["writing", "version", "string"],
descriptionDone=["version", "string", "written"],
doStepIf=lambda step: step.getProperty("release_build"),
command=[
"bash", "-c",
WithProperties(
"""echo '%(release_gcc_ver:-)s' > BASE-VER && echo > DEV-PHASE """
)
]))
# make the tarball
self.addStep(
SetProperty(property="destname",
command=[
"echo",
WithPropertiesRecursive(
WithProperties("%(srcname_format)s"))
]))
self.addStep(
Compile(name="src-package",
description=["tarball", "package"],
descriptionDone=["packaged", "tarball"],
command=["make", "-f", "mingw-makefile", "src-archive"],
env={"SRC_ARCHIVE": Property("filename")}))
# upload the tarball to the master
self.addStep(
FileUpload(name="src-upload",
slavesrc=Property("filename"),
masterdest=Property("filename")))
# trigger upload
self.addStep(
Trigger(
name="src-publish",
doStepIf=lambda step: step.build.getProperty("is_nightly"),
schedulerNames=["sourceforge-upload"],
waitForFinish=True, # needed for the builders
set_properties={
"masterdir": WithProperties("%(masterdir)s"),
"filename": WithProperties("%(filename)s"),
"destname": WithProperties("%(destname)s"),
"datestamp": WithProperties("%(datestamp:-)s"),
"target-os": "src",
"path": WithProperties("%(path:-)s"),
"is_nightly": WithProperties("%(is_nightly:-)s")
}))
# set the path that the build will be uploaded to (so the other slaves can
# go ahead and download the source tarballs from sourceforge rather than
# over the buildbot connection). Note that if the "path" property is set,
# we use that as an override instead.
self.addStep(
SetProperty(
property="src_url",
doStepIf=lambda step: step.build.getProperty("is_nightly"),
command=[
"echo",
WithProperties(
"http://downloads.sourceforge.net/project/%s/%%(path:-%s)s/%%(destname)s"
% (gConfig.get("sourceforge", "group_id"),
gConfig.get("sourceforge", "path-src")))
]))
# trigger building
self.addStep(
Trigger(name="start-build",
schedulerNames=[
"trigger-linux-x86_64-x86_64",
"trigger-linux-x86_64-x86", "trigger-linux-x86-x86_64",
"trigger-linux-x86-x86", "trigger-cygwin-x86-x86_64",
"trigger-cygwin-x86-x86", "trigger-mingw-x86-x86_64",
"trigger-mingw-x86-x86", "trigger-darwin-x86-x86_64",
"trigger-darwin-x86-x86"
],
waitForFinish=False,
updateSourceStamp=True,
set_properties={
"is_nightly":
Property("is_nightly"),
"datestamp":
Property("datestamp"),
"binutils_branch":
Property("binutils_branch"),
"gcc_branch":
Property("gcc_branch"),
"mingw_branch":
Property("mingw_branch"),
"binutils_config_args":
Property("binutils_config_args", default=""),
"gcc_config_args":
Property("gcc_config_args", default=""),
"mingw_config_args":
Property("mingw_config_args", default=""),
"gmp_config_args":
Property("gmp_config_args", default=""),
"mpfr_config_args":
Property("mpfr_config_args", default=""),
"mpc_config_args":
Property("mpc_config_args", default=""),
"gcc_revision":
Property("gcc_revision"),
"mingw_revision":
Property("mingw_revision"),
"masterdir":
Property("masterdir"),
"path":
Property("path"),
"src_archive":
Property("filename"),
"src_url":
Property("src_url", default="")
}))
def __init__(self, **kwargs):
factory.BuildFactory.__init__(self, **kwargs)
# set properties about this builder
self.addStep(
SetProperty(
property="basedir",
command=[
"bash", "-c",
"""pwd | sed 's@\\\\\\\\@/@g; s@^\\\\(.\\\\):@/\\\\1@ '"""
],
haltOnFailure=True))
self.addStep(
SetProperty(property="host_cpu", command=["echo", self.host_cpu]))
self.addStep(
SetProperty(property="host_os", command=["echo", self.host_os]))
target_os = "unknown"
for os_mask in [{
"expr": "x86_64",
"value": "w64"
}, {
"expr": "i.86",
"value": "w32"
}]:
if re.match(os_mask["expr"], self.target) is not None:
target_os = os_mask["value"]
break
self.addStep(
SetProperty(property="target-os", command=["echo", target_os]))
self.addStep(
SetProperty(property="target_arch", command=["echo", self.target]))
self.addStep(
SetProperty(property="gcc_config_args",
command=[
"echo",
Property("gcc_config_args", default=""),
self.gccConfigExtraArgs
]))
self.addStep(
SetProperty(property="mingw_config_args",
command=[
"echo",
Property("mingw_config_args", default=""),
self.crtConfigExtraArgs
]))
self.addStep(
SetProperty(property="binutils_config_args",
command=[
"echo",
Property("binutils_config_args", default=""),
self.binutilsConfigExtraArgs
]))
self.addStep(
SetProperty(property="gmp_config_args",
command=[
"echo",
Property("gmp_config_args", default=""),
self.gmpConfigExtraArgs
]))
self.addStep(
SetProperty(property="mpfr_config_args",
command=[
"echo",
Property("mpfr_config_args", default=""),
self.mpfrConfigExtraArgs
]))
self.addStep(
SetProperty(property="mpc_config_args",
command=[
"echo",
Property("mpc_config_args", default=""),
self.mpcConfigExtraArgs
]))
self.addStep(
SetProperty(command=["echo", self.file_extension],
property="file_extension"))
self._step_AdditionalProperties()
self.addStep(
SetProperty(
property="filename_format",
command=[
"echo",
"mingw-%(target-os)s-bin_%(host_cpu)s-%(host_os)s.%(file_extension)s"
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("filename_format"))))
self.addStep(
SetProperty(property="filename",
command=[
"echo",
WithPropertiesRecursive(
WithProperties("%(filename_format)s"))
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("filename"))))
self.addStep(
SetProperty(
property="destname_format",
command=[
"echo",
"mingw-%(target-os)s-bin_%(host_cpu)s-%(host_os)s%(host-extra:-)s%(datestamp:-)s.%(file_extension)s"
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("destname_format"))))
self.addStep(
SetProperty(property="destname",
command=[
"echo",
WithPropertiesRecursive(
WithProperties("%(destname_format)s"))
],
doStepIf=lambda step:
(not step.build.getProperties().has_key("destname"))))
# dump info about this slave
self.addStep(
ShellCommand(name="sys-info",
workdir=".",
command=[
"bash", "-c", r"""true
; echo -e "\n----------\nuname:\n----------"
; uname -a
; echo -e "\n----------\ndistro:\n----------"
; cat /etc/issue
; echo -e "\n----------\ncompiler version:\n----------"
; gcc --version
; echo -e "\n----------\nlinker version:\n----------"
; ld --version
; echo -e "\n----------\nassembler version:\n----------"
; as --version
; echo -e "\n----------\nflex version:\n----------"
; flex --version
; echo -e "\n----------\nbison version:\n----------"
; bison --version
; echo -e "\n----------\nmake version:\n----------"
; make --version
; true
""".replace("\n", "")
],
haltOnFailure=False,
description=["system info"],
descriptionDone=["system info"]))
# set up build root
if self.clobber:
self.addStep(
ShellCommand(name="clobber",
workdir=".",
command=[
"bash", "-c", " ".join("""
junction -d build/build/root/mingw || true ;
rm -rfv build""".split())
],
haltOnFailure=False,
description=["clobber all"],
descriptionDone=["clobbered"]))
# force the build/ directory to exist so we can download into it
self.addStep(MakeDirectory(dir="build/build", haltOnFailure=True))
self.addStep(
FileDownload(mastersrc="mingw-makefile",
slavedest="mingw-makefile",
maxsize=102400,
mode=0600,
haltOnFailure=True))
def getLLDBWindowsCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=r'C:/src/python',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[],
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
if config.lower() == 'release':
python_lib = 'python27.lib'
python_exe = 'python.exe'
elif config.lower() == 'debug':
python_lib = 'python27_d.lib'
python_exe = 'python_d.exe'
python_lib = os.path.join(python_source_dir, 'PCbuild', python_lib)
python_exe = os.path.join(python_source_dir, 'PCbuild', python_exe)
python_include = os.path.join(python_source_dir, 'Include')
# Use batch files instead of ShellCommand directly, Windows quoting is
# borked. FIXME: See buildbot ticket #595 and buildbot ticket #377.
f.addStep(
batch_file_download.BatchFileDownload(
name='cmakegen',
command=[
cmake,
"-G",
"Ninja",
"../llvm",
"-DCMAKE_BUILD_TYPE=" + config,
# Need to use our custom built version of python
'-DPYTHON_LIBRARY=' + python_lib,
'-DPYTHON_INCLUDE_DIR=' + python_include,
'-DPYTHON_EXECUTABLE=' + python_exe,
"-DCMAKE_INSTALL_PREFIX=../install"
] + extra_cmake_args,
workdir=build_dir))
f.addStep(
ShellCommand(name='cmake',
command=['cmakegen.bat'],
haltOnFailure=True,
description='cmake gen',
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getLLDBScriptCommandsFactory(
downloadBinary=True,
buildAndroid=False,
runTest=True,
scriptExt='.sh',
):
f = buildbot.process.factory.BuildFactory()
if scriptExt is '.bat':
pathSep = '.\\'
else:
pathSep = './'
# Update scripts
getShellCommandStep(f,
name='update scripts',
command=['updateScripts' + scriptExt])
# Acquire lock
if downloadBinary:
getShellCommandStep(
f,
name='acquire lock',
command=[pathSep + 'acquireLock' + scriptExt, 'totBuild'],
description='get')
# Checkout source code
getShellCommandStep(f,
name='checkout source code',
command=[
pathSep + 'checkoutSource' + scriptExt,
WithProperties('%(revision)s')
])
# Set source revision
f.addStep(
SetProperty(name="set revision",
command=[pathSep + 'getRevision' + scriptExt],
property="got_revision",
workdir="scripts"))
# Configure
getShellCommandStep(f,
name='cmake local',
command=[pathSep + 'cmake' + scriptExt])
# Build
getShellCommandStep(f,
name='ninja build local',
command=[pathSep + 'buildLocal' + scriptExt])
if buildAndroid:
getShellCommandStep(f,
name='build android',
command=[pathSep + 'buildAndroid' + scriptExt])
# Get lldb-server binaries
if downloadBinary:
getShellCommandStep(f,
name='get lldb-server binaries',
command=[
pathSep + 'downloadBinaries' + scriptExt,
WithProperties('%(got_revision)s')
])
# Test
if runTest:
getTestSteps(f, scriptExt, pathSep)
# upload test traces
getShellCommandStep(f,
name='upload test traces',
command=[
pathSep + 'uploadTestTrace' + scriptExt,
WithProperties('%(buildnumber)s'),
WithProperties('%(buildername)s')
],
flunkOnFailure=False)
# Upload lldb-server binaries and trigger android builders
if buildAndroid:
getShellCommandStep(f,
name='upload lldb-server binaries',
command=[pathSep + 'uploadBinaries' + scriptExt])
f.addStep(
trigger.Trigger(schedulerNames=['lldb_android_scheduler'],
updateSourceStamp=False,
waitForFinish=False))
# Release lock
if downloadBinary:
getShellCommandStep(
f,
name='release lock',
command=[pathSep + 'releaseLock' + scriptExt, 'totBuild'],
description='release',
alwaysRun=True)
return f
def addGCSUploadSteps(f,
package_name,
install_prefix,
gcs_directory,
env,
gcs_url_property=None,
use_pixz_compression=False,
xz_compression_factor=6):
"""
Add steps to upload to the Google Cloud Storage bucket.
f - The BuildFactory to modify.
package_name - The name of this package for the descriptions (e.g.
'stage 1')
install_prefix - The directory the build has been installed to.
gcs_directory - The subdirectory of the bucket root to upload to. This
should match the builder name.
env - The environment to use. Set BOTO_CONFIG to use a configuration file
in a non-standard location, and BUCKET to use a different GCS bucket.
gcs_url_property - Property to assign the GCS url to.
"""
gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/'
'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz')
time_fmt = '%Y-%m-%d_%H-%M-%S'
output_file_name = '../install.tar.xz'
gcs_url = \
WithProperties(
gcs_url_fmt,
gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'),
gcs_directory=lambda _: gcs_directory,
now=lambda _: datetime.utcnow().strftime(time_fmt))
if gcs_url_property:
f.addStep(
SetProperty(name="record GCS url for " + package_name,
command=['echo', gcs_url],
property=gcs_url_property))
if use_pixz_compression:
# tweak the xz compression level to generate packages faster
tar_command = ['tar', '-Ipixz', '-cvf', output_file_name, '.']
else:
xz_command = 'xz -{0}'.format(xz_compression_factor)
tar_command = ['tar', '-I', xz_command, '-cvf', output_file_name, '.']
f.addStep(
ShellCommand(name='package ' + package_name,
command=tar_command,
description='packaging ' + package_name + '...',
workdir=install_prefix,
env=env))
f.addStep(
ShellCommand(name='upload ' + package_name + ' to storage bucket',
command=['gsutil', 'cp', '../install.tar.xz', gcs_url],
description=('uploading ' + package_name +
'to storage bucket ...'),
workdir=install_prefix,
env=env))
def getLLDBRemoteTestSteps(f, bindir, build_type, remote_config, env):
if None in (remote_config.test_archs, remote_config.test_compilers):
return f
# only supports linux and android as remote target at this time
if remote_config.platform not in ('linux', 'android'):
return f
llvm_srcdir = "llvm"
llvm_builddir = "build"
stepDesc = remote_config.platform + "-" + remote_config.host_arch
# get hostname
slave_hostname = None
f.addStep(
SetProperty(name="get hostname",
command=["hostname"],
property="slave_hostname",
description="set slave hostname",
workdir="."))
# get configuration of remote target
# config file should be placed under builddir on builder machine
# file name: remote_cfg.json
# content: json format with keys [remote_platform]-[remote_arch]
# the value for each key defines "remote_host", "remote_port", "remote_dir", "toolchain", "deviceId"
# example: {"android-i386": {"remote_host":"localhost",
# "remote_port":"5430",
# "remote_dir":"/data/local/tmp/lldb",
# "toolchain_build":"/home/lldb_build/Toolchains/i386-android-toolchain-21",
# "toolchain_test":"/home/lldb_build/Toolchains/i386-android-toolchain-16",
# "deviceid":"XXXXXXX"},
def getRemoteCfg(rc, stdout, stderr):
return json.loads(stdout)[stepDesc]
f.addStep(
SetProperty(name="get remote target " + stepDesc,
command="cat remote_cfg.json",
extract_fn=getRemoteCfg,
description="get remote target",
workdir="."))
# rsync
if remote_config.platform is 'linux':
shellcmd = ['ssh', WithProperties('%(remote_host)s')]
hostname = '%(slave_hostname)s'
launchcmd = shellcmd + ['screen', '-d', '-m']
terminatecmd = shellcmd + ['pkill', 'lldb-server']
cleandircmd = WithProperties(
'ssh %(remote_host)s rm -r %(remote_dir)s/*')
f.addStep(
ShellCommand(name="rsync lldb-server",
command=[
'rsync', '-havL', 'bin/lldb-server',
WithProperties('%(remote_host)s:%(remote_dir)s')
],
description="rsync lldb-server " + stepDesc,
haltOnFailure=True,
env=env,
workdir='%s' % llvm_builddir))
f.addStep(
ShellCommand(name="rsync python2.7",
command=[
'rsync', '-havL', 'lib/python2.7',
WithProperties('%(remote_host)s:%(remote_dir)s')
],
description="rsync python2.7 " + stepDesc,
haltOnFailure=True,
env=env,
workdir='%s' % llvm_builddir))
elif remote_config.platform is 'android':
shellcmd = ['adb', '-s', WithProperties('%(deviceid)s'), 'shell']
hostname = '127.0.0.1'
launchcmd = ['screen', '-d', '-m'] + shellcmd + [
WithProperties("TMPDIR=%(remote_dir)s/tmp")
]
terminatecmd = 'ps | grep lldb-server | awk \'{print $2}\' | xargs'
terminatecmd = WithProperties('adb -s %(deviceid)s shell ' +
terminatecmd +
' adb -s %(deviceid)s shell kill')
cleandircmd = WithProperties(
'adb -s %(deviceid)s shell rm -rf %(remote_dir)s/*')
# compile lldb-server for target platform
f = getLLDBCmakeAndCompileSteps(f, 'gcc', build_type, ['lldb-server'],
bindir, remote_config.platform,
remote_config.host_arch, env)
f.addStep(
ShellCommand(name="adb push lldb-server " + stepDesc,
command=[
'adb', '-s',
WithProperties('%(deviceid)s'), 'push',
remote_config.platform + '-' +
remote_config.host_arch + '/bin/lldb-server',
WithProperties('%(remote_dir)s/')
],
description="lldb-server",
env=env,
haltOnFailure=True,
workdir='%s' % llvm_builddir))
f.addStep(
ShellCommand(name="Build fingerprint " + stepDesc,
command=[
'adb', '-s',
WithProperties('%(deviceid)s'), 'shell',
'getprop', 'ro.build.fingerprint'
],
description="get build fingerprint",
env=env,
haltOnFailure=False,
workdir='%s' % llvm_builddir))
# launch lldb-server
f.addStep(
ShellCommand(
name="launch lldb-server " + stepDesc,
command=launchcmd + [
WithProperties('%(remote_dir)s/lldb-server'), 'platform',
'--listen',
WithProperties(hostname + ':%(remote_port)s'), '--server'
],
description="launch lldb-server on remote host",
env=env,
haltOnFailure=True,
workdir='%s' % llvm_builddir))
# test steps
f = getLLDBTestSteps(f, bindir, remote_config.test_archs,
remote_config.test_compilers, remote_config.platform,
'%(remote_host)s', '%(remote_port)s',
'%(remote_dir)s', env)
# terminate lldb-server on remote host
f.addStep(
ShellCommand(name="terminate lldb-server " + stepDesc,
command=terminatecmd,
description="terminate lldb-server",
env=env,
workdir='%s' % llvm_builddir))
# clean remote test directory
f.addStep(
ShellCommand(name="clean remote dir " + stepDesc,
command=cleandircmd,
description="clean remote dir",
env=env))
return f
def getLLGoBuildFactory(
clean=True,
build_type='Release+Asserts',
test_libgo=True, # run 'check-libgo' target if True
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llgo_srcdir = '%s/tools/llgo' % llvm_srcdir
clang_srcdir = '%s/tools/clang' % llvm_srcdir
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Get LLVM, clang and llgo
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir=clang_srcdir))
f.addStep(
SVN(name='svn-llgo',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llgo/',
defaultBranch='trunk',
workdir=llgo_srcdir))
# Clean build directory, if requested.
f.addStep(
ShellCommand(name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
doStepIf=clean))
# Create configuration files with cmake
f.addStep(
ShellCommand(name="create-build-dir",
command=["mkdir", "-p", llvm_objdir],
haltOnFailure=False,
description=["create build dir"],
workdir="."))
cmakeCommand = [
"cmake",
"-G",
"Ninja",
"../%s" % llvm_srcdir,
"-DCMAKE_BUILD_TYPE=" + build_type,
]
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir))
# Build llgo
f.addStep(
NinjaCommand(name="build_llgo",
targets=["llgo"],
haltOnFailure=True,
description=["build llgo"],
workdir=llvm_objdir))
# Build libgo
f.addStep(
NinjaCommand(
name="build_libgo",
targets=["libgo"],
haltOnFailure=True,
logfiles={
'libgo-build-out':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-build-out.log',
'libgo-build-err':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-build-err.log',
'libgo-configure-out':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-configure-out.log',
'libgo-configure-err':
'tools/llgo/libgo-prefix/src/libgo-stamp/libgo-configure-err.log',
},
lazylogfiles=True,
description=["build libgo"],
workdir=llvm_objdir))
# Build llgoi
f.addStep(
NinjaCommand(name="build_llgoi",
targets=["llgoi"],
haltOnFailure=True,
description=["build llgoi"],
workdir=llvm_objdir))
# Test llgo
f.addStep(
NinjaCommand(name="test_llgo",
targets=["check-llgo"],
haltOnFailure=True,
description=["test llgo"],
workdir=llvm_objdir))
# Test libgo
f.addStep(
NinjaCommand(name="test_libgo",
targets=["check-libgo"],
haltOnFailure=True,
description=["test libgo"],
workdir=llvm_objdir))
return f
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Kill any stale symbolizer processes for the last run. If there are any
# stale processes, the build will fail during linking. This can happen to
# any process, but it is most likely to happen to llvm-symbolizer if its
# pipe isn't closed.
taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
f.addStep(
ShellCommand(name='taskkill',
description='kill stale processes',
command=['cmd', '/c', taskkill_cmd],
haltOnFailure=False))
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f, 'llvm')
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(
ShellCommand(
name='cmake',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON"
] + extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizer runtimes.
f.addStep(
NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Build Clang and LLD next so that most compilation errors occur in a build
# step.
f.addStep(
NinjaCommand(name='build clang lld',
targets=['clang', 'lld'],
haltOnFailure=True,
description='ninja clang lld',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
test_targets = [
'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi'
]
f.addStep(
NinjaCommand(name='run tests',
targets=test_targets,
haltOnFailure=True,
description='ninja test',
workdir=build_dir,
env=Property('slave_env')))
return f
def getLLDWinBuildFactory(
clean = True,
# Default values for VS devenv and build configuration
vs = None, # What to run to configure Visual Studio utils.
target_arch = None, # Native.
extra_configure_args = None,
env = None):
# Set defaults
if vs is None:
vs = r"""%VS140COMNTOOLS%""" # Visual Studio 2015.
if extra_configure_args is None:
extra_configure_args = []
if env is None:
env = {}
f = LLVMBuildFactory(
depends_on_projects=['llvm', 'lld'],
llvm_srcdir="llvm.src",
llvm_objdir="llvm.obj")
# Get LLVM and Lld
f.addSVNSteps()
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean ' + f.llvm_objdir,
dir=f.llvm_objdir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
# If set up environment step is requested, do this now.
if vs:
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
# Always build with ninja.
cmake_options = ["-G", "Ninja"]
# Reconsile configure args with the defaults we want.
if not any(a.startswith('-DCMAKE_BUILD_TYPE=') for a in extra_configure_args):
cmake_options.append('-DCMAKE_BUILD_TYPE=Release')
if not any(a.startswith('-DLLVM_ENABLE_WERROR=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_WERROR=ON')
if not any(a.startswith('-DLLVM_ENABLE_ASSERTIONS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_ENABLE_ASSERTIONS=ON')
if not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extra_configure_args):
cmake_options.append('-DLLVM_LIT_ARGS=\"-v\"')
cmake_options += extra_configure_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
warnOnWarnings=True,
options=cmake_options,
path="../%s" % f.llvm_srcdir,
env=env,
workdir=f.llvm_objdir,
doStepIf=FileDoesNotExist(
"./%s/CMakeCache.txt" % f.llvm_objdir)))
# Build Lld.
f.addStep(NinjaCommand(name='build lld',
haltOnFailure=True,
warnOnWarnings=True,
description='build lld',
workdir=f.llvm_objdir,
env=env))
# Test Lld
f.addStep(NinjaCommand(name='test lld',
targets=['lld-test'],
haltOnFailure=True,
warnOnWarnings=True,
description='test lld',
workdir=f.llvm_objdir,
env=env))
return f
def getLLDBWindowsCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=r'C:/Python35',
# Default values for VS devenv and build configuration
vs=r"""%VS140COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
# get full path to build directory
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=build_dir))
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
cmake_cmd = [
"cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config,
"-DPYTHON_HOME=" + python_source_dir,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLDB_TEST_COMPILER=\"%(builddir)s/bin/clang.exe\""
]
if extra_cmake_args:
cmake_cmd += extra_cmake_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
command=WithProperties(" ".join(cmake_cmd)),
haltOnFailure=True,
warnOnWarnings=True,
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=2400,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getAnnotatedBuildFactory(
script,
clean=False,
depends_on_projects=None,
env=None,
extra_args=None,
timeout=1200,
is_legacy_mode=False,
checkout_llvm_sources=True):
"""
Returns a new build factory that uses AnnotatedCommand, which
allows the build to be run by version-controlled scripts that do
not require a buildmaster restart to update.
script: script under "builders/annotated" to be run by python
clean: set to true for a clean build of llvm
depends_on_projects: which subprojects to enable
llvm must be first in the list
(default: ["llvm", "clang", "compiler-rt", "libcxx",
"libcxxabi", "libunwind", "lld"])
env: environment overrides (map; default is no overrides)
extra_args: extra arguments to pass to the script (default: [])
timeout: specifies the builder's timeout in seconds (default: 1200)
"""
if depends_on_projects is None:
depends_on_projects = [
"llvm",
"clang",
"compiler-rt",
"libcxx",
"libcxxabi",
"libunwind",
"lld"]
if extra_args is None:
extra_args = []
f = LLVMBuildFactory(
is_legacy_mode=is_legacy_mode,
clean=clean,
depends_on_projects=depends_on_projects)
if clean:
f.addStep(SetProperty(property='clean', command='echo 1'))
# We normally use the clean property to indicate that we want a
# clean build, but AnnotatedCommand uses the clobber property
# instead. Therefore, set clobber if clean is set to a truthy
# value. This will cause AnnotatedCommand to set
# BUILDBOT_CLOBBER=1 in the environment, which is how we
# communicate to the script that we need a clean build.
f.addStep(SetProperty(
property='clobber',
command='echo 1',
doStepIf=lambda step: step.build.getProperty('clean', False)))
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
scripts_dir = "annotated"
# Check out zorg so we can run the annotator scripts.
f.addGetSourcecodeForProject(
name='update-annotated-scripts',
project='zorg',
src_dir='llvm-zorg',
alwaysUseLatest=True)
if checkout_llvm_sources:
f.addGetSourcecodeSteps()
extra_args_with_props = [WithProperties(arg) for arg in extra_args]
# Explicitly use '/' as separator, because it works on *nix and Windows.
if script.startswith('/'):
command = [script]
else:
script_path = "../llvm-zorg/zorg/buildbot/builders/annotated/%s" % (script)
command = ["python", script_path, WithProperties("--jobs=%(jobs:-)s")]
command += extra_args_with_props
f.addStep(AnnotatedCommand(name="annotate",
description="annotate",
timeout=timeout,
haltOnFailure=True,
command=command,
env=merged_env))
return f
def get3StageClangLTOBuildFactory(
clean=True,
env=None,
build_gold=False,
cmake_cache_file=None,
extra_cmake_options=None,
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm-stage1.obj"
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
merged_env.update(env)
f = BuildFactory()
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
# We have to programatically determine the current llvm version.
def getClangVer(exit_status, stdout, stderr):
# We expect something like this:
# release = '3.9'
if exit_status:
return {}
k, v = stdout.split('=')
clang_ver = v.strip().strip("\'\"")
return {'clang_ver': clang_ver}
f.addStep(
SetProperty(
name="get_clang_ver",
command=["grep", "release =", "./tools/clang/docs/conf.py"],
extract_fn=getClangVer,
description="get clang release ver",
workdir=llvm_srcdir,
env=merged_env))
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
ShellCommand(doStepIf=cleanBuildRequested,
name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
cmake_command = ["cmake"]
if cmake_cache_file:
cmake_command += ['-C', cmake_cache_file]
if extra_cmake_options:
cmake_command += extra_cmake_options
cmake_command += ["../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
WarningCountingShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(
" ".join(cmake_command)),
workdir=llvm_objdir,
env=merged_env))
if build_gold:
f.addStep(
NinjaCommand(name='build-LLVMgold.so',
targets=['lib/LLVMgold.so'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
workdir=llvm_objdir,
env=merged_env))
f.addStep(
NinjaCommand(
name='build-stage3-clang',
targets=['stage3-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
timeout=3600, # Default value is not enough.
workdir=llvm_objdir,
env=merged_env))
f.addStep(
NinjaCommand(name='build-stage3-check-clang',
targets=['stage3-check-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["Check Clang"],
workdir=llvm_objdir,
env=merged_env))
# Compare stage2 & stage3 clang
shell_command = [
"diff", "-q", "tools/clang/stage2-bins/bin/clang-%(clang_ver)s",
"tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang-%(clang_ver)s"
]
f.addStep(
ShellCommand(name="compare-clang",
description=["comapre stage2 & stage3 clang"],
haltOnFailure=True,
command=WithProperties(" ".join(shell_command)),
workdir=llvm_objdir,
env=merged_env))
return f
def getSanitizerWindowsBuildFactory(
clean=False,
cmake='cmake',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(SetProperty(
command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getSource(f,'llvm')
# Global configurations
build_dir='build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(RemoveDirectory(name='clean '+build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested
))
f.addStep(ShellCommand(name='cmake',
command=[cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE="+config,
"-DLLVM_ENABLE_ASSERTIONS=ON"]
+ extra_cmake_args,
haltOnFailure=True,
workdir=build_dir,
env=Property('slave_env')))
# Build compiler-rt first to speed up detection of Windows-specific
# compiler-time errors in the sanitizers runtime.
f.addStep(NinjaCommand(name='build compiler-rt',
targets=['compiler-rt'],
haltOnFailure=True,
description='ninja compiler-rt',
workdir=build_dir,
env=Property('slave_env')))
# Only run sanitizer tests.
# Don't build targets that are not required in order to speed up the cycle.
test_targets = ['check-asan','check-asan-dynamic','check-sanitizer', 'check-cfi']
f.addStep(NinjaCommand(name='run tests',
targets=test_targets,
haltOnFailure=True,
description='ninja test',
workdir=build_dir,
env=Property('slave_env')))
return f
def getCmakeWithMSVCBuildFactory(
clean=True, # False for incremental builds.
depends_on_projects=None, # List of projects to listen.
cmake_cache=None, # Path to a cmake cache file.
extra_configure_args=None, # Extra CMake args if any.
llvm_srcdir=None, # Source code root directory.
obj_dir=None, # Build tree root directory.
install_dir=None, # Directory to install the results to.
checks=None, # List of checks to test the build.
checks_on_target=None, # [(<name>,[<command tokens>])] array of
# check name and command to run on target.
jobs=None, # Restrict a degree of parallelism.
env=None, # Environmental variables for all steps.
# VS tools environment variable if using MSVC.
# For example, "autodetect" to auto detect, %VS140COMNTOOLS% to select
# the VS 2015 toolchain, or empty string if environment is already set.
vs=None,
**kwargs):
if vs is None:
# We autodetect Visual Studio, unless otherwise is requested.
vs = "autodetect"
if install_dir is None:
install_dir = 'install'
# Prepare environmental variables. Set here all env we want for all steps.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not vs, "Cannot have custom builder env vars with VS setup."
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
# Make a local copy of the configure args, as we are going to modify that.
if extra_configure_args:
cmake_args = extra_configure_args[:]
else:
cmake_args = list()
if depends_on_projects is None:
depends_on_projects = [
'llvm',
'compiler-rt',
'clang',
'clang-tools-extra',
'libunwind',
'libcxx',
'libcxxabi',
'lld',
]
if checks is None:
# Check only host-side tools. Target-side tests should run on a target.
checks = [
"check-llvm",
"check-clang",
"check-lld",
]
source_remove_requested = lambda step: step.build.getProperty("clean")
clean_build_requested = lambda step: \
clean or \
step.build.getProperty("clean", \
default=step.build.getProperty("clean_obj") \
)
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=obj_dir,
install_dir=install_dir,
cleanBuildRequested=clean_build_requested,
**kwargs) # Pass through all the extra arguments.
# Remove the source code tree if requested.
# NOTE: Somehow RemoveDirectory buildbot command often fails on Windows,
# as somthing keeps a lock. We use rm command instead realying on a shell
# to support that.
f.addStep(
ShellCommand(name='clean-%s-dir' % f.monorepo_dir,
command=['rm', '-rf', f.monorepo_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove the source code',
workdir='.',
env=merged_env,
doStepIf=source_remove_requested))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Remove the build directory if requested.
f.addStep(
ShellCommand(name='clean-%s-dir' % f.obj_dir,
command=['rm', '-rf', f.obj_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove build directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
if vs:
# Configure MSVC environment if requested.
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, None),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Since this is a build of a cross toolchain, we build only the host-side
# tools first by the host system compiler. Libraries will be cross-compiled.
cmake_args.append(
InterpolateToPosixPath('-DLLVM_AR=%(workdir)s/' + f.obj_dir +
'/bin/llvm-ar.exe')),
CmakeCommand.applyDefaultOptions(cmake_args, [
('-G', 'Ninja'),
('-DLLVM_ENABLE_PROJECTS=', 'llvm;clang;clang-tools-extra;lld'),
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DCMAKE_CXX_FLAGS=', '-D__OPTIMIZE__'),
('-DLLVM_ENABLE_ASSERTIONS=', 'ON'),
('-DLLVM_LIT_ARGS=', '-v -vv'),
])
if install_dir:
install_dir_rel = LLVMBuildFactory.pathRelativeToBuild(
install_dir, f.obj_dir)
CmakeCommand.applyRequiredOptions(cmake_args, [
('-DCMAKE_INSTALL_PREFIX=', install_dir_rel),
])
# Remove the build directory if requested.
f.addStep(
ShellCommand(name='clean-%s-dir' % install_dir,
command=['rm', '-rf', install_dir],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='Remove install directory',
workdir='.',
env=merged_env,
doStepIf=clean_build_requested))
src_dir_rel = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
f.obj_dir)
# Add given cmake cache at the very end.
if cmake_cache:
cmake_args.append('-C%s' % cmake_cache)
f.addStep(
CmakeCommand(
name="cmake-configure",
haltOnFailure=True,
description=["Cmake", "configure"],
options=cmake_args,
path=src_dir_rel,
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
f.addStep(
NinjaCommand(
name="build-%s" % f.monorepo_dir,
haltOnFailure=True,
description=["Build", f.monorepo_dir],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Test the components if requested one check at a time.
for check in checks:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=["ninja", WithProperties(check)],
description=[
"Testing", "just", "built", "components", "for", check
],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"
],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Run commands on a target if requested.
if checks_on_target:
for check, cmd in checks_on_target:
f.addStep(
LitTestCommand(
haltOnFailure=False, # We want to test as much as we could.
name='test-%s' % check,
command=cmd,
description=[
"Testing", "just", "built", "components", "for", check
],
descriptionDone=[
"Test", "just", "built", "components", "for", check,
"completed"
],
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
# Install just built components
if install_dir:
f.addStep(
NinjaCommand(
name="install-all",
haltOnFailure=True,
targets=["install"],
description=["Install", "just", "built", "components"],
workdir=f.obj_dir,
env=merged_env,
**kwargs # Pass through all the extra arguments.
))
return f
def getLLDBCMakeBuildFactory(
clean=False,
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
testTimeout=2400,
install=False):
############# PREPARING
# Global configurations
build_dir = 'build'
f = LLVMBuildFactory(depends_on_projects=["llvm", "clang", "lldb", "lld"],
obj_dir=build_dir)
env = {}
# Determine Slave Environment and Set MSVC environment.
if vs and vs != 'manual':
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractVSEnvironment))
env = Property('vs_env')
f.addGetSourcecodeSteps()
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
############# CLEANING
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
f.addStep(
steps.RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, f.obj_dir)
cmake_options = [
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
]
if python_source_dir:
cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_options += extra_cmake_args
f.addStep(
CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_options,
path=rel_src_dir,
env=env,
workdir=build_dir))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=env))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=env))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=testTimeout,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=env))
return f
def getClangBuildFactory(
triple=None,
clean=True,
test=True,
package_dst=None,
run_cxx_tests=False,
examples=False,
valgrind=False,
valgrindLeakCheck=False,
useTwoStage=False,
completely_clean=False,
make='make',
jobs="%(jobs)s",
stage1_config='Debug+Asserts',
stage2_config='Release+Asserts',
env={}, # Environmental variables for all steps.
extra_configure_args=[],
stage2_extra_configure_args=[],
use_pty_in_tests=False,
trunk_revision=None,
force_checkout=False,
extra_clean_step=None,
checkout_compiler_rt=False,
checkout_lld=False,
run_gdb=False,
run_modern_gdb=False,
run_gcc=False):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
llvm_srcdir = "llvm.src"
llvm_1_objdir = "llvm.obj"
llvm_1_installdir = "llvm.install.1"
llvm_2_objdir = "llvm.obj.2"
llvm_2_installdir = "llvm.install"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Blow away completely, if requested.
if completely_clean:
f.addStep(
ShellCommand(name="rm-llvm.src",
command=["rm", "-rf", llvm_srcdir],
haltOnFailure=True,
description=["rm src dir", "llvm"],
workdir=".",
env=merged_env))
# Checkout sources.
if trunk_revision:
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout']
if force_checkout:
svn_co += ['--force']
svn_co += ['--revision', WithProperties(trunk_revision)]
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' %
trunk_revision),
llvm_srcdir]
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' %
trunk_revision),
'%s/tools/clang' % llvm_srcdir]
svn_co_clang_tools_extra = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' %
trunk_revision),
'%s/tools/clang/tools/extra' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang-tools-extra',
command=svn_co_clang_tools_extra,
haltOnFailure=True,
workdir='.'))
else:
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if checkout_compiler_rt:
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
# Clean up llvm (stage 1); unless in-dir.
if clean and llvm_srcdir != llvm_1_objdir:
f.addStep(
ShellCommand(name="rm-llvm.obj.stage1",
command=["rm", "-rf", llvm_1_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
if not clean:
expected_makefile = 'Makefile'
f.addStep(
SetProperty(name="Makefile_isready",
workdir=llvm_1_objdir,
command=[
"sh", "-c",
"test -e %s && echo OK || echo Missing" %
expected_makefile
],
flunkOnFailure=False,
property="exists_Makefile"))
cmake_triple_arg = []
if triple:
cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple]
f.addStep(
ShellCommand(name='cmake',
command=[
'cmake', '-DLLVM_BUILD_TESTS=ON',
'-DCMAKE_BUILD_TYPE=%s' % stage1_config
] + cmake_triple_arg + extra_configure_args +
["../" + llvm_srcdir],
description='cmake stage1',
workdir=llvm_1_objdir,
env=merged_env,
doStepIf=lambda step: step.build.getProperty(
"exists_Makefile") != "OK"))
# Make clean if using in-dir builds.
if clean and llvm_srcdir == llvm_1_objdir:
f.addStep(
WarningCountingShellCommand(name="clean-llvm",
command=[make, "clean"],
haltOnFailure=True,
description="cleaning llvm",
descriptionDone="clean llvm",
workdir=llvm_1_objdir,
doStepIf=clean,
env=merged_env))
if extra_clean_step:
f.addStep(extra_clean_step)
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
haltOnFailure=True,
flunkOnFailure=not run_gdb,
description=["compiling", stage1_config],
descriptionDone=["compile", stage1_config],
workdir=llvm_1_objdir,
env=merged_env))
if examples:
f.addStep(
WarningCountingShellCommand(
name="compile.examples",
command=[
'nice', '-n', '10', make,
WithProperties("-j%s" % jobs), "BUILD_EXAMPLES=1"
],
haltOnFailure=True,
description=["compiling", stage1_config, "examples"],
descriptionDone=["compile", stage1_config, "examples"],
workdir=llvm_1_objdir,
env=merged_env))
clangTestArgs = '-v -j %s' % jobs
if valgrind:
clangTestArgs += ' --vg'
if valgrindLeakCheck:
clangTestArgs += ' --vg-leak'
clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp'
extraTestDirs = ''
if run_cxx_tests:
extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests'
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='check-all',
command=[
make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)
],
flunkOnFailure=not run_gdb,
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_1_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# TODO: Install llvm and clang for stage1.
if run_gdb or run_gcc or run_modern_gdb:
ignores = getClangTestsIgnoresFromPath(
os.path.expanduser('~/public/clang-tests'),
'clang-x86_64-darwin10')
install_prefix = "%%(builddir)s/%s" % llvm_1_installdir
if run_gdb:
addClangGDBTests(f, ignores, install_prefix)
if run_modern_gdb:
addModernClangGDBTests(f, jobs, install_prefix)
if run_gcc:
addClangGCCTests(f, ignores, install_prefix)
if not useTwoStage:
if package_dst:
name = WithProperties(
"%(builddir)s/" + llvm_1_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(
ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_1_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(
ShellCommand(name='pkg.upload',
description="upload root",
command=[
"scp", name,
WithProperties(package_dst +
"/%(buildername)s")
],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
# Clean up llvm (stage 2).
#
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we should
# not build in the first place.
f.addStep(
ShellCommand(name="rm-llvm.obj.stage2",
command=["rm", "-rf", llvm_2_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm", "(stage 2)"],
workdir=".",
env=merged_env))
# Configure llvm (stage 2).
f.addStep(
ShellCommand(
name='cmake',
command=['cmake'] + stage2_extra_configure_args + [
'-DLLVM_BUILD_TESTS=ON',
WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang'
% llvm_1_objdir), # FIXME use installdir
WithProperties(
'-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' %
llvm_1_objdir),
'-DCMAKE_BUILD_TYPE=%s' % stage2_config,
"../" + llvm_srcdir
],
description='cmake stage2',
workdir=llvm_2_objdir,
env=merged_env))
# Build llvm (stage 2).
f.addStep(
WarningCountingShellCommand(
name="compile.llvm.stage2",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["compiling", "(stage 2)", stage2_config],
descriptionDone=["compile", "(stage 2)", stage2_config],
workdir=llvm_2_objdir,
env=merged_env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='check-all',
command=[
make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)
],
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_2_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# TODO: Install llvm and clang for stage2.
if package_dst:
name = WithProperties("%(builddir)s/" + llvm_2_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(
ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_2_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(
ShellCommand(name='pkg.upload',
description="upload root",
command=[
"scp", name,
WithProperties(package_dst + "/%(buildername)s")
],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
def getClangAndLLDBuildFactory(
clean=True,
env=None,
withLLD=True,
extraCmakeOptions=None,
extraCompilerOptions=None,
buildWithSanitizerOptions=None,
triple=None,
isMSVC=False,
# Choose VS tools to build with. For example,
# "autodetect" to find the latest installed Visual Studio, or
# %VS140COMNTOOLS% to selects the 2015 toolchain.
vs=None,
target_arch=None,
prefixCommand=["nice", "-n", "10"], # For backward compatibility.
extraLitArgs=None):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
assert not (isMSVC
and vs), "Can't have custom builder env vars with VS"
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Get LLVM, Clang and LLD.
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Sanitizer runtime in compiler-rt, and cannot be built with sanitizer compiler.
if buildWithSanitizerOptions is None:
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if withLLD:
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='%s/tools/lld' % llvm_srcdir))
# Clean directory, if requested.
if clean:
shellCommand = ["rm", "-rf", llvm_objdir]
if isMSVC:
shellCommand = ["rmdir", "/S", "/Q", llvm_objdir]
f.addStep(
ShellCommand(name="rm-llvm_objdir",
command=shellCommand,
haltOnFailure=False,
flunkOnFailure=False,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
# Create configuration files with cmake.
options = ["-Wdocumentation", "-Wno-documentation-deprecated-sync"]
if isMSVC:
options = []
if extraCompilerOptions:
options += extraCompilerOptions
if buildWithSanitizerOptions:
options += buildWithSanitizerOptions
lit_args = ["-v"]
if extraLitArgs:
lit_args += extraLitArgs
lit_args = ["-DLLVM_LIT_ARGS=\"%s\"" % " ".join(lit_args)]
cmakeCommand = [
"cmake", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_ENABLE_ASSERTIONS=ON"
]
if buildWithSanitizerOptions:
cmakeCommand += [
"-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++"
]
if triple:
cmakeCommand += ["-DLLVM_DEFAULT_TARGET_TRIPLE=%s" % triple]
if extraCmakeOptions:
assert not any(a.startswith('-DLLVM_LIT_ARGS=') for a in extraCmakeOptions), \
"Please use extraLitArgs for LIT arguments instead of defining them in extraCmakeOptions."
cmakeCommand += extraCmakeOptions
if not isMSVC:
cmakeCommand += [
"-DCMAKE_C_FLAGS=\"%s\"" % (" ".join(options)),
"-DCMAKE_CXX_FLAGS=\"-std=c++11 %s\"" % (" ".join(options)),
]
cmakeCommand += lit_args
cmakeCommand += ["-GNinja", "../%s" % llvm_srcdir]
if isMSVC and vs:
# Set up VS environment, if requested.
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
merged_env = Property('slave_env')
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand)),
workdir=llvm_objdir,
env=merged_env))
# Build everything.
f.addStep(
NinjaCommand(name="build",
prefixCommand=prefixCommand,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
# Test everything.
f.addStep(
NinjaCommand(name="test",
prefixCommand=prefixCommand,
targets=["check-all"],
haltOnFailure=True,
description=["test"],
workdir=llvm_objdir,
env=merged_env))
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=[],
testsuite_flags=[],
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env={},
extra_cmake_args=[],
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=[]):
############# PREPARING
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if runTestSuite or checkout_test_suite:
depends_on_projects.append('lnt')
depends_on_projects.append('test-suite')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
f.addSVNSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
cmake_cmd2 = [
'env',
WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
cxx), cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def __init__(self, rmConfig, **kwargs):
BuildFactory.__init__(self, **kwargs)
self.addStep(
SetProperty(
name='set_topdir',
command=['pwd'],
property='topdir',
workdir='.',
))
self.addStep(
ShellCommand(
command=['rm', '-rvf', 'tools', 'buildbot-configs'],
workdir='.',
))
self.addStep(
ShellCommand(
command=[
'hg', 'clone',
make_hg_url(rmConfig['HG_HOST'], 'build/tools'), 'tools'
],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=[
rmConfig['PYTHON'], 'scripts/preproduction/repo_setup.py',
'-c', 'scripts/preproduction/repo_setup_config.py'
],
workdir='tools',
haltOnFailure=True,
))
self.addStep(
SetProperty(
property='previousSetupMakefile',
command='ls %s/Makefile 2>/dev/null || exit 0' %
rmConfig['releaseMasterDir'],
flunkOnFailure=False,
haltOnFailure=False,
warnOnFailure=True,
))
def previousSetupExists(step):
return \
step.build.getProperties().has_key('previousSetupMakefile') \
and len(step.build.getProperty('previousSetupMakefile')) > 0
self.addStep(
ShellCommand(
command=[
rmConfig['PYTHON'],
'tools/buildfarm/maintenance/buildbot-wrangler.py', 'stop',
'%s/master' % rmConfig['releaseMasterDir']
],
workdir=rmConfig['releaseMasterDir'],
flunkOnFailure=False,
doStepIf=previousSetupExists,
))
self.addStep(
ShellCommand(
command=['rm', '-rvf', rmConfig['releaseMasterDir']],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=['rm', '-rvf', 'buildbot-configs'],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=[
'hg', 'clone',
make_hg_url(rmConfig['HG_HOST'],
'%s/buildbot-configs' % rmConfig['HG_DIR']),
'buildbot-configs'
],
workdir='.',
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=[
'make', '-f', 'Makefile.setup',
'PYTHON=%s' % rmConfig['PYTHON'],
'VIRTUALENV=%s' % rmConfig['VIRTUALENV'],
'HG_DIR=%s' % rmConfig['HG_DIR'], 'MASTER_NAME=pp-release',
'MASTERS_JSON=/home/cltbld/conf/master_config.json',
'BASEDIR=%s' % rmConfig['releaseMasterDir'],
'BUILDBOTCUSTOM_BRANCH=%s' %
rmConfig['BUILDBOTCUSTOM_BRANCH'],
'BUILDBOTCONFIGS_BRANCH=%s' %
rmConfig['BUILDBOTCONFIGS_BRANCH'], 'virtualenv', 'deps',
'install-buildbot', 'master', 'master-makefile'
],
workdir='buildbot-configs',
env={
'PIP_DOWNLOAD_CACHE': WithProperties('%(topdir)s/cache'),
'CC': None,
'CXX': None
},
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=['make', 'checkconfig'],
workdir=rmConfig['releaseMasterDir'],
haltOnFailure=True,
))
self.addStep(
ShellCommand(
command=['touch', 'twistd.log'],
workdir='%s/master' % rmConfig['releaseMasterDir'],
))
self.addStep(
ShellCommand(
command=[
'bash', '-c',
'if [ -e ~/conf/passwords.py ]; then cp -fv ~/conf/passwords.py ./; fi'
],
workdir='%s/master' % rmConfig['releaseMasterDir'],
))
self.addStep(
ShellCommand(
command=[
rmConfig['PYTHON'],
'tools/buildfarm/maintenance/buildbot-wrangler.py',
'start',
'%s/master' % rmConfig['releaseMasterDir']
],
workdir=rmConfig['releaseMasterDir'],
))
for release_config in rmConfig['releaseConfigs']:
self.addStep(
SetProperty(
property='release_tag',
command=[
rmConfig['PYTHON'], '-c',
'execfile("%s"); \
print releaseConfig["baseTag"] + "_RELEASE"' %
release_config
],
workdir='%s/buildbot-configs' %
rmConfig['releaseMasterDir'],
))
self.addStep(
SetProperty(
property='release_branch',
command=[
rmConfig['PYTHON'], '-c',
'execfile("%s"); \
print releaseConfig["sourceRepositories"]["mozilla"]["path"]'
% release_config
],
workdir='%s/buildbot-configs' %
rmConfig['releaseMasterDir'],
))
self.addStep(
ShellCommand(
command=[
'buildbot', 'sendchange', '--username',
'preproduction', '--master',
rmConfig['releaseMasterHostPort'], '--branch',
WithProperties('%(release_branch)s'), '-p',
'products:firefox', '-p',
WithProperties('script_repo_revision:%(release_tag)s'),
'release_build'
],
workdir='.',
))
def getLLDBUbuntuCMakeBuildFactory(build_compiler,
build_type,
local_test_archs=None,
local_test_compilers=None,
remote_configs=None,
jobs='%(jobs)s',
env=None):
"""Generate factory steps for ubuntu cmake builder
Arguments:
build_compiler -- string of compile name, example 'clang',
the compiler will be used to build binaries for host platform
build_type -- 'Debug' or 'Release',
used to define build type for host platform as well as remote platform if any
local_test_archs -- list of architectures, example ['i386','x86_64'],
defines architectures to run local tests against, if None, local tests won't be executed
local_test_compiler -- list of compilers, example ['clang','gcc4.8.2'],
definds compilers to run local tests with, if None, local tests won't be executed
remote_configs -- list of RemoteConfig objects, example [RemoteConfig(...)], if None, remote tests won't be executed
jobs -- number of threads for compilation step, example 40
default value is jobs number defined during slave creation
env -- environment variables passed to shell commands
"""
if env is None:
env = {}
llvm_srcdir = "llvm"
llvm_builddir = "build"
bindir = '%(builddir)s/' + llvm_builddir + '/bin'
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Determine the binary directory of *-tblgen.
f.addStep(
SetProperty(name="get tblgen dir",
command=["echo", WithProperties(bindir)],
property="tblgen_bindir",
description="set tblgen dir",
workdir="."))
# Get source code
f = getLLDBSource(f, llvm_srcdir)
# Clean Build Folder
f.addStep(
ShellCommand(name="clean",
command="rm -rf *",
description="clear build folder",
env=env,
workdir='%s' % llvm_builddir))
f = getLLDBCmakeAndCompileSteps(f, build_compiler, build_type, [], bindir,
'linux', 'x86_64', env)
# TODO: it will be good to check that architectures listed in test_archs are compatible with host architecture
# For now, the caller of this function should make sure that each target architecture is supported by builder machine
# Add local test steps
f = getLLDBTestSteps(f, bindir, local_test_archs, local_test_compilers)
# Remote test steps
if remote_configs is not None:
for config in remote_configs:
f = getLLDBRemoteTestSteps(f, bindir, build_type, config, env)
# archive test traces
f = archiveLLDBTestTraces(f, "build/lldb-test-traces-*")
return f
def CreateWinChromeFactory(builder):
"""Run chrome tests with the latest drmemory.
Do *not* build TOT chrome or sync it. Building chrome takes a lot of
resources and the tests are flaky, so we only do at known good revisions. We
don't want to fall too far behind, or we're not really testing Chrome's full
test suite.
"""
ret = factory.BuildFactory()
sfx_name = 'drm-sfx' # TODO: add .exe when BB supports that, d'oh!
ret.addStep(
FileDownload(mastersrc=LATEST_WIN_BUILD,
slavedest=(sfx_name + '.exe'),
name='Download the latest build'))
ret.addStep(
ShellCommand(command=[sfx_name, '-ounpacked', '-y'],
haltOnFailure=True,
name='Unpack the build',
description='unpack the build'))
# Find out the revision number using -version
def get_revision(rc, stdout, stderr):
m = re.search(r'version \d+\.\d+\.(\d+)', stdout)
if m:
return {'got_revision': int(m.groups()[0])}
return {'failed_to_parse': stdout}
ret.addStep(
SetProperty(command=['unpacked\\bin\\drmemory', '-version'],
extract_fn=get_revision,
name='Get the revision number',
description='get revision',
descriptionDone='get revision'))
# VP8 tests
# TODO(rnk): Add back the VP8 test step. We might be able to make this part
# of the buildbot steps if it doesn't update often and builds incrementally.
if False:
ret.addStep(
ToolStep(DrMemoryTest,
'windows',
command=[
'bash',
'E:\\vpx\\vp8-test-vectors\\run_tests.sh',
('--exec=unpacked/bin/drmemory.exe -batch '
'-no_check_leaks -no_count_leaks '
'-no_check_uninitialized '
'e:/vpx/b/Win32/Debug/vpxdec.exe'),
'E:\\vpx\\vp8-test-vectors',
],
env={'PATH': 'C:\\cygwin\\bin;%PATH%'},
name='VP8 tests',
descriptionDone='VP8 tests',
description='run vp8 tests'))
# Chromium tests
for test in [
'url', 'printing', 'media', 'sql', 'crypto_unittests', 'remoting',
'ipc_tests', 'base_unittests', 'net', 'unit'
]:
ret.addStep(
Test(
command=[
# Use the build dir of the chrome builder on this slave.
('..\\..\\' + builder + '\\build\\' +
'src\\tools\\valgrind\\chrome_tests.bat'),
'-t',
test,
'--tool',
'drmemory_light',
'--keep_logs',
],
env={'DRMEMORY_COMMAND': 'unpacked/bin/drmemory.exe'},
name=('Chromium \'%s\' tests' % test),
descriptionDone=('\'%s\' tests' % test),
description=('run \'%s\' tests' % test)))
return ret
def getLLDBxcodebuildFactory(use_cc=None,
build_type='Debug',
remote_configs=None,
env=None):
if env is None:
env = {}
f = buildbot.process.factory.BuildFactory()
f.addStep(
SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.'))
lldb_srcdir = 'lldb'
OBJROOT = '%(builddir)s/' + lldb_srcdir + '/build'
f.addStep(
SetProperty(name='get_bindir',
command=[
'echo',
WithProperties('%(builddir)s/' + lldb_srcdir +
'/build/' + build_type)
],
property='lldb_bindir',
description='set bin dir',
workdir='.'))
# cleaning out the build directory is vital for codesigning.
f.addStep(
ShellCommand(name='clean.lldb-buid',
command=['rm', '-rf',
WithProperties(OBJROOT)],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(name='clean.llvm-buid',
command=['rm', '-rf',
'%s/llvm-build' % lldb_srcdir],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(name='clean.test trace',
command='rm -rf build/*',
haltOnFailure=False,
flunkOnFailure=False,
workdir='.'))
# Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency
f.addStep(
ShellCommand(name='remove symbolic link lldb',
command=['rm', lldb_srcdir + '/llvm/tools/lldb'],
haltOnFailure=False,
flunkOnFailure=False,
workdir=WithProperties('%(builddir)s')))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
workdir=lldb_srcdir))
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir='%s/llvm' % lldb_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/llvm/tools/clang' % lldb_srcdir))
# setup keychain for codesign
# In order for the codesigning to work inside of buildbot, security must be
# called to unlock the keychain, which requires a password.
# I've set up a special keychain for this purpose, so as to not compromise
# the login password of the buildslave.
# This means I have to set the special keychain as the default and unlock it
# prior to building the sources.
f.addStep(
ShellCommand(name='check.keychain',
command=['security', 'default-keychain'],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(
name='find.certificate',
command=['security', 'find-certificate', '-c', 'lldb_codesign'],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
# Building the sources
#
buildcmd = ' '.join([
'xcrun', 'xcodebuild', '-target', 'desktop', '-configuration',
build_type, 'SYMROOT=' + OBJROOT, 'OBJROOT=' + OBJROOT
])
f.addStep(
ShellCommand(name='lldb-build',
command=WithProperties(buildcmd + " || " + buildcmd),
haltOnFailure=True,
workdir=lldb_srcdir))
# Testing
#
if not use_cc:
use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/'
use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang'
f.addStep(
SetProperty(name='set.cc',
command=['xcrun', '-find', 'clang'],
property='use_cc',
description='set cc',
workdir=lldb_srcdir))
else:
f.addStep(
SetProperty(name='set.cc',
command=['echo', use_cc],
property='use_cc',
description='set cc',
workdir=lldb_srcdir))
DOTEST_OPTS = ' '.join([
'--executable', '%(lldb_bindir)s/lldb', '--framework',
'%(lldb_bindir)s/LLDB.framework', '-A', 'x86_64', '-C', 'clang', '-s',
'../../build/lldb-test-traces'
])
f.addStep(
LitTestCommand(
name='lldb-test',
command=['./dosep.py', '--options',
WithProperties(DOTEST_OPTS)],
haltOnFailure=False,
workdir='%s/test' % lldb_srcdir,
env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')}))
# Remote test steps
if remote_configs is not None:
# Source structure to use cmake command
f.addStep(
SetProperty(
name='get tblgen bindir',
command=[
'echo',
WithProperties(
'%(builddir)s/' + lldb_srcdir +
'/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin'
)
],
property='tblgen_bindir',
description='set tblgen binaries dir',
workdir='.'))
f = getSymbLinkSteps(f, lldb_srcdir)
for config in remote_configs:
f = getLLDBRemoteTestSteps(
f,
'%(lldb_bindir)s',
build_type,
config,
env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')})
# Compress and upload test log
f = archiveLLDBTestTraces(f, "build/lldb-test-traces*")
# Results go in a directory coded named according to the date and time of the test run, e.g.:
#
# 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log
#
# Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error. return f
return f
}))
self.addStep(
Compile(name="gcc-install",
description=["gcc install"],
descriptionDone=["gcc installed"],
command=["make", "-f", "mingw-makefile", "gcc-install"],
env={
"SRC_ARCHIVE": WithProperties("%(src_archive)s"),
"TARGET_ARCH": WithProperties("%(target_arch)s")
}))
self.addStep(
SetProperty(property="filename",
command=[
"echo",
WithPropertiesRecursive(
WithProperties("%(filename_format)s"))
]))
# make the tarball
self._step_Archive()
# do a very simple test
self.addStep(
Compile(
name="sanity-check",
description=["sanity check"],
descriptionDone=["sanity check", "completed"],
command=[
"bash", "-c",
WithProperties("""echo "int main(){return 0;}" > test.c ;
def getLLDBBuildFactory(triple,
useTwoStage=False,
make='make',
jobs='%(jobs)s',
extra_configure_args=[],
env={},
*args,
**kwargs):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Find out what version of llvm and clang are needed to build this version
# of lldb. Right now we will assume they use the same version.
# XXX - could this be done directly on the master instead of the slave?
f.addStep(
SetProperty(
command=
'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2',
property='llvmrev'))
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout', '--force']
svn_co += ['--revision', WithProperties('%(llvmrev)s')]
# build llvm svn checkout command
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
llvm_srcdir]
# build clang svn checkout command
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
'%s/tools/clang' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
always_purge=True,
workdir='%s/tools/lldb' % llvm_srcdir))
# Run configure
config_args = [
WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
"--disable-bindings",
"--without-llvmgcc",
"--without-llvmgxx",
]
if triple:
config_args += ['--build=%s' % triple]
config_args += extra_configure_args
f.addStep(
Configure(name='configure',
command=config_args,
env=env,
workdir=llvm_objdir))
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
env=env,
haltOnFailure=True,
workdir=llvm_objdir))
# Test.
f.addStep(
LitTestCommand(name="test lldb",
command=['nice', '-n', '10', make],
description="test lldb",
env=env,
workdir='%s/tools/lldb/test' % llvm_objdir))
return f
def getLLDWinBuildFactory(clean=True):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
f = buildbot.process.factory.BuildFactory()
# Get LLVM and Lld
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='%s/tools/lld' % llvm_srcdir))
# Clean directory, if requested.
if clean:
f.addStep(
ShellCommand(name="rm-llvm_objdir",
command=[
"if", "exist", llvm_objdir, "rmdir", "/S", "/Q",
llvm_objdir
],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir="."))
f.addStep(
ShellCommand(
name="create-build-dir",
command=["if", "not", "exist", llvm_objdir, "mkdir", llvm_objdir],
haltOnFailure=True,
description=["create build dir"],
workdir="."))
# Is CMake configuration already done?
checkCMakeCommand = [
"dir", "CMakeCache.txt", ">", "NUL", "&&", "echo", "Yes", "||", "echo",
"No", ">", "NUL"
]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
SetProperty(
name="CMake_done",
workdir=llvm_objdir,
command=WithProperties(" ".join(checkCMakeCommand)),
#"cmd", "/C",
#" ".join(checkCMakeCommand)],
haltOnFailure=True,
description=["check CMake_done"],
property="CMake_done"))
# Create configuration files with cmake
cmakeCommand = [
"cmake", "-DCMAKE_BUILD_TYPE=Release", "-DLLVM_TARGETS_TO_BUILD=X86",
"../%s" % llvm_srcdir
]
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand)),
workdir=llvm_objdir,
doStepIf=lambda step: step.build.getProperty("CMake_done")
!= "Yes"))
# Build Lld
f.addStep(
ShellCommand(
name="build_Lld",
command=[
"msbuild",
#"/maxcpucount:1",
"/verbosity:minimal",
"/property:Configuration=Release",
"ALL_BUILD.vcxproj"
],
haltOnFailure=True,
description=["build lld"],
workdir=llvm_objdir))
# Test Lld
#f.addStep(ShellCommand(name="test_lld",
# command=["make", "lld-test"],
# haltOnFailure=True,
# description=["test lld"],
# workdir=llvm_objdir))
return f
