def getLLDBSource(f, llvmTopDir='llvm'):
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvmTopDir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvmTopDir))
f.addStep(
SVN(name='svn-lld',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='%s/tools/lld' % llvmTopDir))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
workdir='%s/tools/lldb' % llvmTopDir))
return f
def createLLVMFactory(srcdir, builddir, installdir, test_install_dir):
f = factory.BuildFactory()
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % srcdir))
f.addStep(
ShellCommand(command=[
'%s/configure' % srcdir, '--prefix=' + installdir,
'--enable-optimized', '--enable-targets=host', '--enable-shared'
],
workdir=builddir,
haltOnFailure=True,
name="configure",
descriptionDone='configure',
description='configuring'))
f.addStep(
ShellCommand(command=['make', '-j', '4'],
workdir=builddir,
haltOnFailure=True,
name="compile",
descriptionDone='compile',
description='compiling'))
f.addStep(
ShellCommand(command=['make', 'check'],
workdir=builddir,
name='check',
descriptionDone='check',
haltOnFailure=True,
description='checking'))
f.addStep(
ShellCommand(command=['make', 'install'],
env={'DESTDIR': test_install_dir},
workdir=builddir,
haltOnFailure=True,
name='install',
descriptionDone='install',
description='installing'))
f = createPoclFactory(llvm_dir=test_install_dir + installdir, f=f)
f.addStep(
ShellCommand(command=['make', 'install'],
workdir=builddir,
haltOnFailure=True,
name='install',
descriptionDone='install',
description='installing'))
return f
def __init__(self, svnurl=None, baseURL=None, defaultBranch=None,
directory=None, workdir=None, mode='update',alwaysUseLatest=True,timeout=20*60, retry=None,**kwargs):
LoggingBuildStep.__init__(self, **kwargs)
SVN.__init__(self, svnurl=svnurl, baseURL=baseURL, defaultBranch=defaultBranch,
directory=directory, workdir=workdir, mode=mode, alwaysUseLatest=alwaysUseLatest, timeout=timeout, retry=retry,**kwargs)
self.name = 'svn-update'
self.description = ["updating", "branch %s%s"%(baseURL,defaultBranch)]
self.descriptionDone = ["updated", "branch %s%s"%(baseURL,defaultBranch)]
def startVC(self, branch, revision, patch):
SVN.startVC(self, branch, revision, patch)
for d in [self.description, self.descriptionDone]:
try:
d.remove("[branch]")
d.append(branch)
except ValueError:
pass
def addSVNUpdateSteps(f,
checkout_clang_tools_extra,
checkout_compiler_rt,
checkout_test_suite,
checkout_lld,
checkout_libcxx):
# We *must* checkout at least Clang+LLVM
f.addStep(SVN(name='svn-llvm',
mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir='llvm'))
f.addStep(SVN(name='svn-clang',
mode='update', baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='llvm/tools/clang'))
# Extra stuff that will be built/tested
if checkout_clang_tools_extra:
f.addStep(SVN(name='svn-clang-tools-extra',
mode='update', baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='llvm/tools/clang/tools/extra'))
if checkout_compiler_rt:
f.addStep(SVN(name='svn-compiler-rt',
mode='update', baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='llvm/projects/compiler-rt'))
if checkout_test_suite:
f.addStep(SVN(name='svn-lnt',
mode='update', baseURL='http://llvm.org/svn/llvm-project/lnt/',
defaultBranch='trunk',
workdir='test/lnt'))
f.addStep(SVN(name='svn-test-suite',
mode='update', baseURL='https://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir='test/test-suite'))
if checkout_lld:
f.addStep(SVN(name='svn-lld',
mode='update', baseURL='http://llvm.org/svn/llvm-project/lld/',
defaultBranch='trunk',
workdir='llvm/tools/lld'))
if checkout_libcxx:
f.addStep(SVN(name='svn-libcxx',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir='llvm/projects/libcxx'))
f.addStep(SVN(name='svn-libcxxabi',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir='llvm/projects/libcxxabi'))
f.addStep(SVN(name='svn-libunwind',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/libunwind/',
defaultBranch='trunk',
workdir='llvm/projects/libunwind'))
def startVC(self, branch, revision, patch):
svnurl = self.baseURL + self.branch
if svnurl == branch:
pass
else:
revision= None
patch=None
branch = self.branch
SVN.startVC(self,self.branch, revision, patch)
def __init__(self, branch=None, **kwargs):
if branch is None or branch == "trunk":
svnurl = "http://code.djangoproject.com/svn/django/trunk"
else:
svnurl = "http://code.djangoproject.com/svn/django/branches/releases/%s" % branch
kwargs["svnurl"] = svnurl
kwargs["mode"] = "clobber"
SVN.__init__(self, **kwargs)
self.addFactoryArguments(branch=branch)
def __init__(self,
repository,
vcs_type=None,
branch=None,
username=None,
password=None):
factory.BuildFactory.__init__(self, [])
if not repository.endswith("/"):
repository += "/"
if not vcs_type:
if repository.startswith("https://svn."):
vcs_type = "svn"
elif repository.startswith("git://github.com/"):
vcs_type = "git"
if not branch:
branch = dict(svn="trunk", git="master")[vcs_type]
if vcs_type == "svn":
self.addStep(
SVN(
baseURL=repository,
defaultBranch=branch,
username=username,
password=password,
))
elif vcs_type == "git":
self.addStep(Git(
repourl=repository,
branch=branch,
))
def addClangGCCTests(f,
ignores={},
install_prefix="%(builddir)s/llvm.install",
languages=('gcc', 'g++', 'objc', 'obj-c++')):
make_vars = [
WithProperties('CC_UNDER_TEST=%s/bin/clang' % install_prefix),
WithProperties('CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)
]
f.addStep(
SVN(name='svn-clang-gcc-tests',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tests/',
defaultBranch='trunk',
workdir='clang-tests'))
gcc_dg_ignores = ignores.get('gcc-4_2-testsuite', {})
for lang in languages:
f.addStep(
commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand(
name='test-gcc-4_2-testsuite-%s' % lang,
command=["make", "-k", "check-%s" % lang] + make_vars,
description="gcc-4_2-testsuite (%s)" % lang,
workdir='clang-tests/gcc-4_2-testsuite',
logfiles={
'dg.sum': 'obj/%s/%s.sum' % (lang, lang),
'%s.log' % lang: 'obj/%s/%s.log' % (lang, lang)
},
ignore=gcc_dg_ignores.get(lang, [])))
def addModernClangGDBTests(f, jobs, install_prefix):
make_vars = [
WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' '
'CXX_FOR_TARGET=\'{0}/bin/clang++\' '
'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\''.format(
install_prefix))
]
f.addStep(
SVN(name='svn-clang-modern-gdb-tests',
mode='update',
svnurl=
'http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5',
workdir='clang-tests/src'))
f.addStep(
Configure(command='../src/configure', workdir='clang-tests/build/'))
f.addStep(
WarningCountingShellCommand(
name='gdb-75-build',
command=['make', WithProperties('-j%s' % jobs)],
haltOnFailure=True,
workdir='clang-tests/build'))
f.addStep(
commands.DejaGNUCommand.DejaGNUCommand(
name='gdb-75-check',
command=['make', '-k',
WithProperties('-j%s' % jobs), 'check'] + make_vars,
workdir='clang-tests/build',
logfiles={
'dg.sum': 'gdb/testsuite/gdb.sum',
'gdb.log': 'gdb/testsuite/gdb.log'
}))
def AddExternalPollyBuildFactory(f, llvm_installdir, build_type="Release"):
polly_srcdir = 'polly.src'
polly_objdir = 'polly.obj'
polly_installdir = 'polly.install'
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
f.addStep(
SVN(name='svn-polly',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/polly/',
defaultBranch='trunk',
workdir=polly_srcdir))
# Create configuration files with cmake
f.addStep(
ShellCommand(name="create-build-dir",
command=["mkdir", "-p", polly_objdir],
haltOnFailure=False,
description=["create build dir"],
workdir="."))
cmakeCommand = ["cmake", "../%s" % polly_srcdir]
cmakeCommand.append('-DCMAKE_PREFIX_PATH=../%s/lib/cmake/llvm' %
llvm_installdir)
cmakeCommand.append('-DCMAKE_BUILD_TYPE=' + build_type)
cmakeCommand.append('-DCMAKE_INSTALL_PREFIX=../' + polly_installdir)
cmakeCommand.append('-DCMAKE_COLOR_MAKEFILE=OFF')
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=polly_objdir))
# Build Polly
f.addStep(
ShellCommand(name="build-polly",
command=["make"],
haltOnFailure=True,
description=["build polly"],
workdir=polly_objdir))
f.addStep(
ShellCommand(name="remove-polly-install",
command=["rm", "-rf", polly_installdir],
haltOnFailure=True,
description=["remove polly install"],
workdir="."))
f.addStep(
ShellCommand(name="install-polly",
command=["make", "install"],
haltOnFailure=True,
description=["install polly"],
workdir=polly_objdir))
def initFactory(self,arch):
f = GNUAutoconf(SVN("http://svn.cri.ensmp.fr/svn/linear/trunk"),
test=None,
configureFlags=self.configure_flags)
if arch == "linux-64":
f.addStep(FileUpload(slavesrc=os.path.join("BUILD",self.targz),
masterdest=self.www+self.targz, mode=0644))
return f
def getAnnotatedBuildFactory(script,
clean=False,
depends_on_projects=None,
env=None,
timeout=1200):
"""
Returns a new build factory that uses AnnotatedCommand, which
allows the build to be run by version-controlled scripts that do
not require a buildmaster restart to update.
"""
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm.src')
if clean:
f.addStep(SetProperty(property='clean', command='echo 1'))
# We normally use the clean property to indicate that we want a
# clean build, but AnnotatedCommand uses the clobber property
# instead. Therefore, set clobber if clean is set to a truthy
# value. This will cause AnnotatedCommand to set
# BUILDBOT_CLOBBER=1 in the environment, which is how we
# communicate to the script that we need a clean build.
f.addStep(
SetProperty(
property='clobber',
command='echo 1',
doStepIf=lambda step: step.build.getProperty('clean', False)))
merged_env = {
'TERM': 'dumb' # Be cautious and disable color output from all tools.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set
# anything.
merged_env.update(env)
scripts_dir = "annotated"
f.addStep(
SVN(name='update-annotate-scripts',
mode='update',
svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
'zorg/buildbot/builders/annotated',
workdir=scripts_dir,
alwaysUseLatest=True))
# Explicitly use '/' as separator, because it works on *nix and Windows.
script_path = "../%s/%s" % (scripts_dir, script)
f.addStep(
AnnotatedCommand(name="annotate",
description="annotate",
timeout=timeout,
haltOnFailure=True,
command=WithProperties(
"python %(script)s --jobs=%(jobs:-)s",
script=lambda _: script_path),
env=merged_env))
return f
def addGetSourcecodeForProject(self,
project,
name=None,
src_dir=None,
**kwargs):
# Remove 'is_legacy_mode' if it leaked in to kwargs.
kwargs.pop('is_legacy_mode', None)
# Bail out if we are in the legacy mode and SVN checkout is required.
if self.is_legacy_mode:
workdir, baseURL = svn_repos[project]
if not name:
name = 'svn-%s' % project
# Check out to the given directory if any.
# Otherwise this is a part of the unified source tree.
if src_dir is None:
src_dir = workdir % {'llvm_srcdir': self.llvm_srcdir}
if not kwargs.get('mode', None):
kwargs['mode'] = 'update'
if not kwargs.get('defaultBranch', None):
kwargs['defaultBranch'] = 'trunk'
self.addStep(
SVN(name=name,
workdir=src_dir,
baseURL=WithProperties(baseURL),
**kwargs))
else:
# project contains a repo name which is not a part of the monorepo.
# We do not enforce it here, though.
_repourl = kwargs.pop('repourl', None)
if not _repourl:
_repourl = self.repourl_prefix + "llvm-%s.git" % project
if not name:
name = 'Checkout %s' % project
# Check out to the given directory if any.
# Otherwise this is a part of the unified source tree.
if src_dir is None:
src_dir = 'llvm-%s' % project
# Ignore workdir if given. We check out to src_dir.
kwargs.pop('workdir', None)
self.addStep(
Git(name=name,
repourl=_repourl,
progress=True,
workdir=WithProperties(src_dir),
**kwargs))
def __init__(self,
svnurl,
configure=None,
configureEnv={},
compile="make all",
test="make check"):
source = SVN(svnurl=svnurl, mode="update")
GNUAutoconf.__init__(self,
source,
configure=configure,
configureEnv=configureEnv,
compile=compile,
test=test)
def initFactory(self,arch):
self.make = "gmake" if arch == "bsd-64" else "make"
f = GNUAutoconf( SVN("http://svn.cri.ensmp.fr/svn/pips/trunk"),
configureFlags=self.configure_flags,
compile=[self.make, "all"],
test=None,
install=[self.make, "install"],
distcheck=None #[self.make, "distcheck"]
)
if arch == "linux-64":
f.addStep(FileUpload(slavesrc=os.path.join("BUILD",self.targz),
masterdest=self.www+self.targz, mode=0644))
return f
def addClangGDBTests(f, ignores={}, install_prefix="%(builddir)s/llvm.install"):
make_vars = [WithProperties(
'CC_UNDER_TEST=%s/bin/clang' % install_prefix),
WithProperties(
'CXX_UNDER_TEST=%s/bin/clang++' % install_prefix)]
f.addStep(SVN(name='svn-clang-gdb-tests', mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tests/',
defaultBranch='trunk', workdir='clang-tests'))
f.addStep(commands.SuppressionDejaGNUCommand.SuppressionDejaGNUCommand(
name='test-gdb-1472-testsuite',
command=["make", "-k", "check"] + make_vars,
description="gdb-1472-testsuite",
workdir='clang-tests/gdb-1472-testsuite',
logfiles={ 'dg.sum' : 'obj/filtered.gdb.sum',
'gdb.log' : 'obj/gdb.log' }))
def getSanitizerBuildFactory(
clean=False,
env=None,
timeout=1200):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
}
# Use env variables defined in the system.
merged_env.update(os.environ)
# Clobber bot if we need a clean build.
if clean:
merged_env["BUILDBOT_CLOBBER"] = "1"
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
sanitizer_buildbot = "sanitizer_buildbot"
sanitizer_script_dir = os.path.join(sanitizer_buildbot, "sanitizers")
# Get sanitizer buildbot scripts.
f.addStep(SVN(name='svn-sanitizer-buildbot',
mode='update',
svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
'zorg/buildbot/builders/sanitizers',
workdir=sanitizer_script_dir,
alwaysUseLatest=True))
sanitizer_script = os.path.join("..", sanitizer_script_dir, "buildbot_selector.py")
# Run annotated command for sanitizer.
f.addStep(AnnotatedCommand(name="annotate",
description="annotate",
timeout=timeout,
haltOnFailure=True,
command="python " + sanitizer_script,
env=merged_env))
return f
def getGlibcAnnotatedFactory(
clean=False,
env=None,
timeout=1200):
merged_env = {}
# Use env variables defined in the system.
merged_env.update(os.environ)
# Clobber bot if we need a clean build.
if clean:
merged_env['BUILDBOT_CLOBBER'] = '1'
# Overwrite pre-set items with the given ones, so user can set anything.
if env is not None:
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.',
env=merged_env))
# Get buildbot scripts.
f.addStep(SVN(name='update scripts',
mode='update',
svnurl='http://address-sanitizer.googlecode.com/svn/trunk/'
'glibc_buildbot/scripts',
workdir='scripts',
alwaysUseLatest=True))
selector_script = os.path.join('..', 'scripts', 'slave', 'buildbot_selector.py')
# Run annotated command.
f.addStep(AnnotatedCommand(name='annotate',
description='annotate',
timeout=timeout,
haltOnFailure=True,
command='python ' + selector_script,
env=merged_env))
return f
def addSVNSteps(self, llvm_srcdir=None, **kwargs):
if llvm_srcdir is None:
llvm_srcdir = self.llvm_srcdir
if not kwargs.get('mode', None):
kwargs['mode'] = 'update'
if not kwargs.get('defaultBranch', None):
kwargs['defaultBranch'] = 'trunk'
# Add a SVM step for each project this builder depends on.
# We want the projects be always checked out in a certain order.
for project in svn_repos.keys():
if project in self.depends_on_projects:
workdir, baseURL = svn_repos[project]
self.addStep(
SVN(name='svn-%s' % project,
workdir=workdir % {'llvm_srcdir' : llvm_srcdir},
baseURL=WithProperties(baseURL),
**kwargs))
def addRepository(self,
project=None,
repository=None,
vcs_type=None,
branch=None,
username=None,
password=None,
**kwargs):
kwargs = dict(kwargs)
if not repository.endswith("/"):
repository += "/"
if not vcs_type:
if repository.startswith("https://svn."):
vcs_type = "svn"
elif repository.startswith("git://github.com/"):
vcs_type = "git"
if not branch:
branch = dict(svn="trunk", git="master")[vcs_type]
if vcs_type == "svn":
kwargs.update(
dict(
baseURL=repository,
defaultBranch=branch,
username=username,
password=password,
codebase=project,
))
self.addStep(SVN(**kwargs))
elif vcs_type == "git":
kwargs.update(
dict(
repourl=repository,
branch=branch,
codebase=project,
))
self.addStep(Git(**kwargs))
def initFactory(self,arch):
if arch != "linux-64":
self.make="gmake"
else:
self.make="make"
f = factory.BuildFactory()
f.addStep(SVN("http://svn.cri.ensmp.fr/svn/validation/trunk"))
s=PipsValid(command=[self.make, 'validate',
# 'TARGET=TutorialPPoPP2010',
],
logfiles={'summary':'SUMMARY.short'},
env={
'PATH':'{0}:{1}'.format(os.environ['PATH'],os.path.join(self.install_prefix,"bin")),
'PYTHONPATH':'{0}'.format(os.path.join(self.install_prefix,"lib","python2.6","site-packages","pips")),
'LD_LIBRARY_PATH':'{0}'.format(os.path.join(self.install_prefix,"lib")),
'PIPS_F77':'gfortran',
'PIPS_F90':'gfortran',
})
f.addStep(s)
return f
def getPollyBuildFactory():
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
polly_srcdir = '%s/tools/polly' % llvm_srcdir
clang_srcdir = '%s/tools/clang' % llvm_srcdir
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Get LLVM, clang and Polly
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir=clang_srcdir))
f.addStep(
SVN(name='svn-polly',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/polly/',
defaultBranch='trunk',
workdir=polly_srcdir))
# Create configuration files with cmake
f.addStep(
ShellCommand(name="create-build-dir",
command=["mkdir", "-p", llvm_objdir],
haltOnFailure=False,
description=["create build dir"],
workdir="."))
cmakeCommand = [
"cmake",
"../%s" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF",
"-DPOLLY_TEST_DISABLE_BAR=ON", "-DCMAKE_BUILD_TYPE=Release"
]
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir))
# Build Polly
f.addStep(
ShellCommand(name="build_polly",
command=["make"],
haltOnFailure=True,
description=["build polly"],
workdir=llvm_objdir))
# Test Polly
f.addStep(
ShellCommand(name="test_polly",
command=["make", "polly-test"],
haltOnFailure=True,
description=["test polly"],
workdir=llvm_objdir))
# Check formatting
f.addStep(
ShellCommand(name="test_polly_format",
command=["make", "polly-check-format"],
haltOnFailure=False,
description=["Check formatting"],
workdir=llvm_objdir))
return f
def getLLDBxcodebuildFactory(use_cc=None,
build_type='Debug',
remote_configs=None,
env=None):
if env is None:
env = {}
f = buildbot.process.factory.BuildFactory()
f.addStep(
SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.'))
lldb_srcdir = 'lldb'
OBJROOT = '%(builddir)s/' + lldb_srcdir + '/build'
f.addStep(
SetProperty(name='get_bindir',
command=[
'echo',
WithProperties('%(builddir)s/' + lldb_srcdir +
'/build/' + build_type)
],
property='lldb_bindir',
description='set bin dir',
workdir='.'))
# cleaning out the build directory is vital for codesigning.
f.addStep(
ShellCommand(name='clean.lldb-buid',
command=['rm', '-rf',
WithProperties(OBJROOT)],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(name='clean.llvm-buid',
command=['rm', '-rf',
'%s/llvm-build' % lldb_srcdir],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(name='clean.test trace',
command='rm -rf build/*',
haltOnFailure=False,
flunkOnFailure=False,
workdir='.'))
# Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency
f.addStep(
ShellCommand(name='remove symbolic link lldb',
command=['rm', lldb_srcdir + '/llvm/tools/lldb'],
haltOnFailure=False,
flunkOnFailure=False,
workdir=WithProperties('%(builddir)s')))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
workdir=lldb_srcdir))
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir='%s/llvm' % lldb_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/llvm/tools/clang' % lldb_srcdir))
# setup keychain for codesign
# In order for the codesigning to work inside of buildbot, security must be
# called to unlock the keychain, which requires a password.
# I've set up a special keychain for this purpose, so as to not compromise
# the login password of the buildslave.
# This means I have to set the special keychain as the default and unlock it
# prior to building the sources.
f.addStep(
ShellCommand(name='check.keychain',
command=['security', 'default-keychain'],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
f.addStep(
ShellCommand(
name='find.certificate',
command=['security', 'find-certificate', '-c', 'lldb_codesign'],
haltOnFailure=True,
workdir=WithProperties('%(builddir)s')))
# Building the sources
#
buildcmd = ' '.join([
'xcrun', 'xcodebuild', '-target', 'desktop', '-configuration',
build_type, 'SYMROOT=' + OBJROOT, 'OBJROOT=' + OBJROOT
])
f.addStep(
ShellCommand(name='lldb-build',
command=WithProperties(buildcmd + " || " + buildcmd),
haltOnFailure=True,
workdir=lldb_srcdir))
# Testing
#
if not use_cc:
use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/'
use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang'
f.addStep(
SetProperty(name='set.cc',
command=['xcrun', '-find', 'clang'],
property='use_cc',
description='set cc',
workdir=lldb_srcdir))
else:
f.addStep(
SetProperty(name='set.cc',
command=['echo', use_cc],
property='use_cc',
description='set cc',
workdir=lldb_srcdir))
DOTEST_OPTS = ' '.join([
'--executable', '%(lldb_bindir)s/lldb', '--filecheck',
'%(lldb_bindir)s/FileCheck', '--framework',
'%(lldb_bindir)s/LLDB.framework', '-A', 'x86_64', '-C', 'clang', '-s',
'../../build/lldb-test-traces'
])
f.addStep(
LitTestCommand(
name='lldb-test',
command=['./dosep.py', '--options',
WithProperties(DOTEST_OPTS)],
haltOnFailure=False,
workdir='%s/test' % lldb_srcdir,
env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')}))
# Remote test steps
if remote_configs is not None:
# Source structure to use cmake command
f.addStep(
SetProperty(
name='get tblgen bindir',
command=[
'echo',
WithProperties(
'%(builddir)s/' + lldb_srcdir +
'/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin'
)
],
property='tblgen_bindir',
description='set tblgen binaries dir',
workdir='.'))
f = getSymbLinkSteps(f, lldb_srcdir)
for config in remote_configs:
f = getLLDBRemoteTestSteps(
f,
'%(lldb_bindir)s',
build_type,
config,
env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')})
# Compress and upload test log
f = archiveLLDBTestTraces(f, "build/lldb-test-traces*")
# Results go in a directory coded named according to the date and time of the test run, e.g.:
#
# 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log
#
# Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error. return f
return f
def getDragonEggNightlyTestBuildFactory(gcc='gcc',
gxx='g++',
llvm_configure_args=[],
testsuite_configure_args=[],
xfails=[],
clean=True,
env={},
jobs='%(jobs)s',
timeout=20):
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.',
env=env))
# Checkout LLVM sources.
llvm_src_dir = 'llvm.src'
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_src_dir,
env=env))
# Checkout DragonEgg sources.
dragonegg_src_dir = 'dragonegg.src'
f.addStep(
SVN(name='svn-dragonegg',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/dragonegg/',
defaultBranch='trunk',
workdir=dragonegg_src_dir,
env=env))
# Checkout the test-suite sources.
testsuite_src_dir = 'test-suite.src'
f.addStep(
SVN(name='svn-test-suite',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir=testsuite_src_dir,
env=env))
# Build and install LLVM.
llvm_obj_dir = 'llvm.obj'
llvm_install_dir = 'llvm.install'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % llvm_obj_dir,
command=['rm', '-rf', llvm_obj_dir, llvm_install_dir],
description='rm build dir llvm',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
Configure(
name='configure.llvm',
command=([
'../' + llvm_src_dir + '/configure',
WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir)
] + llvm_configure_args),
description='configuring llvm',
descriptionDone='configure llvm',
haltOnFailure=True,
workdir=llvm_obj_dir,
env=env))
f.addStep(
WarningCountingShellCommand(name='compile.llvm',
command=[
'nice', '-n', '10', 'make',
WithProperties('-j%s' % jobs)
],
haltOnFailure=True,
description='compiling llvm',
descriptionDone='compile llvm',
workdir=llvm_obj_dir,
env=env,
timeout=timeout * 60))
f.addStep(
WarningCountingShellCommand(
name='install.llvm',
command=['nice', '-n', '10', 'make', 'install'],
haltOnFailure=True,
description='installing llvm',
descriptionDone='install llvm',
workdir=llvm_obj_dir,
env=env))
# Build dragonegg with the just built LLVM.
dragonegg_obj_dir = 'dragonegg.obj'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % dragonegg_obj_dir,
command=['rm', '-rf', dragonegg_obj_dir],
description='rm build dir dragonegg',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
WarningCountingShellCommand(
name='compile.dragonegg',
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('-j%s' % jobs),
WithProperties('GCC=' + gcc),
WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir +
'/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir)
],
haltOnFailure=True,
description='compiling dragonegg',
descriptionDone='compile dragonegg',
workdir=dragonegg_obj_dir,
env=env,
timeout=timeout * 60))
# Pretend that DragonEgg is llvm-gcc by creating llvm-gcc and llvm-g++
# scripts that dispatch to gcc with dragonegg and g++ with dragonegg.
if clean:
f.addStep(
ShellCommand(name='rm-bin',
command=['rm', '-rf', 'bin'],
description='rm bin dir',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
ShellCommand(
name='create.llvm-gcc.script',
command=WithProperties('echo "#!/bin/sh" > llvm-gcc'
'; echo "exec ' + gcc +
' -fplugin=%(builddir)s/' +
dragonegg_obj_dir +
'/dragonegg.so \\"\$@\\"" >> llvm-gcc'
'; chmod a+x llvm-gcc'),
description='create llvm-gcc script',
haltOnFailure=True,
workdir='bin',
env=env))
f.addStep(
ShellCommand(
name='create.llvm-g++.script',
command=WithProperties('echo "#!/bin/sh" > llvm-g++'
'; echo "exec ' + gxx +
' -fplugin=%(builddir)s/' +
dragonegg_obj_dir +
'/dragonegg.so \\"\$@\\"" >> llvm-g++'
'; chmod a+x llvm-g++'),
description='create llvm-g++ script',
haltOnFailure=True,
workdir='bin',
env=env))
# Configure the test-suite.
testsuite_obj_dir = 'test-suite.obj'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % testsuite_obj_dir,
command=['rm', '-rf', testsuite_obj_dir],
description='rm test-suite build dir',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
Configure(
name='configure.test-suite',
command=[
'../' + testsuite_src_dir + '/configure',
WithProperties('--with-llvmsrc=%(builddir)s/' + llvm_src_dir),
WithProperties('--with-llvmobj=%(builddir)s/' + llvm_obj_dir),
WithProperties('--with-llvmgccdir=%(builddir)s/'),
'--with-llvmcc=llvm-gcc', 'CC=' + gcc, 'CXX=' + gxx
] + testsuite_configure_args,
description='configuring nightly test-suite',
descriptionDone='configure nightly test-suite',
haltOnFailure=True,
workdir=testsuite_obj_dir,
env=env))
# Build and test.
f.addStep(
ShellCommand(name='rm.test-suite.report',
command=[
'rm', '-rf', testsuite_obj_dir + '/report',
testsuite_obj_dir + '/report.nightly.raw.out',
testsuite_obj_dir + '/report.nightly.txt'
],
description='rm test-suite report',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
NightlyTestCommand(name='make.test-suite',
command=[
'make',
WithProperties('-j%s' % jobs),
'ENABLE_PARALLEL_REPORT=1', 'DISABLE_CBE=1',
'DISABLE_JIT=1',
'RUNTIMELIMIT=%s' % (timeout * 60),
'TEST=nightly', 'report'
],
logfiles={'report': 'report.nightly.txt'},
description='running nightly test-suite',
descriptionDone='run nightly test-suite',
haltOnFailure=True,
xfails=xfails,
timeout=timeout * 60,
workdir=testsuite_obj_dir,
env=env))
return f
def getLLDBBuildFactory(triple,
useTwoStage=False,
make='make',
jobs='%(jobs)s',
extra_configure_args=[],
env={},
*args,
**kwargs):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Find out what version of llvm and clang are needed to build this version
# of lldb. Right now we will assume they use the same version.
# XXX - could this be done directly on the master instead of the slave?
f.addStep(
SetProperty(
command=
'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2',
property='llvmrev'))
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout', '--force']
svn_co += ['--revision', WithProperties('%(llvmrev)s')]
# build llvm svn checkout command
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
llvm_srcdir]
# build clang svn checkout command
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
'%s/tools/clang' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
always_purge=True,
workdir='%s/tools/lldb' % llvm_srcdir))
# Run configure
config_args = [
WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
"--disable-bindings",
"--without-llvmgcc",
"--without-llvmgxx",
]
if triple:
config_args += ['--build=%s' % triple]
config_args += extra_configure_args
f.addStep(
Configure(name='configure',
command=config_args,
env=env,
workdir=llvm_objdir))
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
env=env,
haltOnFailure=True,
workdir=llvm_objdir))
# Test.
f.addStep(
LitTestCommand(name="test lldb",
command=['nice', '-n', '10', make],
description="test lldb",
env=env,
workdir='%s/tools/lldb/test' % llvm_objdir))
return f
def getToolchainBuildFactory(
clean=False,
test=True,
env=None, # Environmental variables for all steps.
extra_configure_args=None):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
"TERM" : "dumb", # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
src_dir = "llvm.src"
obj_dir = "llvm.obj"
install_dir = "llvm.install"
f = LLVMBuildFactory(
depends_on_projects=[
"llvm",
"clang",
"clang-tools-extra",
"compiler-rt",
"libcxx",
"libcxxabi",
"libunwind",
"lld"
])
# Get Fuchsia SDK.
sdk_dir = "fuchsia.sdk"
sdk_platform = {
"Linux": "linux-amd64",
"Darwin": "mac-amd64",
}[platform.system()]
sdk_version = "latest"
sdk_url = WithProperties(
"https://chrome-infra-packages.appspot.com/dl/"
"fuchsia/sdk/%(sdk_platform)s/+/%(sdk_version)s",
sdk_platform=lambda _: sdk_platform,
sdk_version=lambda _: sdk_version)
f.addStep(RemoveDirectory(name="clean-sdk",
dir=sdk_dir,
haltOnFailure=True))
f.addStep(ShellCommand(name="fetch-sdk",
command=["curl", "-SLf", "-o", "sdk.cipd", sdk_url],
description=["download", "fuchsia sdk"],
workdir=sdk_dir))
f.addStep(ShellCommand(name="extract-sdk",
command=["unzip", "-fo", "sdk.cipd"],
description=["extract", "fuchsia sdk"],
workdir=sdk_dir))
cleanCheckoutRequested = lambda step: step.build.getProperty("clean", default=False) or clean
# Clean up llvm sources.
f.addStep(RemoveDirectory(name="clean-llvm.src",
dir=src_dir,
haltOnFailure=True,
doStepIf=cleanCheckoutRequested))
# Get sources.
for project in f.depends_on_projects:
_, baseURL = svn_repos[project]
f.addStep(SVN(name="svn-%s" % project,
workdir=src_dir + "/" + project,
baseURL=baseURL))
cleanBuildRequested = lambda step: step.build.getProperty("clean", default=step.build.getProperty("clean_obj")) or clean
# Clean up llvm build.
f.addStep(RemoveDirectory(name="clean-llvm.obj",
dir=obj_dir,
haltOnFailure=True,
doStepIf=cleanBuildRequested))
# Configure.
if extra_configure_args is None:
cmake_options = []
else:
cmake_options = extra_configure_args[:]
# Some options are required for this stage no matter what.
CmakeCommand.applyRequiredOptions(cmake_options, [
("-G", "Ninja"),
("-DLLVM_ENABLE_PROJECTS=", "clang;clang-tools-extra;lld"),
("-DLLVM_ENABLE_RUNTIMES=", "compiler-rt;libcxx;libcxxabi;libunwind"),
])
# Set proper defaults.
CmakeCommand.applyDefaultOptions(cmake_options, [
("-DBOOTSTRAP_LLVM_ENABLE_LTO=", "OFF"),
("-DLLVM_ENABLE_LTO=", "OFF"),
])
cmake_options.append(
WithProperties(
"-DCMAKE_INSTALL_PREFIX=%(workdir)s/" + install_dir
))
cmake_options.append(
WithProperties(
"-DFUCHSIA_SDK=%(workdir)s/" + sdk_dir
))
CmakeCommand.applyRequiredOptions(cmake_options, [
("-C", "../" + src_dir + "/clang/cmake/caches/Fuchsia.cmake"),
])
f.addStep(CmakeCommand(name="cmake-configure",
options=cmake_options,
path='../' + src_dir + '/llvm',
haltOnFailure=True,
description=["configure"],
workdir=obj_dir,
env=merged_env,
doStepIf=FileDoesNotExist("CMakeCache.txt")))
# Build distribution.
f.addStep(NinjaCommand(name="ninja-build",
targets=["stage2-distribution"],
haltOnFailure=True,
description=["build"],
workdir=obj_dir,
env=merged_env))
# Test llvm, clang and lld.
f.addStep(NinjaCommand(name="check",
targets=["stage2-check-%s" % p for p in ("llvm", "clang", "lld")],
haltOnFailure=True,
description=["check"],
workdir=obj_dir,
env=merged_env,
doStepIf=test))
# Install distribution.
f.addStep(NinjaCommand(name="install",
targets=["stage2-install-distribution"],
haltOnFailure=True,
description=["install"],
workdir=obj_dir,
env=merged_env))
return f
def getDragonEggTestBuildFactory(gcc='gcc',
svn_testsuites=[],
llvm_configure_args=[],
xfails=[],
clean=True,
env={},
jobs='%(jobs)s',
timeout=20):
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.',
env=env))
# Checkout LLVM sources.
llvm_src_dir = 'llvm.src'
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_src_dir,
env=env))
# Checkout DragonEgg sources.
dragonegg_src_dir = 'dragonegg.src'
f.addStep(
SVN(name='svn-dragonegg',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/dragonegg/',
defaultBranch='trunk',
workdir=dragonegg_src_dir,
env=env))
# Checkout victims for the compilator.
compilator_dir = dragonegg_src_dir + '/test/compilator'
# Checkout testsuites held in subversion repositories. These are usually
# specified using a known good revision (supplied by appending @revision to
# the URL). The SVN step can't handle that. As it provides no mechanism at
# all for checking out a specific revision, run the command directly here.
for svn_testsuite in svn_testsuites:
url = svn_testsuite[0]
name = svn_testsuite[1]
if url is None:
f.addStep(
ShellCommand(name='rm-%s' % name,
command=['rm', '-rf', name],
description=['rm', name],
haltOnFailure=True,
workdir=compilator_dir,
env=env))
else:
svn_co = ['svn', 'checkout', url, name]
f.addStep(
ShellCommand(name=name,
command=svn_co,
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Checkout miscellaneous testsuites.
# LAPACK.
f.addStep(
ShellCommand(
name='wget.lapack',
command='wget -N http://www.netlib.org/lapack/lapack-3.4.0.tgz',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.lapack',
command='tar xzf lapack-3.4.0.tgz',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Nist Fortran 77 test suite.
f.addStep(
ShellCommand(
name='wget.nist',
command=
'wget -N http://www.itl.nist.gov/div897/ctg/suites/fcvs21.tar.Z',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.nist',
command='tar xzf ../fcvs21.tar.Z',
haltOnFailure=True,
workdir=compilator_dir + '/fcvs21/',
env=env))
# Polyhedron.
f.addStep(
ShellCommand(
name='wget.polyhedron',
command=
'wget -N http://www.polyhedron.com/web_images/documents/pb11.zip',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.polyhedron',
command='unzip -o pb11.zip',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Large single compilation-unit C programs.
f.addStep(
ShellCommand(
name='wget.bzip2',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/bzip2.c',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(
name='wget.gzip',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gzip.c',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='fix.gzip',
command='sed -i "s/^static char \*$/char */" gzip.c',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(
name='wget.oggenc',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/oggenc.c',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(
name='wget.gcc',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gcc.c.bz2',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.gcc',
command='bunzip2 -f -k gcc.c.bz2',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Build and install LLVM.
llvm_obj_dir = 'llvm.obj'
llvm_install_dir = 'llvm.install'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % llvm_obj_dir,
command=['rm', '-rf', llvm_obj_dir, llvm_install_dir],
description='rm build dir llvm',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
Configure(
name='configure.llvm',
command=([
'../' + llvm_src_dir + '/configure',
WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir)
] + llvm_configure_args),
description='configuring llvm',
descriptionDone='configure llvm',
haltOnFailure=True,
workdir=llvm_obj_dir,
env=env))
f.addStep(
WarningCountingShellCommand(name='compile.llvm',
command=[
'nice', '-n', '10', 'make',
WithProperties('-j%s' % jobs)
],
haltOnFailure=True,
description='compiling llvm',
descriptionDone='compile llvm',
workdir=llvm_obj_dir,
env=env,
timeout=timeout * 60))
f.addStep(
WarningCountingShellCommand(
name='install.llvm',
command=['nice', '-n', '10', 'make', 'install'],
haltOnFailure=True,
description='installing llvm',
descriptionDone='install llvm',
workdir=llvm_obj_dir,
env=env))
# Build dragonegg with the just built LLVM.
dragonegg_obj_dir = 'dragonegg.obj'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % dragonegg_obj_dir,
command=['rm', '-rf', dragonegg_obj_dir],
description='rm build dir dragonegg',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
WarningCountingShellCommand(
name='compile.dragonegg',
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('-j%s' % jobs),
WithProperties('GCC=' + gcc),
WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir +
'/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir)
],
haltOnFailure=True,
description='compiling dragonegg',
descriptionDone='compile dragonegg',
workdir=dragonegg_obj_dir,
env=env,
timeout=timeout * 60))
# Run the dragonegg testsuite.
testsuite_output_dir = dragonegg_obj_dir + '/test/Output'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % testsuite_output_dir,
command=['rm', '-rf', testsuite_output_dir],
description='rm test-suite output directory',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
LitTestCommand(
name='make.check',
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('GCC=' + gcc),
WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir +
'/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir),
WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check'
],
description='running test-suite',
descriptionDone='run test-suite',
haltOnFailure=True,
workdir=dragonegg_obj_dir,
env=env,
timeout=timeout * 60))
return f
def getClangBuildFactory(
triple=None,
clean=True,
test=True,
package_dst=None,
run_cxx_tests=False,
examples=False,
valgrind=False,
valgrindLeakCheck=False,
useTwoStage=False,
completely_clean=False,
make='make',
jobs="%(jobs)s",
stage1_config='Debug+Asserts',
stage2_config='Release+Asserts',
env={}, # Environmental variables for all steps.
extra_configure_args=[],
stage2_extra_configure_args=[],
use_pty_in_tests=False,
trunk_revision=None,
force_checkout=False,
extra_clean_step=None,
checkout_compiler_rt=False,
run_gdb=False,
run_modern_gdb=False,
run_gcc=False):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
llvm_srcdir = "llvm.src"
llvm_1_objdir = "llvm.obj"
llvm_1_installdir = "llvm.install.1"
llvm_2_objdir = "llvm.obj.2"
llvm_2_installdir = "llvm.install"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Blow away completely, if requested.
if completely_clean:
f.addStep(
ShellCommand(name="rm-llvm.src",
command=["rm", "-rf", llvm_srcdir],
haltOnFailure=True,
description=["rm src dir", "llvm"],
workdir=".",
env=merged_env))
# Checkout sources.
if trunk_revision:
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout']
if force_checkout:
svn_co += ['--force']
svn_co += ['--revision', WithProperties(trunk_revision)]
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' %
trunk_revision),
llvm_srcdir]
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' %
trunk_revision),
'%s/tools/clang' % llvm_srcdir]
svn_co_clang_tools_extra = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' %
trunk_revision),
'%s/tools/clang/tools/extra' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang-tools-extra',
command=svn_co_clang_tools_extra,
haltOnFailure=True,
workdir='.'))
else:
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if checkout_compiler_rt:
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
# Clean up llvm (stage 1); unless in-dir.
if clean and llvm_srcdir != llvm_1_objdir:
f.addStep(
ShellCommand(name="rm-llvm.obj.stage1",
command=["rm", "-rf", llvm_1_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
if not clean:
expected_makefile = 'Makefile'
f.addStep(
SetProperty(name="Makefile_isready",
workdir=llvm_1_objdir,
command=[
"sh", "-c",
"test -e %s && echo OK || echo Missing" %
expected_makefile
],
flunkOnFailure=False,
property="exists_Makefile"))
cmake_triple_arg = []
if triple:
cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple]
f.addStep(
ShellCommand(name='cmake',
command=[
'cmake', '-DLLVM_BUILD_TESTS=ON',
'-DCMAKE_BUILD_TYPE=%s' % stage1_config
] + cmake_triple_arg + extra_configure_args +
["../" + llvm_srcdir],
description='cmake stage1',
workdir=llvm_1_objdir,
env=merged_env,
doStepIf=lambda step: step.build.getProperty(
"exists_Makefile") != "OK"))
# Make clean if using in-dir builds.
if clean and llvm_srcdir == llvm_1_objdir:
f.addStep(
WarningCountingShellCommand(name="clean-llvm",
command=[make, "clean"],
haltOnFailure=True,
description="cleaning llvm",
descriptionDone="clean llvm",
workdir=llvm_1_objdir,
doStepIf=clean,
env=merged_env))
if extra_clean_step:
f.addStep(extra_clean_step)
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
haltOnFailure=True,
flunkOnFailure=not run_gdb,
description=["compiling", stage1_config],
descriptionDone=["compile", stage1_config],
workdir=llvm_1_objdir,
env=merged_env))
if examples:
f.addStep(
WarningCountingShellCommand(
name="compile.examples",
command=[
'nice', '-n', '10', make,
WithProperties("-j%s" % jobs), "BUILD_EXAMPLES=1"
],
haltOnFailure=True,
description=["compiling", stage1_config, "examples"],
descriptionDone=["compile", stage1_config, "examples"],
workdir=llvm_1_objdir,
env=merged_env))
clangTestArgs = '-v -j %s' % jobs
if valgrind:
clangTestArgs += ' --vg'
if valgrindLeakCheck:
clangTestArgs += ' --vg-leak'
clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp'
extraTestDirs = ''
if run_cxx_tests:
extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests'
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='check-all',
command=[
make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)
],
flunkOnFailure=not run_gdb,
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_1_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# TODO: Install llvm and clang for stage1.
if run_gdb or run_gcc or run_modern_gdb:
ignores = getClangTestsIgnoresFromPath(
os.path.expanduser('~/public/clang-tests'),
'clang-x86_64-darwin10')
install_prefix = "%%(builddir)s/%s" % llvm_1_installdir
if run_gdb:
addClangGDBTests(f, ignores, install_prefix)
if run_modern_gdb:
addModernClangGDBTests(f, jobs, install_prefix)
if run_gcc:
addClangGCCTests(f, ignores, install_prefix)
if not useTwoStage:
if package_dst:
name = WithProperties(
"%(builddir)s/" + llvm_1_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(
ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_1_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(
ShellCommand(name='pkg.upload',
description="upload root",
command=[
"scp", name,
WithProperties(package_dst +
"/%(buildername)s")
],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
# Clean up llvm (stage 2).
#
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we should
# not build in the first place.
f.addStep(
ShellCommand(name="rm-llvm.obj.stage2",
command=["rm", "-rf", llvm_2_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm", "(stage 2)"],
workdir=".",
env=merged_env))
# Configure llvm (stage 2).
f.addStep(
ShellCommand(
name='cmake',
command=['cmake'] + stage2_extra_configure_args + [
'-DLLVM_BUILD_TESTS=ON',
WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang'
% llvm_1_objdir), # FIXME use installdir
WithProperties(
'-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' %
llvm_1_objdir),
'-DCMAKE_BUILD_TYPE=%s' % stage2_config,
"../" + llvm_srcdir
],
description='cmake stage2',
workdir=llvm_2_objdir,
env=merged_env))
# Build llvm (stage 2).
f.addStep(
WarningCountingShellCommand(
name="compile.llvm.stage2",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["compiling", "(stage 2)", stage2_config],
descriptionDone=["compile", "(stage 2)", stage2_config],
workdir=llvm_2_objdir,
env=merged_env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='check-all',
command=[
make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)
],
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_2_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# TODO: Install llvm and clang for stage2.
if package_dst:
name = WithProperties("%(builddir)s/" + llvm_2_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(
ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_2_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(
ShellCommand(name='pkg.upload',
description="upload root",
command=[
"scp", name,
WithProperties(package_dst + "/%(buildername)s")
],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
def getClangMSVCBuildFactory(update=True,
clean=True,
vcDrive='c',
jobs=1,
cmake=r"cmake"):
f = buildbot.process.factory.BuildFactory()
if update:
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir='llvm'))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='llvm/tools/clang'))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='llvm/tools/clang/tools/extra'))
# Full & fast clean.
if clean:
f.addStep(
ShellCommand(name='clean-1',
command=['del', '/s/q', 'build'],
warnOnFailure=True,
description='cleaning',
descriptionDone='clean',
workdir='llvm'))
f.addStep(
ShellCommand(name='clean-2',
command=['rmdir', '/s/q', 'build'],
warnOnFailure=True,
description='cleaning',
descriptionDone='clean',
workdir='llvm'))
# Create the project files.
# Use batch files instead of ShellCommand directly, Windows quoting is
# borked. FIXME: See buildbot ticket #595 and buildbot ticket #377.
f.addStep(
batch_file_download.BatchFileDownload(
name='cmakegen',
command=[
cmake, "-DLLVM_TARGETS_TO_BUILD:=X86",
"-DLLVM_INCLUDE_EXAMPLES:=OFF", "-DLLVM_INCLUDE_TESTS:=OFF",
"-DLLVM_TARGETS_TO_BUILD:=X86", "-G", "Visual Studio 9 2008",
".."
],
workdir="llvm\\build"))
f.addStep(
ShellCommand(name='cmake',
command=['cmakegen.bat'],
haltOnFailure=True,
description='cmake gen',
workdir='llvm\\build'))
# Build it.
f.addStep(
batch_file_download.BatchFileDownload(
name='vcbuild',
command=[
vcDrive +
r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""",
"/M%d" % jobs, "LLVM.sln", "Debug|Win32"
],
workdir="llvm\\build"))
f.addStep(
WarningCountingShellCommand(name='vcbuild',
command=['vcbuild.bat'],
haltOnFailure=True,
description='vcbuild',
workdir='llvm\\build',
warningPattern=" warning C.*:"))
# Build clang-test project.
f.addStep(
batch_file_download.BatchFileDownload(
name='vcbuild_test',
command=[
vcDrive +
r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""",
"clang-test.vcproj", "Debug|Win32"
],
workdir="llvm\\build\\tools\\clang\\test"))
f.addStep(
lit_test_command.LitTestCommand(
name='test-clang',
command=["vcbuild_test.bat"],
workdir="llvm\\build\\tools\\clang\\test"))
return f
def getPollyBuildFactory(clean=False,
install=False,
make='make',
jobs=None,
checkAll=False,
env=None,
extraCmakeArgs=[]):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llvm_instdir = "llvm.inst"
polly_srcdir = '%s/tools/polly' % llvm_srcdir
clang_srcdir = '%s/tools/clang' % llvm_srcdir
jobs_cmd = []
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
build_cmd = [make] + jobs_cmd
install_cmd = [make, 'install'] + jobs_cmd
check_all_cmd = [make, 'check-all'] + jobs_cmd
check_polly_cmd = [make, 'check-polly'] + jobs_cmd
cmake_install = []
if install:
cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir]
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
merged_env.update(
env
) # Overwrite pre-set items with the given ones, so user can set anything.
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Get LLVM, Clang and Polly
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir=clang_srcdir))
f.addStep(
SVN(name='svn-polly',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/polly/',
defaultBranch='trunk',
workdir=polly_srcdir))
# Clean build dir
if clean:
f.addStep(
ShellCommand(name='clean-build-dir',
command=['rm', '-rf', llvm_objdir],
warnOnFailure=True,
description=["clean build dir"],
workdir='.',
env=merged_env))
# Create configuration files with cmake
f.addStep(
ShellCommand(name="create-build-dir",
command=["mkdir", "-p", llvm_objdir],
haltOnFailure=False,
description=["create build dir"],
workdir=".",
env=merged_env))
cmakeCommand = [
"cmake",
"../%s" % llvm_srcdir, "-DCMAKE_COLOR_MAKEFILE=OFF",
"-DPOLLY_TEST_DISABLE_BAR=ON", "-DPOLLY_ENABLE_GPGPU_CODEGEN=ON",
"-DCMAKE_BUILD_TYPE=Release"
] + cmake_install + extraCmakeArgs
f.addStep(
ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir,
env=merged_env))
# Build
f.addStep(
ShellCommand(name="build",
command=build_cmd,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
# Clean install dir
if install and clean:
f.addStep(
ShellCommand(name='clean-install-dir',
command=['rm', '-rf', llvm_instdir],
haltOnFailure=False,
description=["clean install dir"],
workdir='.',
env=merged_env))
# Install
if install:
f.addStep(
ShellCommand(name="install",
command=install_cmd,
haltOnFailure=False,
description=["install"],
workdir=llvm_objdir,
env=merged_env))
# Test
if checkAll:
f.addStep(
ShellCommand(name="check_all",
command=check_all_cmd,
haltOnFailure=False,
description=["check all"],
workdir=llvm_objdir,
env=merged_env))
else:
f.addStep(
ShellCommand(name="check_polly",
command=check_polly_cmd,
haltOnFailure=False,
description=["check polly"],
workdir=llvm_objdir,
env=merged_env))
return f
def getLibompCMakeBuildFactory(clean=True,
env=None,
test=True,
c_compiler="gcc",
cxx_compiler="g++"):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
openmp_srcdir = "openmp.src"
openmp_builddir = "openmp.build"
llvm_srcdir = "llvm.src"
llvm_builddir = "llvm.build"
f = buildbot.process.factory.BuildFactory()
# Get libomp
f.addStep(
SVN(name='svn-libomp',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/openmp/',
defaultBranch='trunk',
workdir=openmp_srcdir))
# Get llvm to build llvm-lit
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Clean directory, if requested.
if clean:
f.addStep(
ShellCommand(name="clean",
command=["rm", "-rf", openmp_builddir, llvm_builddir],
warnOnFailure=True,
description=["clean"],
workdir='.',
env=merged_env))
# CMake llvm
f.addStep(
ShellCommand(name='cmake llvm',
command=[
"cmake", "../" + llvm_srcdir,
"-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
],
haltOnFailure=True,
description='cmake llvm',
workdir=llvm_builddir,
env=merged_env))
# Make llvm utils
f.addStep(
WarningCountingShellCommand(name='make llvm utils build',
command=['make', 'LLVMX86Utils', '-j8'],
haltOnFailure=True,
description='make llvm utils build',
workdir=llvm_builddir,
env=merged_env))
# Add llvm-lit to PATH
merged_env.update({
'PATH':
WithProperties("${PATH}:" + "%(workdir)s/" + llvm_builddir + "/bin")
})
# CMake libomp
f.addStep(
ShellCommand(name='cmake libomp',
command=[
"cmake", "../" + openmp_srcdir,
"-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
],
haltOnFailure=True,
description='cmake libomp',
workdir=openmp_builddir,
env=merged_env))
# Make libomp
f.addStep(
WarningCountingShellCommand(name='make libomp build',
command=['make'],
haltOnFailure=True,
description='make libomp build',
workdir=openmp_builddir,
env=merged_env))
# Test, if requested
if test:
f.addStep(
lit_test_command.LitTestCommand(name='make check-libomp',
command=['make', 'check-libomp'],
haltOnFailure=True,
description=["make check-libomp"],
descriptionDone=["build checked"],
workdir=openmp_builddir,
env=merged_env))
return f
