def __init__(self, prefixCommand=None, options=None, path=None, **kwargs):
self.prefixCommand = prefixCommand
self.path = [path]
if options is None:
self.options = list()
else:
self.options = list(options)
command = []
if prefixCommand:
command += prefixCommand
command += ["cmake"]
if self.options:
command += self.options
if self.path:
command += self.path
# Remove here all the kwargs any of our LLVM buildbot command could consume.
# Note: We will remove all the empty items from the command at start, as we
# still didn't get yet WithProperties rendered.
sanitized_kwargs = self.sanitize_kwargs(kwargs)
sanitized_kwargs["command"] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **sanitized_kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand,
options=self.options,
path=path)
def __init__(self, prefixCommand=None, targets=None, **kwargs):
self.prefixCommand = prefixCommand
self.targets = targets
command = []
if prefixCommand:
command += prefixCommand
command += [
"ninja",
WithProperties("%(jobs:+-j)s"),
WithProperties("%(jobs:-)s"),
WithProperties("%(loadaverage:+-l)s"),
WithProperties("%(loadaverage:-)s")
]
if targets:
command += targets
# Don't forget to remove all the empty items from the command,
# which we could get because of WithProperties rendered as empty strings.
kwargs['command'] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand, targets=targets)
def __init__(self, prefixCommand=None, options=None, targets=None, **kwargs):
self.prefixCommand = prefixCommand
self.targets = targets
command = []
if prefixCommand:
command += prefixCommand
command += ["ninja"]
if options is None:
self.options = list()
else:
self.options = list(options)
j_opt = re.compile(r'^-j$|^-j\d+$')
l_opt = re.compile(r'^-l$|^-l\d+(\.(\d+)?)?$')
# We can get jobs in the options. If so, we would use that.
if not any(j_opt.search(opt) for opt in self.options if isinstance(opt, basestring)):
# Otherwise let's see if we got it in the kwargs.
if kwargs.get('jobs', None):
self.options += ["-j", kwargs['jobs']]
else:
# Use the property if option was not explicitly
# specified.
command += [
WithProperties("%(jobs:+-j)s"),
WithProperties("%(jobs:-)s"),
]
# The same logic is for hanling the loadaverage option.
if not any(l_opt.search(opt) for opt in self.options if isinstance(opt, basestring)):
if kwargs.get('loadaverage', None):
self.options += ["-l", kwargs['loadaverage']]
else:
command += [
WithProperties("%(loadaverage:+-l)s"),
WithProperties("%(loadaverage:-)s"),
]
if self.options:
command += self.options
if targets:
command += targets
# Remove here all the kwargs any of our LLVM buildbot command could consume.
# Note: We will remove all the empty items from the command at start, as we
# still didn't get yet WithProperties rendered.
sanitized_kwargs = self.sanitize_kwargs(kwargs)
sanitized_kwargs["command"] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **sanitized_kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand,
options=self.options,
targets=targets)
def setupEnvironment(self, cmd):
# First upcall to get everything prepared.
WarningCountingShellCommand.setupEnvironment(self, cmd)
# Set default status format string.
if cmd.args['env'] is None:
cmd.args['env'] = {}
cmd.args['env']['NINJA_STATUS'] = cmd.args['env'].get('NINJA_STATUS', "%e [%u/%r/%f] ")
def setupEnvironment(self, cmd):
# First upcall to get everything prepared.
WarningCountingShellCommand.setupEnvironment(self, cmd)
# Set default status format string.
if cmd.args['env'] is None:
cmd.args['env'] = {}
cmd.args['env']['NINJA_STATUS'] = cmd.args['env'].get(
'NINJA_STATUS', "%e [%u/%r/%f] ")
def __init__(self,
prefixCommand=None,
options=None,
targets=None,
**kwargs):
self.prefixCommand = prefixCommand
self.targets = targets
command = []
if prefixCommand:
command += prefixCommand
command += ["make"]
if options is None:
self.options = list()
else:
self.options = list(options)
j_opt = re.compile(r'^-j$|^-j\d+$')
# We can get jobs in the options. If so, we would use that.
if not any(
j_opt.search(opt)
for opt in self.options if isinstance(opt, basestring)):
# Otherwise let's see if we got it in the kwargs.
if kwargs.get('jobs', None):
self.options += ["-j", kwargs['jobs']]
else:
# Use the property if option was not explicitly
# specified.
command += [
WithProperties("%(jobs:+-j)s"),
WithProperties("%(jobs:-)s"),
]
if self.options:
command += self.options
if targets:
command += targets
# Remove here all the kwargs any of our LLVM buildbot command could consume.
# Note: We will remove all the empty items from the command at start, as we
# still didn't get yet WithProperties rendered.
sanitized_kwargs = self.sanitize_kwargs(kwargs)
sanitized_kwargs["command"] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **sanitized_kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand,
options=self.options,
targets=targets)
def getLLDBCmakeAndCompileSteps(f,
build_compiler,
build_type,
ninja_target,
bindir,
target_platform,
target_arch,
env=None):
if env is None:
env = {}
llvm_builddir = 'build'
if target_platform is 'android':
llvm_builddir = 'build/android-' + target_arch
# Configure
f = getLLDBCMakeStep(f, build_compiler, build_type, bindir,
target_platform, target_arch, env)
# Compile
f.addStep(
WarningCountingShellCommand(name='ninja-%s-%s' %
(target_platform, target_arch),
command=[
'nice', '-n', '10', 'ninja',
WithProperties('-j%(jobs)s')
] + ninja_target,
env=env,
haltOnFailure=True,
workdir=llvm_builddir))
return f
def addModernClangGDBTests(f, jobs, install_prefix):
make_vars = [
WithProperties('RUNTESTFLAGS=CC_FOR_TARGET=\'{0}/bin/clang\' '
'CXX_FOR_TARGET=\'{0}/bin/clang++\' '
'CFLAGS_FOR_TARGET=\'-w -fno-limit-debug-info\''.format(
install_prefix))
]
f.addStep(
SVN(name='svn-clang-modern-gdb-tests',
mode='update',
svnurl=
'http://llvm.org/svn/llvm-project/clang-tests-external/trunk/gdb/7.5',
workdir='clang-tests/src'))
f.addStep(
Configure(command='../src/configure', workdir='clang-tests/build/'))
f.addStep(
WarningCountingShellCommand(
name='gdb-75-build',
command=['make', WithProperties('-j%s' % jobs)],
haltOnFailure=True,
workdir='clang-tests/build'))
f.addStep(
commands.DejaGNUCommand.DejaGNUCommand(
name='gdb-75-check',
command=['make', '-k',
WithProperties('-j%s' % jobs), 'check'] + make_vars,
workdir='clang-tests/build',
logfiles={
'dg.sum': 'gdb/testsuite/gdb.sum',
'gdb.log': 'gdb/testsuite/gdb.log'
}))
def __init__(self, prefixCommand=None, options=None, path=None, **kwargs):
self.prefixCommand = prefixCommand
self.path = [path]
if options is None:
self.options = list()
else:
self.options = list(options)
command = []
if prefixCommand:
command += prefixCommand
command += ["cmake"]
# Set some default options.
CmakeCommand.applyDefaultOptions(self.options, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_ENABLE_WERROR=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if self.options:
command += self.options
if self.path:
command += self.path
# Remove here all the kwargs any of our LLVM buildbot command could consume.
# Note: We will remove all the empty items from the command at start, as we
# still didn't get yet WithProperties rendered.
sanitized_kwargs = self.sanitize_kwargs(kwargs)
sanitized_kwargs["command"] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **sanitized_kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand,
options=self.options,
path=path)
def __init__(self, prefixCommand=None, options=None, path=None, **kwargs):
self.prefixCommand = prefixCommand
self.path = [path]
if options is None:
self.options = list()
else:
self.options = list(options)
command = []
if prefixCommand:
command += prefixCommand
command += ["cmake"]
# Set some default options.
CmakeCommand.applyDefaultOptions(self.options, [
('-DCMAKE_BUILD_TYPE=', 'Release'),
('-DLLVM_ENABLE_WERROR=', 'ON'),
('-DLLVM_OPTIMIZED_TABLEGEN=', 'ON'),
])
if self.options:
command += self.options
if self.path:
command += self.path
# Remove here all the kwargs any of our LLVM buildbot command could consume.
# Note: We will remove all the empty items from the command at start, as we
# still didn't get yet WithProperties rendered.
sanitized_kwargs = self.sanitize_kwargs(kwargs)
sanitized_kwargs["command"] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **sanitized_kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand,
options=self.options,
path=path)
def testSuppressingLinelessWarningsPossible(self):
# Use a warningExtractor that does not provide line
# information
w = WarningCountingShellCommand(
warningExtractor=WarningCountingShellCommand.warnExtractWholeLine,
command="echo")
# Add suppression manually instead of using suppressionFile
fileRe = None
warnRe = ".*SUPPRESS.*"
start = None
end = None
suppression = (fileRe, warnRe, start, end)
w.addSuppression([suppression])
# Now call maybeAddWarning
warnings = []
line = "this warning should be SUPPRESSed"
match = re.match(".*warning.*", line)
w.maybeAddWarning(warnings, line, match)
# Finally make the suppressed warning was *not* added to the
# list of warnings
expectedWarnings = 0
self.assertEquals(len(warnings), expectedWarnings)
def __init__(self, prefixCommand=None, targets=None, **kwargs):
self.prefixCommand = prefixCommand
self.targets = targets
command = []
if prefixCommand:
command += prefixCommand
command += ["ninja",
WithProperties("%(jobs:+-j)s"), WithProperties("%(jobs:-)s"),
WithProperties("%(loadaverage:+-l)s"), WithProperties("%(loadaverage:-)s")]
if targets:
command += targets
# Don't forget to remove all the empty items from the command,
# which we could get because of WithProperties rendered as empty strings.
kwargs['command'] = command
# And upcall to let the base class do its work
WarningCountingShellCommand.__init__(self, **kwargs)
self.addFactoryArguments(prefixCommand=prefixCommand,
targets=targets)
def start(self):
# Don't forget to remove all the empty items from the command,
# which we could get because of WithProperties rendered as empty strings.
self.command = filter(bool, self.command)
# Then upcall.
WarningCountingShellCommand.start(self)
def start(self):
# Don't forget to remove all the empty items from the command,
# which we could get because of WithProperties rendered as empty strings.
self.command = filter(bool, self.command)
# Then upcall.
WarningCountingShellCommand.start(self)
def getLibompCMakeBuildFactory(clean=True,
env=None,
test=True,
c_compiler="gcc",
cxx_compiler="g++"):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
openmp_srcdir = "openmp.src"
openmp_builddir = "openmp.build"
llvm_srcdir = "llvm.src"
llvm_builddir = "llvm.build"
f = buildbot.process.factory.BuildFactory()
# Get libomp
f.addStep(
SVN(name='svn-libomp',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/openmp/',
defaultBranch='trunk',
workdir=openmp_srcdir))
# Get llvm to build llvm-lit
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Clean directory, if requested.
if clean:
f.addStep(
ShellCommand(name="clean",
command=["rm", "-rf", openmp_builddir, llvm_builddir],
warnOnFailure=True,
description=["clean"],
workdir='.',
env=merged_env))
# CMake llvm
f.addStep(
ShellCommand(name='cmake llvm',
command=[
"cmake", "../" + llvm_srcdir,
"-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
],
haltOnFailure=True,
description='cmake llvm',
workdir=llvm_builddir,
env=merged_env))
# Make llvm utils
f.addStep(
WarningCountingShellCommand(name='make llvm utils build',
command=['make', 'LLVMX86Utils', '-j8'],
haltOnFailure=True,
description='make llvm utils build',
workdir=llvm_builddir,
env=merged_env))
# Add llvm-lit to PATH
merged_env.update({
'PATH':
WithProperties("${PATH}:" + "%(workdir)s/" + llvm_builddir + "/bin")
})
# CMake libomp
f.addStep(
ShellCommand(name='cmake libomp',
command=[
"cmake", "../" + openmp_srcdir,
"-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
],
haltOnFailure=True,
description='cmake libomp',
workdir=openmp_builddir,
env=merged_env))
# Make libomp
f.addStep(
WarningCountingShellCommand(name='make libomp build',
command=['make'],
haltOnFailure=True,
description='make libomp build',
workdir=openmp_builddir,
env=merged_env))
# Test, if requested
if test:
f.addStep(
lit_test_command.LitTestCommand(name='make check-libomp',
command=['make', 'check-libomp'],
haltOnFailure=True,
description=["make check-libomp"],
descriptionDone=["build checked"],
workdir=openmp_builddir,
env=merged_env))
return f
def startBuild(self, dummy):
return WarningCountingShellCommand.start(self)
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=[],
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env={},
extra_cmake_args=[],
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Triggers
trigger_after_stage1=[]):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
addSVNUpdateSteps(f,
checkout_clang_tools_extra=checkout_clang_tools_extra,
checkout_compiler_rt=checkout_compiler_rt,
checkout_test_suite=runTestSuite)
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
check_build_cmd = [
"sh", "-c", "test -e build.ninja && echo OK || echo Missing"
]
if vs:
check_build_cmd = [
"cmd", "/c",
"if exist build.ninja (echo OK) " + " else (echo Missing & exit 1)"
]
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs:
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
if clean:
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
env=env))
else:
f.addStep(
SetProperty(name="check ninja files 1",
workdir=stage1_build,
command=check_build_cmd,
flunkOnFailure=False,
property="exists_ninja_1"))
############# STAGE 1
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=lambda step: step.build.getProperty("exists_ninja_1"
) != "OK",
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url')
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.',
env=env))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
cmake_cmd2 = [
'env',
WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
cxx), cmake, "-G", "Ninja", "../llvm",
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
doStepIf=lambda step: step.build.getProperty(
"exists_ninja_2") != "OK",
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags to get the same effect)
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
if testerName:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
commands.LitTestCommand.LitTestCommand(
name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def getClangMSVCBuildFactory(update=True,
clean=True,
vcDrive='c',
jobs=1,
cmake=r"cmake"):
f = buildbot.process.factory.BuildFactory()
if update:
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir='llvm'))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='llvm/tools/clang'))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='llvm/tools/clang/tools/extra'))
# Full & fast clean.
if clean:
f.addStep(
ShellCommand(name='clean-1',
command=['del', '/s/q', 'build'],
warnOnFailure=True,
description='cleaning',
descriptionDone='clean',
workdir='llvm'))
f.addStep(
ShellCommand(name='clean-2',
command=['rmdir', '/s/q', 'build'],
warnOnFailure=True,
description='cleaning',
descriptionDone='clean',
workdir='llvm'))
# Create the project files.
# Use batch files instead of ShellCommand directly, Windows quoting is
# borked. FIXME: See buildbot ticket #595 and buildbot ticket #377.
f.addStep(
batch_file_download.BatchFileDownload(
name='cmakegen',
command=[
cmake, "-DLLVM_TARGETS_TO_BUILD:=X86",
"-DLLVM_INCLUDE_EXAMPLES:=OFF", "-DLLVM_INCLUDE_TESTS:=OFF",
"-DLLVM_TARGETS_TO_BUILD:=X86", "-G", "Visual Studio 9 2008",
".."
],
workdir="llvm\\build"))
f.addStep(
ShellCommand(name='cmake',
command=['cmakegen.bat'],
haltOnFailure=True,
description='cmake gen',
workdir='llvm\\build'))
# Build it.
f.addStep(
batch_file_download.BatchFileDownload(
name='vcbuild',
command=[
vcDrive +
r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""",
"/M%d" % jobs, "LLVM.sln", "Debug|Win32"
],
workdir="llvm\\build"))
f.addStep(
WarningCountingShellCommand(name='vcbuild',
command=['vcbuild.bat'],
haltOnFailure=True,
description='vcbuild',
workdir='llvm\\build',
warningPattern=" warning C.*:"))
# Build clang-test project.
f.addStep(
batch_file_download.BatchFileDownload(
name='vcbuild_test',
command=[
vcDrive +
r""":\Program Files\Microsoft Visual Studio 9.0\VC\VCPackages\vcbuild.exe""",
"clang-test.vcproj", "Debug|Win32"
],
workdir="llvm\\build\\tools\\clang\\test"))
f.addStep(
lit_test_command.LitTestCommand(
name='test-clang',
command=["vcbuild_test.bat"],
workdir="llvm\\build\\tools\\clang\\test"))
return f
def getPollyBuildFactory(
clean=False,
install=False,
make='make',
jobs=None,
checkAll=False,
env=None,
extraCmakeArgs=None,
testsuite=False,extraTestsuiteCmakeArgs=None,
**kwargs):
if extraCmakeArgs is None:
extraCmakeArgs=[]
if extraTestsuiteCmakeArgs is None:
extraTestsuiteCmakeArgs = []
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llvm_instdir = "llvm.inst"
testsuite_srcdir = "test-suite.src"
testsuite_builddir = "test-suite.build"
jobs_cmd = []
if jobs is not None:
jobs_cmd = ["-j{}".format(jobs)]
build_cmd = [make] + jobs_cmd
install_cmd = [make, 'install'] + jobs_cmd
check_all_cmd = [make, 'check-all'] + jobs_cmd
check_polly_cmd = [make, 'check-polly'] + jobs_cmd
cmake_install = []
if install:
cmake_install = ["-DCMAKE_INSTALL_PREFIX=../%s" % llvm_instdir]
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env:
merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything.
depends_on_projects = ['llvm','clang','polly']
if testsuite:
# XRay tests in test-suite require compiler-rt
depends_on_projects += ['compiler-rt']
cleanBuildRequestedByProperty = lambda step: step.build.getProperty("clean", False)
cleanBuildRequested = lambda step: clean or step.build.getProperty("clean", default=step.build.getProperty("clean_obj"))
f = LLVMBuildFactory(
depends_on_projects=depends_on_projects,
llvm_srcdir=llvm_srcdir,
obj_dir=llvm_objdir,
install_dir=llvm_instdir,
cleanBuildRequested=cleanBuildRequested,
**kwargs) # Pass through all the extra arguments.
f.addStep(steps.RemoveDirectory(name='clean-src-dir',
dir=f.monorepo_dir,
warnOnFailure=True,
doStepIf=cleanBuildRequestedByProperty))
# Get the source code.
f.addGetSourcecodeSteps(**kwargs)
# Clean build dir
f.addStep(steps.RemoveDirectory(name='clean-build-dir',
dir=llvm_objdir,
warnOnFailure=True,
doStepIf=cleanBuildRequested))
# Create configuration files with cmake
cmakeCommand = ["cmake", "../%s/llvm" % llvm_srcdir,
"-DCMAKE_COLOR_MAKEFILE=OFF",
"-DPOLLY_TEST_DISABLE_BAR=ON",
"-DPOLLY_ENABLE_GPGPU_CODEGEN=ON",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_POLLY_LINK_INTO_TOOLS=ON",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
] + cmake_install + extraCmakeArgs
f.addStep(ShellCommand(name="cmake-configure",
command=cmakeCommand,
haltOnFailure=False,
description=["cmake configure"],
workdir=llvm_objdir,
env=merged_env))
# Build
f.addStep(WarningCountingShellCommand(name="build",
command=build_cmd,
haltOnFailure=True,
description=["build"],
workdir=llvm_objdir,
env=merged_env))
clangexe = "%(builddir)s/" + llvm_objdir + "/bin/clang"
clangxxexe = "%(builddir)s/" + llvm_objdir + "/bin/clang++"
litexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-lit"
sizeexe = "%(builddir)s/" + llvm_objdir + "/bin/llvm-size"
# Clean install dir
if install:
f.addStep(steps.RemoveDirectory(name='clean-install-dir',
dir=llvm_instdir,
haltOnFailure=False,
doStepIf=cleanBuildRequested))
f.addStep(ShellCommand(name="install",
command=install_cmd,
haltOnFailure=True,
description=["install"],
workdir=llvm_objdir,
env=merged_env))
# If installing, use the installed version of clang.
clangexe = "%(builddir)s/" + llvm_instdir + "/bin/clang"
clangxxexe = "%(builddir)s/" + llvm_instdir + "/bin/clang++"
sizeexe = "%(builddir)s/" + llvm_instdir + "/bin/llvm-size"
# Test
if checkAll:
f.addStep(LitTestCommand(name="check_all",
command=check_all_cmd,
haltOnFailure=False,
description=["check all"],
workdir=llvm_objdir,
env=merged_env))
else:
f.addStep(LitTestCommand(name="check_polly",
command=check_polly_cmd,
haltOnFailure=False,
description=["check polly"],
workdir=llvm_objdir,
env=merged_env))
if testsuite:
f.addStep(steps.RemoveDirectory(name='test-suite_clean-src-dir',
dir=testsuite_srcdir,
haltOnFailure=False,
warnOnFailure=True,
doStepIf=cleanBuildRequestedByProperty))
f.addGetSourcecodeForProject(
project='test-suite',
src_dir=testsuite_srcdir,
alwaysUseLatest=True)
f.addStep(steps.RemoveDirectory(name='test-suite_clean-build-dir',
dir=testsuite_builddir,
haltOnFailure=False,
warnOnFailure=True))
# -Wno-unused-command-line-argument is needed because linking will not uses the "-mllvm -polly" argument.
f.addStep(ShellCommand(name='test-suite_cmake-configure',
description=["Test-Suite: cmake"],
command=["cmake", '-B', testsuite_builddir, '-S', testsuite_srcdir,
"-DCMAKE_BUILD_TYPE=Release",
"-DTEST_SUITE_COLLECT_STATS=ON",
"-DTEST_SUITE_EXTRA_C_FLAGS=-Wno-unused-command-line-argument -mllvm -polly",
"-DTEST_SUITE_EXTRA_CXX_FLAGS=-mllvm -polly",
"-DTEST_SUITE_LIT_FLAGS=-vv;-o;report.json",
WithProperties("-DCMAKE_C_COMPILER=" + clangexe),
WithProperties("-DCMAKE_CXX_COMPILER=" + clangxxexe),
WithProperties("-DTEST_SUITE_LLVM_SIZE=" + sizeexe),
WithProperties("-DTEST_SUITE_LIT=" + litexe),
] + extraTestsuiteCmakeArgs,
haltOnFailure=True,
workdir='.',
env=merged_env))
f.addStep(WarningCountingShellCommand(name='test-suite_build',
description=["Test-Suite: build"],
# Continue building; programs that don't compile will fail with NOEXE.
command=[make, 'all', '-k0'] + jobs_cmd,
haltOnFailure=False,
flunkOnFailure=True,
workdir=testsuite_builddir,
env=merged_env))
f.addStep(LitTestCommand(name='test-suite_run',
description=['Test-Suite: run'],
command=[WithProperties(litexe), '-vv', '-o', 'report.json', '.'],
haltOnFailure=True,
workdir=testsuite_builddir,
logfiles={
'test.log' : 'test.log',
'report.json': 'report.json'
},
env=merged_env))
return f
def getClangBuildFactory(
triple=None,
clean=True,
test=True,
package_dst=None,
run_cxx_tests=False,
examples=False,
valgrind=False,
valgrindLeakCheck=False,
useTwoStage=False,
completely_clean=False,
make='make',
jobs="%(jobs)s",
stage1_config='Debug+Asserts',
stage2_config='Release+Asserts',
env={}, # Environmental variables for all steps.
extra_configure_args=[],
stage2_extra_configure_args=[],
use_pty_in_tests=False,
trunk_revision=None,
force_checkout=False,
extra_clean_step=None,
checkout_compiler_rt=False,
run_gdb=False,
run_modern_gdb=False,
run_gcc=False):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
llvm_srcdir = "llvm.src"
llvm_1_objdir = "llvm.obj"
llvm_1_installdir = "llvm.install.1"
llvm_2_objdir = "llvm.obj.2"
llvm_2_installdir = "llvm.install"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Blow away completely, if requested.
if completely_clean:
f.addStep(
ShellCommand(name="rm-llvm.src",
command=["rm", "-rf", llvm_srcdir],
haltOnFailure=True,
description=["rm src dir", "llvm"],
workdir=".",
env=merged_env))
# Checkout sources.
if trunk_revision:
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout']
if force_checkout:
svn_co += ['--force']
svn_co += ['--revision', WithProperties(trunk_revision)]
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%s' %
trunk_revision),
llvm_srcdir]
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%s' %
trunk_revision),
'%s/tools/clang' % llvm_srcdir]
svn_co_clang_tools_extra = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/clang-tools-extra/trunk@%s' %
trunk_revision),
'%s/tools/clang/tools/extra' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang-tools-extra',
command=svn_co_clang_tools_extra,
haltOnFailure=True,
workdir='.'))
else:
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(
SVN(name='svn-clang-tools-extra',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/clang-tools-extra/',
defaultBranch='trunk',
workdir='%s/tools/clang/tools/extra' % llvm_srcdir))
if checkout_compiler_rt:
f.addStep(
SVN(name='svn-compiler-rt',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
# Clean up llvm (stage 1); unless in-dir.
if clean and llvm_srcdir != llvm_1_objdir:
f.addStep(
ShellCommand(name="rm-llvm.obj.stage1",
command=["rm", "-rf", llvm_1_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
if not clean:
expected_makefile = 'Makefile'
f.addStep(
SetProperty(name="Makefile_isready",
workdir=llvm_1_objdir,
command=[
"sh", "-c",
"test -e %s && echo OK || echo Missing" %
expected_makefile
],
flunkOnFailure=False,
property="exists_Makefile"))
cmake_triple_arg = []
if triple:
cmake_triple_arg = ['-DLLVM_HOST_TRIPLE=%s' % triple]
f.addStep(
ShellCommand(name='cmake',
command=[
'cmake', '-DLLVM_BUILD_TESTS=ON',
'-DCMAKE_BUILD_TYPE=%s' % stage1_config
] + cmake_triple_arg + extra_configure_args +
["../" + llvm_srcdir],
description='cmake stage1',
workdir=llvm_1_objdir,
env=merged_env,
doStepIf=lambda step: step.build.getProperty(
"exists_Makefile") != "OK"))
# Make clean if using in-dir builds.
if clean and llvm_srcdir == llvm_1_objdir:
f.addStep(
WarningCountingShellCommand(name="clean-llvm",
command=[make, "clean"],
haltOnFailure=True,
description="cleaning llvm",
descriptionDone="clean llvm",
workdir=llvm_1_objdir,
doStepIf=clean,
env=merged_env))
if extra_clean_step:
f.addStep(extra_clean_step)
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
haltOnFailure=True,
flunkOnFailure=not run_gdb,
description=["compiling", stage1_config],
descriptionDone=["compile", stage1_config],
workdir=llvm_1_objdir,
env=merged_env))
if examples:
f.addStep(
WarningCountingShellCommand(
name="compile.examples",
command=[
'nice', '-n', '10', make,
WithProperties("-j%s" % jobs), "BUILD_EXAMPLES=1"
],
haltOnFailure=True,
description=["compiling", stage1_config, "examples"],
descriptionDone=["compile", stage1_config, "examples"],
workdir=llvm_1_objdir,
env=merged_env))
clangTestArgs = '-v -j %s' % jobs
if valgrind:
clangTestArgs += ' --vg'
if valgrindLeakCheck:
clangTestArgs += ' --vg-leak'
clangTestArgs += ' --vg-arg --suppressions=%(builddir)s/llvm/tools/clang/utils/valgrind/x86_64-pc-linux-gnu_gcc-4.3.3.supp --vg-arg --suppressions=%(builddir)s/llvm/utils/valgrind/x86_64-pc-linux-gnu.supp'
extraTestDirs = ''
if run_cxx_tests:
extraTestDirs += '%(builddir)s/llvm/tools/clang/utils/C++Tests'
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='check-all',
command=[
make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)
],
flunkOnFailure=not run_gdb,
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_1_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# TODO: Install llvm and clang for stage1.
if run_gdb or run_gcc or run_modern_gdb:
ignores = getClangTestsIgnoresFromPath(
os.path.expanduser('~/public/clang-tests'),
'clang-x86_64-darwin10')
install_prefix = "%%(builddir)s/%s" % llvm_1_installdir
if run_gdb:
addClangGDBTests(f, ignores, install_prefix)
if run_modern_gdb:
addModernClangGDBTests(f, jobs, install_prefix)
if run_gcc:
addClangGCCTests(f, ignores, install_prefix)
if not useTwoStage:
if package_dst:
name = WithProperties(
"%(builddir)s/" + llvm_1_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(
ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_1_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(
ShellCommand(name='pkg.upload',
description="upload root",
command=[
"scp", name,
WithProperties(package_dst +
"/%(buildername)s")
],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
# Clean up llvm (stage 2).
#
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we should
# not build in the first place.
f.addStep(
ShellCommand(name="rm-llvm.obj.stage2",
command=["rm", "-rf", llvm_2_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm", "(stage 2)"],
workdir=".",
env=merged_env))
# Configure llvm (stage 2).
f.addStep(
ShellCommand(
name='cmake',
command=['cmake'] + stage2_extra_configure_args + [
'-DLLVM_BUILD_TESTS=ON',
WithProperties('-DCMAKE_C_COMPILER=%%(builddir)s/%s/bin/clang'
% llvm_1_objdir), # FIXME use installdir
WithProperties(
'-DCMAKE_CXX_COMPILER=%%(builddir)s/%s/bin/clang++' %
llvm_1_objdir),
'-DCMAKE_BUILD_TYPE=%s' % stage2_config,
"../" + llvm_srcdir
],
description='cmake stage2',
workdir=llvm_2_objdir,
env=merged_env))
# Build llvm (stage 2).
f.addStep(
WarningCountingShellCommand(
name="compile.llvm.stage2",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["compiling", "(stage 2)", stage2_config],
descriptionDone=["compile", "(stage 2)", stage2_config],
workdir=llvm_2_objdir,
env=merged_env))
if test:
f.addStep(
lit_test_command.LitTestCommand(
name='check-all',
command=[
make, "check-all", "VERBOSE=1",
WithProperties("LIT_ARGS=%s" % clangTestArgs),
WithProperties("EXTRA_TESTDIRS=%s" % extraTestDirs)
],
description=["checking"],
descriptionDone=["checked"],
workdir=llvm_2_objdir,
usePTY=use_pty_in_tests,
env=merged_env))
# TODO: Install llvm and clang for stage2.
if package_dst:
name = WithProperties("%(builddir)s/" + llvm_2_objdir +
"/clang-r%(got_revision)s-b%(buildnumber)s.tgz")
f.addStep(
ShellCommand(name='pkg.tar',
description="tar root",
command=["tar", "zcvf", name, "./"],
workdir=llvm_2_installdir,
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
f.addStep(
ShellCommand(name='pkg.upload',
description="upload root",
command=[
"scp", name,
WithProperties(package_dst + "/%(buildername)s")
],
workdir=".",
warnOnFailure=True,
flunkOnFailure=False,
haltOnFailure=False,
env=merged_env))
return f
def startBuild(self, cmd):
if cmd.rc != 0:
log.msg("Failure when running %s." % cmd)
self.finished(FAILURE)
else:
return WarningCountingShellCommand.start(self)
def get3StageClangLTOBuildFactory(
clean=True,
env=None,
build_gold=False,
cmake_cache_file=None,
extra_cmake_options=None,
):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm-stage1.obj"
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
merged_env.update(env)
f = BuildFactory()
f.addStep(
SVN(
name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir
)
)
f.addStep(
SVN(
name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir
)
)
# Clean directory, if requested.
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
ShellCommand(
doStepIf=cleanBuildRequested,
name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env
)
)
cmake_command = ["cmake"]
if cmake_cache_file:
cmake_command += ['-C', cmake_cache_file]
if extra_cmake_options:
cmake_command += extra_cmake_options
cmake_command += [
"../%s" % llvm_srcdir
]
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
WarningCountingShellCommand(
name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
command=WithProperties(" ".join(cmake_command)),
workdir=llvm_objdir,
env=merged_env
)
)
if build_gold:
f.addStep(
NinjaCommand(name='build-LLVMgold.so',
targets=['lib/LLVMgold.so'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
workdir=llvm_objdir,
env=merged_env)
)
f.addStep(
NinjaCommand(name='build-stage3-clang',
targets=['stage3-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["3 Stage Build Clang"],
timeout=3600, # Default value is not enough.
workdir=llvm_objdir,
env=merged_env)
)
f.addStep(
NinjaCommand(name='build-stage3-check-clang',
targets=['stage3-check-clang'],
haltOnFailure=True,
warnOnWarnings=True,
description=["Check Clang"],
timeout=3600, # Default value is not enough.
workdir=llvm_objdir,
env=merged_env)
)
# Compare stage2 & stage3 clang
shell_command = [
"diff",
"-q",
"tools/clang/stage2-bins/bin/clang",
"tools/clang/stage2-bins/tools/clang/stage3-bins/bin/clang"
]
f.addStep(
ShellCommand(
name="compare-clang",
description=["comapre stage2 & stage3 clang"],
haltOnFailure=True,
command=WithProperties(" ".join(shell_command)),
workdir=llvm_objdir,
env=merged_env
)
)
return f
def getSanitizerBuildFactoryII(
clean=False,
sanity_check=True,
sanitizers=['sanitizer','asan','lsan','msan','tsan','ubsan','dfsan'],
build_type="Release",
common_cmake_options=None,
support_32_bit=True,
env=None,
jobs="%(jobs)s",
timeout=1200):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
llvm_objdir64 = "llvm64.obj"
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb', # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=".",
env=merged_env))
# Clean up the source and build tree, if requested.
if clean:
f.addStep(ShellCommand(name="rm-llvm_llvm_srcdir",
command=["rm", "-rf", llvm_srcdir],
haltOnFailure=True,
description=["rm src dir", "llvm"],
workdir=".",
env=merged_env))
f.addStep(ShellCommand(name="rm-llvm_objdir",
command=["rm", "-rf", llvm_objdir],
haltOnFailure=True,
description=["rm build dir", "llvm"],
workdir=".",
env=merged_env))
f.addStep(ShellCommand(name="rm-llvm_objdir64",
command=["rm", "-rf", llvm_objdir64],
haltOnFailure=True,
description=["rm build64 dir", "llvm"],
workdir=".",
env=merged_env))
# Get llvm, clang, ompiler-rt, libcxx, libcxxabi, libunwind
f.addStep(SVN(name='svn-llvm',
mode='update',
description='svn-llvm',
descriptionDone='svn-llvm',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
f.addStep(SVN(name='svn-clang',
mode='update',
description='svn-clang',
descriptionDone='svn-clang',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
f.addStep(SVN(name='svn-compiler-rt',
mode='update',
description='svn-compiler-rt',
descriptionDone='svn--compiler-rt',
baseURL='http://llvm.org/svn/llvm-project/compiler-rt/',
defaultBranch='trunk',
workdir='%s/projects/compiler-rt' % llvm_srcdir))
f.addStep(SVN(name='svn-libcxx',
mode='update',
description='svn-libcxx',
descriptionDone='svn-libcxx',
baseURL='http://llvm.org/svn/llvm-project/libcxx/',
defaultBranch='trunk',
workdir='%s/projects/libcxx' % llvm_srcdir))
f.addStep(SVN(name='svn-libcxxabi',
mode='update',
description='svn-libcxxabi',
descriptionDone='svn-libcxxabi',
baseURL='http://llvm.org/svn/llvm-project/libcxxabi/',
defaultBranch='trunk',
workdir='%s/projects/libcxxabi' % llvm_srcdir))
f.addStep(SVN(name='svn-libunwind',
mode='update',
description='svn-libunwind',
descriptionDone='svn-libunwind',
baseURL='http://llvm.org/svn/llvm-project/libunwind/',
defaultBranch='trunk',
workdir='%s/projects/libunwind' % llvm_srcdir))
lint_script = os.path.join("..", llvm_srcdir, "projects", "compiler-rt",
"lib", "sanitizer_common", "scripts",
"check_lint.sh")
# Run annotated command for sanitizer.
f.addStep(AnnotatedCommand(name="lint",
description="lint",
timeout=timeout,
haltOnFailure=True,
command=lint_script,
env=merged_env))
# Create configuration files with cmake.
f.addStep(ShellCommand(name="create-build-dir",
command=["mkdir", "-p", llvm_objdir],
haltOnFailure=True,
description=["create build dir"],
workdir=".",
env=merged_env))
# TODO: make it better way - use list for common_cmake_options and just merge.
if common_cmake_options:
cmakeCommand = [
"cmake",
"-DCMAKE_BUILD_TYPE=%s" % build_type,
"-DCMAKE_C_COMPILER=clang",
"-DCMAKE_CXX_COMPILER=clang++",
"-DCMAKE_CXX_FLAGS='-std=c++11 -stdlib=libc++'",
"%s" % common_cmake_options,
"../%s" % llvm_srcdir]
else:
cmakeCommand = [
"cmake",
"-DCMAKE_BUILD_TYPE=%s" % build_type,
"-DCMAKE_C_COMPILER=clang",
"-DCMAKE_CXX_COMPILER=clang++",
"-DCMAKE_CXX_FLAGS='-std=c++11 -stdlib=libc++'",
"../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols correctly.
# The " ".join is a workaround for this bug.
f.addStep(ShellCommand(name="cmake-configure-1",
description=["cmake configure, phase 1"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand)),
workdir=llvm_objdir,
env=merged_env))
# Build everything.
f.addStep(WarningCountingShellCommand(name="compile",
command=['nice', '-n', '10',
'make', WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["compiling"],
descriptionDone=["compile"],
workdir=llvm_objdir,
env=merged_env))
# Run tests for each of the requested sanitizers.
if sanity_check and sanitizers:
for item in sanitizers:
f.addStep(WarningCountingShellCommand(name="make-check-%s"% item,
command=['make', 'check-%s' % item,
WithProperties("-j%s" % jobs)],
haltOnFailure=False,
description=["make check-%s" % item,],
descriptionDone=["make check-%s" % item,],
workdir=llvm_objdir,
env=merged_env))
# build 64-bit llvm using clang
# Create configuration files with cmake.
f.addStep(ShellCommand(name="create-llvm_build64-dir",
command=["mkdir", "-p", llvm_objdir64],
haltOnFailure=True,
description=["create llvm_build64 dir"],
workdir=".",
env=merged_env))
# Use just built compiler.
clang_path = "%(builddir)s" + "/%s/bin" % llvm_objdir
if common_cmake_options:
cmakeCommand_llvm64 = [
"cmake",
"-DCMAKE_BUILD_TYPE=%s" % build_type,
"%s" % common_cmake_options,
"-DCMAKE_C_COMPILER=%s/clang" % clang_path,
"-DCMAKE_CXX_COMPILER=%s/clang++" % clang_path,
"-DCMAKE_CXX_FLAGS='-I" + "%(builddir)s" + "/%s/projects/libcxx/include -std=c++11 -stdlib=libc++'" % llvm_srcdir,
"../%s" % llvm_srcdir]
else:
cmakeCommand_llvm64 = [
"cmake",
"-DCMAKE_BUILD_TYPE=%s" % build_type,
"-DCMAKE_C_COMPILER=%s/clang" % clang_path,
"-DCMAKE_CXX_COMPILER=%s/clang++" % clang_path,
"-DCMAKE_CXX_FLAGS='-I" + "%(builddir)s" + "/%s/projects/libcxx/include -std=c++11 -stdlib=libc++'" % llvm_srcdir,
"../%s" % llvm_srcdir]
# Note: ShellCommand does not pass the params with special symbols correctly.
# The " ".join is a workaround for this bug.
f.addStep(ShellCommand(name="cmake-configure-2",
description=["cmake configure 64-bit llvm"],
haltOnFailure=True,
command=WithProperties(" ".join(cmakeCommand_llvm64)),
workdir=llvm_objdir64,
env=merged_env))
f.addStep(WarningCountingShellCommand(name="compile 64-bit llvm",
command=['nice', '-n', '10',
'make', WithProperties("-j%s" % jobs)],
haltOnFailure=True,
description=["compiling"],
descriptionDone=["compile"],
workdir=llvm_objdir64,
env=merged_env))
# Run tests for each of the requested sanitizers.
if sanity_check and sanitizers:
for item in sanitizers:
f.addStep(WarningCountingShellCommand(name="make-check-%s 64-bit"% item,
command=['make', 'check-%s' % item,
WithProperties("-j%s" % jobs)],
haltOnFailure=False,
description=["make check-%s 64-bit" % item,],
descriptionDone=["make check-%s" % item,],
workdir=llvm_objdir64,
env=merged_env))
return f
def getLLDBCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
if vs:
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
# get full path to build directory
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir=build_dir))
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
cmake_cmd = [
"cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install"
]
if python_source_dir:
cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_cmd += extra_cmake_args
# Note: ShellCommand does not pass the params with special symbols right.
# The " ".join is a workaround for this bug.
f.addStep(
ShellCommand(name="cmake-configure",
description=["cmake configure"],
command=WithProperties(" ".join(cmake_cmd)),
haltOnFailure=True,
warnOnWarnings=True,
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=2400,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getKLEEBuildFactory(triple, jobs='%(jobs)d', llvm_branch='trunk',
config_name='Release+Asserts', clean=True, llvmgccdir=None,
*args, **kwargs):
if False:
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
else:
# If we are building from trunk, we need to build Clang as
# well so we have access to an LLVM capable compiler.
if llvm_branch == 'trunk':
f = ClangBuilder.getClangBuildFactory(triple, jobs=jobs,
stage1_config=config_name, extra_configure_args=['--with-built-clang',
'--enable-targets=host',
'--with-llvmcc=clang'],
clean=clean, test=False, *args, **kwargs)
else:
f = LLVMBuilder.getLLVMBuildFactory(triple, jobs=jobs, defaultBranch=llvm_branch,
config_name=config_name, llvmgccdir=llvmgccdir,
enable_targets='x86', clean=clean, test=False,
*args, **kwargs)
# Checkout sources.
f.addStep(SVN(name='svn-klee',
mode='update', baseURL='http://llvm.org/svn/llvm-project/klee/',
defaultBranch='trunk', workdir='klee'))
# Configure.
configure_args = ["./configure", WithProperties("--with-llvm=%(builddir)s/llvm")]
configure_args += getConfigArgs(config_name)
if triple:
configure_args += ['--build=%s' % triple,
'--host=%s' % triple,
'--target=%s' % triple]
f.addStep(Configure(command=configure_args, workdir='klee',
description=['configure','klee',config_name]))
# Clean, if requested.
if clean:
f.addStep(WarningCountingShellCommand(name="clean-klee",
command=['make', 'clean'],
haltOnFailure=True,
description="clean klee",
workdir='klee'))
# Compile.
f.addStep(WarningCountingShellCommand(name="compile",
command=['nice', '-n', '10',
'make', WithProperties("-j%s" % jobs)],
haltOnFailure=True, description="compile klee",
workdir='klee'))
# Test.
f.addStep(DejaGNUCommand(name="test",
command=['nice', '-n', '10',
'make', 'check'],
haltOnFailure=True, description="test klee",
workdir='klee',
logfiles={ 'dg.sum' : 'test/testrun.sum' }))
return f
def startBuild(self, cmd):
if cmd.rc != 0:
log.msg("Failure when running %s." % cmd)
self.finished(FAILURE)
else:
return WarningCountingShellCommand.start(self)
def getLLDBWindowsCMakeBuildFactory(
clean=False,
cmake='cmake',
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=r'C:/src/python',
# Default values for VS devenv and build configuration
vs=r"""%VS120COMNTOOLS%""",
config='Release',
target_arch='x86',
extra_cmake_args=[],
test=False,
install=False):
############# PREPARING
f = buildbot.process.factory.BuildFactory()
# Determine Slave Environment and Set MSVC environment.
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f = getLLDBSource(f, 'llvm')
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
# Global configurations
build_dir = 'build'
############# CLEANING
cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
if config.lower() == 'release':
python_lib = 'python27.lib'
python_exe = 'python.exe'
elif config.lower() == 'debug':
python_lib = 'python27_d.lib'
python_exe = 'python_d.exe'
python_lib = os.path.join(python_source_dir, 'PCbuild', python_lib)
python_exe = os.path.join(python_source_dir, 'PCbuild', python_exe)
python_include = os.path.join(python_source_dir, 'Include')
# Use batch files instead of ShellCommand directly, Windows quoting is
# borked. FIXME: See buildbot ticket #595 and buildbot ticket #377.
f.addStep(
batch_file_download.BatchFileDownload(
name='cmakegen',
command=[
cmake,
"-G",
"Ninja",
"../llvm",
"-DCMAKE_BUILD_TYPE=" + config,
# Need to use our custom built version of python
'-DPYTHON_LIBRARY=' + python_lib,
'-DPYTHON_INCLUDE_DIR=' + python_include,
'-DPYTHON_EXECUTABLE=' + python_exe,
"-DCMAKE_INSTALL_PREFIX=../install"
] + extra_cmake_args,
workdir=build_dir))
f.addStep(
ShellCommand(name='cmake',
command=['cmakegen.bat'],
haltOnFailure=True,
description='cmake gen',
workdir=build_dir,
env=Property('slave_env')))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def __init__(self,
architecture=None,
distribution=None,
basetgz=None,
mirror=None,
extrapackages=None,
keyring=None,
components=None,
**kwargs):
"""
Creates the DebPbuilder object.
@type architecture: str
@param architecture: the name of the architecture to build
@type distribution: str
@param distribution: the man of the distribution to use
@type basetgz: str
@param basetgz: the path or path template of the basetgz
@type mirror: str
@param mirror: the mirror for building basetgz
@type extrapackages: list
@param extrapackages: adds packages specified to buildroot
@type keyring: str
@param keyring: keyring file to use for verification
@type components: str
@param components: components to use for chroot creation
@type kwargs: dict
@param kwargs: All further keyword arguments.
"""
WarningCountingShellCommand.__init__(self, **kwargs)
if architecture:
self.architecture = architecture
if distribution:
self.distribution = distribution
if mirror:
self.mirror = mirror
if extrapackages:
self.extrapackages = extrapackages
if keyring:
self.keyring = keyring
if components:
self.components = components
if self.architecture:
kwargs['architecture'] = self.architecture
else:
kwargs['architecture'] = 'local'
kwargs['distribution'] = self.distribution
if basetgz:
self.basetgz = basetgz % kwargs
else:
self.basetgz = self.basetgz % kwargs
if not self.distribution:
config.error("You must specify a distribution.")
self.command = [
'pdebuild', '--buildresult', '.', '--pbuilder', self.pbuilder
]
if self.architecture:
self.command += ['--architecture', self.architecture]
self.command += [
'--', '--buildresult', '.', self.baseOption, self.basetgz
]
if self.extrapackages:
self.command += ['--extrapackages', " ".join(self.extrapackages)]
self.suppressions.append(
(None, re.compile(r"\.pbuilderrc does not exist"), None, None))
def getLLVMGCCBuildFactory(jobs='%(jobs)s', update=True, clean=True,
gxxincludedir=None,
triple=None, build=None, host=None, target=None,
useTwoStage=True, stage1_config='Release+Asserts',
stage2_config='Release+Asserts', make='make',
extra_configure_args=[], extra_languages=None,
verbose=False, env = {}, defaultBranch='trunk',
timeout=20, package_dst=None):
if build or host or target:
if not build or not host or not target:
raise ValueError,"Must specify all of 'build', 'host', 'target' if used."
if triple:
raise ValueError,"Cannot specify 'triple' and 'build', 'host', 'target' options."
elif triple:
build = host = target = triple
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM' : 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
merged_env.update(env) # Overwrite pre-set items with the given ones, so user can set anything.
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(buildbot.steps.shell.SetProperty(name="get_builddir",
command=["pwd"],
property = "builddir",
description = "set build dir",
workdir = ".",
env = merged_env))
# Get the sources.
if update:
f.addStep(SVN(name='svn-llvm',
mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch = defaultBranch,
workdir = "llvm.src"))
f.addStep(SVN(name='svn-llvm-gcc',
mode='update', baseURL='http://llvm.org/svn/llvm-project/llvm-gcc-4.2/',
defaultBranch = defaultBranch,
workdir = "llvm-gcc.src"))
# Clean up llvm (stage 1).
if clean:
f.addStep(ShellCommand(name="rm-llvm.obj.stage1",
command=["rm", "-rf", "llvm.obj"],
haltOnFailure = True,
description = ["rm build dir",
"llvm",
"(stage 1)"],
workdir = ".",
env = merged_env))
# Configure llvm (stage 1).
base_llvm_configure_args = [WithProperties("%(builddir)s/llvm.src/configure")]
if build:
base_llvm_configure_args.append('--build=' + build)
base_llvm_configure_args.append('--host=' + host)
base_llvm_configure_args.append('--target=' + target)
stage_configure_args = getConfigArgs(stage1_config)
f.addStep(Configure(name='configure.llvm.stage1',
command=base_llvm_configure_args +
stage_configure_args +
["--without-llvmgcc",
"--without-llvmgxx"],
description = [ "configure",
"llvm",
"(stage 1)",
stage1_config ],
workdir = "llvm.obj",
env = merged_env))
# Build llvm (stage 1).
base_llvm_make_args = ['nice', '-n', '10',
make, WithProperties("-j%s" % jobs)]
if verbose:
base_llvm_make_args.append('VERBOSE=1')
f.addStep(WarningCountingShellCommand(name = "compile.llvm.stage1",
command = base_llvm_make_args,
haltOnFailure = True,
description = ["compile",
"llvm",
"(stage 1)",
stage1_config],
workdir = "llvm.obj",
env = merged_env,
timeout = timeout * 60))
# Run LLVM tests (stage 1).
f.addStep(LitTestCommand(name = 'test.llvm.stage1',
command = [make, "check-lit", "VERBOSE=1"],
description = ["testing", "llvm"],
descriptionDone = ["test", "llvm"],
workdir = 'llvm.obj',
env = merged_env))
# Clean up llvm-gcc.
if clean:
f.addStep(ShellCommand(name="rm-llvm-gcc.obj.stage1",
command=["rm", "-rf", "llvm-gcc.obj"],
haltOnFailure = True,
description = ["rm build dir",
"llvm-gcc"],
workdir = ".",
env = merged_env))
# Configure llvm-gcc.
base_llvmgcc_configure_args = ["../llvm-gcc.src/configure"]
llvmgcc_languages = "--enable-languages=c,c++"
if extra_languages:
llvmgcc_languages = llvmgcc_languages + "," + extra_languages
base_llvmgcc_configure_args.append(llvmgcc_languages)
if gxxincludedir:
base_llvmgcc_configure_args.append('--with-gxx-include-dir=' + gxxincludedir)
base_llvmgcc_configure_args.extend(extra_configure_args)
if build:
base_llvmgcc_configure_args.append('--build=' + build)
base_llvmgcc_configure_args.append('--host=' + host)
base_llvmgcc_configure_args.append('--target=' + target)
f.addStep(Configure(name='configure.llvm-gcc.stage1',
command=(base_llvmgcc_configure_args +
["--program-prefix=llvm-",
WithProperties("--prefix=%(builddir)s/llvm-gcc.install"),
WithProperties("--enable-llvm=%(builddir)s/llvm.obj")]),
haltOnFailure = True,
description = ["configure",
"llvm-gcc",
"(stage 1)"],
workdir = "llvm-gcc.obj",
env = merged_env))
# Build llvm-gcc.
f.addStep(WarningCountingShellCommand(name="compile.llvm-gcc.stage1",
command=['nice', '-n', '10',
make, WithProperties("-j%s" % jobs)],
haltOnFailure = True,
description = ["compile",
"llvm-gcc"],
workdir = "llvm-gcc.obj",
env = merged_env,
timeout = timeout * 60))
# Clean up llvm-gcc install.
if clean:
f.addStep(ShellCommand(name="rm-llvm-gcc.install.stage1",
command=["rm", "-rf", "llvm-gcc.install"],
haltOnFailure = True,
description = ["rm install dir",
"llvm-gcc"],
workdir = ".",
env = merged_env))
# Install llvm-gcc.
f.addStep(WarningCountingShellCommand(name="install.llvm-gcc.stage1",
command=['nice', '-n', '10',
make, 'install'],
haltOnFailure = True,
description = ["install",
"llvm-gcc"],
workdir = "llvm-gcc.obj",
env = merged_env))
# We are done if not doing a two-stage build.
if not useTwoStage:
return f
# Clean up llvm (stage 2).
if clean:
f.addStep(ShellCommand(name="rm-llvm.obj.stage2",
command=["rm", "-rf", "llvm.obj.2"],
haltOnFailure = True,
description = ["rm build dir",
"llvm",
"(stage 2)"],
workdir = ".",
env = merged_env))
# Configure llvm (stage 2).
stage_configure_args = getConfigArgs(stage2_config)
local_env = dict(merged_env)
local_env['CC'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-gcc")
local_env['CXX'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-g++")
f.addStep(Configure(name="configure.llvm.stage2",
command=base_llvm_configure_args +
stage_configure_args +
[WithProperties("--with-llvmgcc=%(builddir)s/llvm-gcc.install/bin/llvm-gcc"),
WithProperties("--with-llvmgxx=%(builddir)s/llvm-gcc.install/bin/llvm-g++")],
haltOnFailure = True,
description = ["configure",
"llvm",
"(stage 2)",
stage2_config],
workdir = "llvm.obj.2",
env = local_env))
# Build LLVM (stage 2).
f.addStep(WarningCountingShellCommand(name = "compile.llvm.stage2",
command = base_llvm_make_args,
haltOnFailure = True,
description = ["compile",
"llvm",
"(stage 2)",
stage2_config],
workdir = "llvm.obj.2",
env = merged_env,
timeout = timeout * 60))
# Run LLVM tests (stage 2).
f.addStep(LitTestCommand(name = 'test.llvm.stage2',
command = [make, "check-lit", "VERBOSE=1"],
description = ["testing", "llvm", "(stage 2)"],
descriptionDone = ["test", "llvm", "(stage 2)"],
workdir = 'llvm.obj.2',
env = merged_env))
# Clean up llvm-gcc (stage 2).
if clean:
f.addStep(ShellCommand(name="rm-llvm-gcc.obj.stage2",
command=["rm", "-rf", "llvm-gcc.obj.2"],
haltOnFailure = True,
description = ["rm build dir",
"llvm-gcc",
"(stage 2)"],
workdir = ".",
env = merged_env))
# Configure llvm-gcc (stage 2).
local_env = dict(merged_env)
local_env['CC'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-gcc")
local_env['CXX'] = WithProperties("%(builddir)s/llvm-gcc.install/bin/llvm-g++")
f.addStep(Configure(name = 'configure.llvm-gcc.stage2',
command=base_llvmgcc_configure_args + [
"--program-prefix=llvm.2-",
WithProperties("--prefix=%(builddir)s/llvm-gcc.install.2"),
WithProperties("--enable-llvm=%(builddir)s/llvm.obj.2")],
haltOnFailure = True,
description = ["configure",
"llvm-gcc",
"(stage 2)"],
workdir = "llvm-gcc.obj.2",
env = local_env))
# Build llvm-gcc (stage 2).
f.addStep(WarningCountingShellCommand(name="compile.llvm-gcc.stage2",
command=['nice', '-n', '10',
make, WithProperties("-j%s" % jobs)],
haltOnFailure = True,
description = ["compile",
"llvm-gcc",
"(stage 2)"],
workdir = "llvm-gcc.obj.2",
env = merged_env,
timeout = timeout * 60))
# Clean up llvm-gcc install (stage 2).
if clean:
f.addStep(ShellCommand(name="rm-llvm-gcc.install.stage2",
command=["rm", "-rf", "llvm-gcc.install.2"],
haltOnFailure = True,
description = ["rm install dir",
"llvm-gcc",
"(stage 2)"],
workdir = ".",
env = merged_env))
# Install llvm-gcc.
f.addStep(WarningCountingShellCommand(name="install.llvm-gcc.stage2",
command = ['nice', '-n', '10',
make, 'install'],
haltOnFailure = True,
description = ["install",
"llvm-gcc",
"(stage 2)"],
workdir = "llvm-gcc.obj.2",
env = merged_env))
if package_dst:
addPackageStep(f, package_dst, obj_path = 'llvm-gcc.install.2', env = merged_env)
return f
def _getClangCMakeBuildFactory(
clean=True,
test=True,
cmake='cmake',
jobs=None,
# VS tools environment variable if using MSVC. For example,
# %VS120COMNTOOLS% selects the 2013 toolchain.
vs=None,
vs_target_arch='x86',
# Multi-stage compilation
useTwoStage=False,
testStage1=True,
stage1_config='Release',
stage2_config='Release',
# Test-suite
runTestSuite=False,
nt_flags=None,
testsuite_flags=None,
submitURL=None,
testerName=None,
# Environmental variables for all steps.
env=None,
extra_cmake_args=None,
# Extra repositories
checkout_clang_tools_extra=True,
checkout_compiler_rt=True,
checkout_lld=True,
checkout_libcxx=False,
checkout_test_suite=False,
checkout_flang=False,
# Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
stage1_upload_directory=None,
# Use a lower compression level to generate the build-cache package faster
# default is 6 according to documentation
xz_compression_factor=6,
use_pixz_compression=False,
# Triggers
trigger_after_stage1=None):
############# PREPARING
if nt_flags is None:
nt_flags = []
if testsuite_flags is None:
testsuite_flags = []
if env is None:
env = {}
if extra_cmake_args is None:
extra_cmake_args = []
if trigger_after_stage1 is None:
trigger_after_stage1 = []
clean_build_requested = lambda step: \
step.build.getProperty( \
"clean", \
default=step.build.getProperty("clean_obj") \
) or clean
# We *must* checkout at least Clang+LLVM
depends_on_projects = ['llvm', 'clang']
if checkout_clang_tools_extra:
depends_on_projects.append('clang-tools-extra')
if checkout_compiler_rt:
depends_on_projects.append('compiler-rt')
if checkout_lld:
depends_on_projects.append('lld')
if checkout_libcxx:
depends_on_projects.append('libcxx')
depends_on_projects.append('libcxxabi')
depends_on_projects.append('libunwind')
if checkout_flang:
depends_on_projects.append('flang')
depends_on_projects.append('mlir')
f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
llvm_srcdir='llvm')
# Checkout the latest code for LNT
# and the test-suite separately. Le's do this first,
# so we wouldn't poison got_revision property.
if runTestSuite or checkout_test_suite:
f.addGetSourcecodeForProject(project='lnt',
src_dir='test/lnt',
alwaysUseLatest=True)
f.addGetSourcecodeForProject(project='test-suite',
src_dir='test/test-suite',
alwaysUseLatest=True)
# Then get the LLVM source code revision this particular build is for.
f.addGetSourcecodeSteps()
# If jobs not defined, Ninja will choose a suitable value
jobs_cmd = []
lit_args = "'-v"
if jobs is not None:
jobs_cmd = ["-j" + str(jobs)]
lit_args += " -j" + str(jobs) + "'"
else:
lit_args += "'"
ninja_cmd = ['ninja'] + jobs_cmd
ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
# Global configurations
stage1_build = 'stage1'
stage1_install = 'stage1.install'
stage2_build = 'stage2'
stage2_install = 'stage2.install'
# Set up VS environment, if appropriate.
if vs and vs != "manual":
f.addStep(
SetProperty(command=builders_util.getVisualStudioEnvironment(
vs, vs_target_arch),
extract_fn=builders_util.extractSlaveEnvironment))
assert not env, "Can't have custom builder env vars with VS"
env = Property('slave_env')
############# CLEANING
f.addStep(
ShellCommand(name='clean stage 1',
command=['rm', '-rf', stage1_build],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1',
descriptionDone='clean',
workdir='.',
doStepIf=clean_build_requested))
############# STAGE 1
CmakeCommand.applyRequiredOptions(extra_cmake_args, [
('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
])
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)
f.addStep(
ShellCommand(
name='cmake stage 1',
command=[
cmake, "-G", "Ninja", rel_src_dir,
"-DCMAKE_BUILD_TYPE=" + stage1_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage1_install
] + extra_cmake_args,
haltOnFailure=True,
description='cmake stage 1',
workdir=stage1_build,
doStepIf=FileDoesNotExist("build.ninja"),
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 1',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage1_build,
env=env))
if test and testStage1:
haltOnStage1Check = not useTwoStage and not runTestSuite
f.addStep(
LitTestCommand(name='ninja check 1',
command=ninja_check_cmd,
haltOnFailure=haltOnStage1Check,
description=["checking stage 1"],
descriptionDone=["stage 1 checked"],
workdir=stage1_build,
env=env))
if useTwoStage or runTestSuite or stage1_upload_directory:
f.addStep(
ShellCommand(name='clean stage 1 install',
command=['rm', '-rf', stage1_install],
warnOnFailure=True,
haltOnFailure=False,
flunkOnFailure=False,
description='cleaning stage 1 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 1',
command=ninja_install_cmd,
description='ninja install',
workdir=stage1_build,
env=env))
if stage1_upload_directory:
addGCSUploadSteps(f,
'stage 1',
stage1_install,
stage1_upload_directory,
env,
gcs_url_property='stage1_package_gcs_url',
use_pixz_compression=use_pixz_compression,
xz_compression_factor=xz_compression_factor)
# Compute the cmake define flag to set the C and C++ compiler to clang. Use
# clang-cl if we used MSVC for stage1.
if not vs:
cc = 'clang'
cxx = 'clang++'
else:
cc = 'clang-cl.exe'
cxx = 'clang-cl.exe'
############# STAGE 2
if useTwoStage:
# We always cleanly build the stage 2. If the compiler has been
# changed on the stage 1, we cannot trust any of the intermediate file
# from the old compiler. And if the stage 1 compiler is the same, we
# should not build in the first place.
f.addStep(
ShellCommand(name='clean stage 2',
command=['rm', '-rf', stage2_build],
warnOnFailure=True,
description='cleaning stage 2',
descriptionDone='clean',
workdir='.'))
# Set the compiler using the CC and CXX environment variables to work around
# backslash string escaping bugs somewhere between buildbot and cmake. The
# env.exe helper is required to run the tests, so hopefully it's already on
# PATH.
rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir,
stage2_build)
cmake_cmd2 = [
cmake, "-G", "Ninja", rel_src_dir,
WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install +
"/bin/" + cc),
WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" +
stage1_install + "/bin/" + cxx),
"-DCMAKE_BUILD_TYPE=" + stage2_config,
"-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
"-DCMAKE_INSTALL_PREFIX=../" + stage2_install
] + extra_cmake_args
f.addStep(
ShellCommand(name='cmake stage 2',
command=cmake_cmd2,
haltOnFailure=True,
description='cmake stage 2',
workdir=stage2_build,
env=env))
f.addStep(
WarningCountingShellCommand(name='build stage 2',
command=ninja_cmd,
haltOnFailure=True,
description='ninja all',
workdir=stage2_build,
env=env))
if test:
f.addStep(
LitTestCommand(name='ninja check 2',
command=ninja_check_cmd,
haltOnFailure=not runTestSuite,
description=["checking stage 2"],
descriptionDone=["stage 2 checked"],
workdir=stage2_build,
env=env))
############# TEST SUITE
## Test-Suite (stage 2 if built, stage 1 otherwise)
if runTestSuite:
compiler_path = stage1_install
if useTwoStage:
compiler_path = stage2_install
f.addStep(
ShellCommand(name='clean stage 2 install',
command=['rm', '-rf', stage2_install],
warnOnFailure=True,
description='cleaning stage 2 install',
descriptionDone='clean',
workdir='.'))
f.addStep(
ShellCommand(name='install stage 2',
command=ninja_install_cmd,
description='ninja install 2',
workdir=stage2_build,
env=env))
# Get generated python, lnt
python = WithProperties('%(workdir)s/test/sandbox/bin/python')
lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')
# Paths
sandbox = WithProperties('%(workdir)s/test/sandbox')
test_suite_dir = WithProperties('%(workdir)s/test/test-suite')
# Get latest built Clang (stage1 or stage2)
cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)
# LNT Command line (don't pass -jN. Users need to pass both --threads
# and --build-threads in nt_flags/test_suite_flags to get the same effect)
use_runtest_testsuite = len(nt_flags) == 0
if not use_runtest_testsuite:
test_suite_cmd = [
python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
cxx
]
# Append any option provided by the user
test_suite_cmd.extend(nt_flags)
else:
lit = WithProperties('%(workdir)s/' + stage1_build +
'/bin/llvm-lit')
test_suite_cmd = [
python, lnt, 'runtest', 'test-suite', '--no-timestamp',
'--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
cc, '--cxx', cxx, '--use-lit', lit
]
# Append any option provided by the user
test_suite_cmd.extend(testsuite_flags)
# Only submit if a URL has been specified
if submitURL is not None:
if not isinstance(submitURL, list):
submitURL = [submitURL]
for url in submitURL:
test_suite_cmd.extend(['--submit', url])
# lnt runtest test-suite doesn't understand --no-machdep-info:
if testerName and not use_runtest_testsuite:
test_suite_cmd.extend(['--no-machdep-info', testerName])
# CC and CXX are needed as env for build-tools
test_suite_env = copy.deepcopy(env)
test_suite_env['CC'] = cc
test_suite_env['CXX'] = cxx
# Steps to prepare, build and run LNT
f.addStep(
ShellCommand(name='clean sandbox',
command=['rm', '-rf', 'sandbox'],
haltOnFailure=True,
description='removing sandbox directory',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='recreate sandbox',
command=['virtualenv', 'sandbox'],
haltOnFailure=True,
description='recreating sandbox',
workdir='test',
env=env))
f.addStep(
ShellCommand(name='setup lit',
command=[python, lnt_setup, 'develop'],
haltOnFailure=True,
description='setting up LNT in sandbox',
workdir='test/sandbox',
env=env))
f.addStep(
LitTestCommand(name='test-suite',
command=test_suite_cmd,
haltOnFailure=True,
description=['running the test suite'],
workdir='test/sandbox',
logfiles={
'configure.log': 'build/configure.log',
'build-tools.log': 'build/build-tools.log',
'test.log': 'build/test.log',
'report.json': 'build/report.json'
},
env=test_suite_env))
return f
def getLibompCMakeBuildFactory(clean=True,
env=None,
ompt=False,
test=True,
c_compiler="gcc",
cxx_compiler="g++"):
# Prepare environmental variables. Set here all env we want everywhere.
merged_env = {
'TERM': 'dumb' # Make sure Clang doesn't use color escape sequences.
}
if env is not None:
# Overwrite pre-set items with the given ones, so user can set anything.
merged_env.update(env)
openmp_srcdir = "openmp.src"
openmp_builddir = "openmp.build"
llvm_srcdir = "llvm.src"
llvm_builddir = "llvm.build"
f = buildbot.process.factory.BuildFactory()
# Get libomp
f.addStep(
SVN(name='svn-libomp',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/openmp/',
defaultBranch='trunk',
workdir=openmp_srcdir))
# Get llvm to build llvm-lit
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_srcdir))
# Get clang if c/c++ compilers is clang/clang++
if c_compiler == "clang":
f.addStep(
SVN(name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % llvm_srcdir))
# Clean directory, if requested.
if clean:
f.addStep(
ShellCommand(name="clean",
command=["rm", "-rf", openmp_builddir, llvm_builddir],
warnOnFailure=True,
description=["clean"],
workdir='.',
env=merged_env))
# CMake llvm
f.addStep(
ShellCommand(name='cmake llvm',
command=[
"cmake", "../" + llvm_srcdir, "-G", "Ninja",
"-DCMAKE_BUILD_TYPE=Release",
"-DLLVM_ENABLE_ASSERTIONS=ON",
"-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
],
haltOnFailure=True,
description='cmake llvm',
workdir=llvm_builddir,
env=merged_env))
# Make clang build or just llvm utils build
if c_compiler == "clang":
f.addStep(
NinjaCommand(name="make clang build",
haltOnFailure=True,
description=["make clang build"],
workdir=llvm_builddir,
env=merged_env))
else:
f.addStep(
NinjaCommand(name="make llvm utils build",
targets=["LLVMX86Utils"],
haltOnFailure=True,
description=["make llvm utils build"],
workdir=llvm_builddir,
env=merged_env))
if ompt:
f.addStep(
NinjaCommand(name="make ompt test utils",
targets=["FileCheck"],
haltOnFailure=True,
description=["make ompt test utils"],
workdir=llvm_builddir,
env=merged_env))
# Add clang/llvm-lit to PATH
merged_env.update({
'PATH':
WithProperties("%(workdir)s/" + llvm_builddir + "/bin:" + "${PATH}")
})
# CMake libomp
command = [
"cmake", "../" + openmp_srcdir, "-DCMAKE_C_COMPILER=" + c_compiler,
"-DCMAKE_CXX_COMPILER=" + cxx_compiler
]
if ompt:
command.append("-DLIBOMP_OMPT_SUPPORT=ON")
f.addStep(
ShellCommand(name='cmake libomp',
command=command,
haltOnFailure=True,
description='cmake libomp',
workdir=openmp_builddir,
env=merged_env))
# Make libomp
f.addStep(
WarningCountingShellCommand(name='make libomp build',
command=['make'],
haltOnFailure=True,
description='make libomp build',
workdir=openmp_builddir,
env=merged_env))
# Test, if requested
if test:
f.addStep(
LitTestCommand(name="make check-libomp",
command=["make", "check-libomp"],
haltOnFailure=True,
description=["make check-libomp"],
descriptionDone=["build checked"],
workdir=openmp_builddir,
env=merged_env))
return f
def getLLDBBuildFactory(triple,
useTwoStage=False,
make='make',
jobs='%(jobs)s',
extra_configure_args=[],
env={},
*args,
**kwargs):
llvm_srcdir = "llvm.src"
llvm_objdir = "llvm.obj"
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
SetProperty(name="get_builddir",
command=["pwd"],
property="builddir",
description="set build dir",
workdir="."))
# Find out what version of llvm and clang are needed to build this version
# of lldb. Right now we will assume they use the same version.
# XXX - could this be done directly on the master instead of the slave?
f.addStep(
SetProperty(
command=
'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2',
property='llvmrev'))
# The SVN build step provides no mechanism to check out a specific revision
# based on a property, so just run the commands directly here.
svn_co = ['svn', 'checkout', '--force']
svn_co += ['--revision', WithProperties('%(llvmrev)s')]
# build llvm svn checkout command
svn_co_llvm = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
llvm_srcdir]
# build clang svn checkout command
svn_co_clang = svn_co + \
[WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
'%s/tools/clang' % llvm_srcdir]
f.addStep(
ShellCommand(name='svn-llvm',
command=svn_co_llvm,
haltOnFailure=True,
workdir='.'))
f.addStep(
ShellCommand(name='svn-clang',
command=svn_co_clang,
haltOnFailure=True,
workdir='.'))
f.addStep(
SVN(name='svn-lldb',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/lldb/',
defaultBranch='trunk',
always_purge=True,
workdir='%s/tools/lldb' % llvm_srcdir))
# Run configure
config_args = [
WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
"--disable-bindings",
"--without-llvmgcc",
"--without-llvmgxx",
]
if triple:
config_args += ['--build=%s' % triple]
config_args += extra_configure_args
f.addStep(
Configure(name='configure',
command=config_args,
env=env,
workdir=llvm_objdir))
f.addStep(
WarningCountingShellCommand(
name="compile",
command=['nice', '-n', '10', make,
WithProperties("-j%s" % jobs)],
env=env,
haltOnFailure=True,
workdir=llvm_objdir))
# Test.
f.addStep(
LitTestCommand(name="test lldb",
command=['nice', '-n', '10', make],
description="test lldb",
env=env,
workdir='%s/tools/lldb/test' % llvm_objdir))
return f
def getDragonEggBootstrapFactory(gcc_repository,
extra_languages=[],
extra_gcc_configure_args=[],
extra_llvm_configure_args=[],
check_llvm=True,
check_dragonegg=True,
clean=True,
env={},
jobs='%(jobs)s',
timeout=20):
# Add gcc configure arguments required by the plugin.
gcc_configure_args = extra_gcc_configure_args + [
'--enable-plugin', '--enable-lto',
','.join(['--enable-languages=c,c++'] + extra_languages)
]
llvm_configure_args = extra_llvm_configure_args
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.',
env=env))
# Checkout LLVM sources.
llvm_src_dir = 'llvm.src'
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_src_dir,
env=env))
# Checkout DragonEgg sources.
dragonegg_src_dir = 'dragonegg.src'
f.addStep(
SVN(name='svn-dragonegg',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/dragonegg/',
defaultBranch='trunk',
workdir=dragonegg_src_dir,
env=env))
# Checkout GCC. This is usually a specific known good revision (supplied by
# appending @revision to the URL). The SVN step can't handle that. As it
# provides no mechanism at all for checking out a specific revision, just
# run the command directly here.
gcc_src_dir = 'gcc.src'
svn_co = ['svn', 'checkout', gcc_repository, gcc_src_dir]
f.addStep(
ShellCommand(name='svn-gcc',
command=svn_co,
haltOnFailure=True,
workdir='.',
env=env))
gcc_install_dir = 'gcc.install' # Names are embedded in object files, so
llvm_install_dir = 'llvm.install' # would cause bootstrap comparison fails
# if they were different for each stage.
# Remove any install directories hanging over from a previous run.
if clean:
f.addStep(
ShellCommand(name='rm-%s' % gcc_install_dir,
command=['rm', '-rf', gcc_install_dir],
haltOnFailure=True,
description=['rm install dir', 'gcc'],
workdir='.',
env=env))
f.addStep(
ShellCommand(name='rm-%s' % llvm_install_dir,
command=['rm', '-rf', llvm_install_dir],
haltOnFailure=True,
description=['rm install dir', 'llvm'],
workdir='.',
env=env))
# Do the boostrap.
cur_env = env
prev_gcc = None # C compiler built during the previous stage.
prev_gxx = None # C++ compiler built during the previous stage.
prev_plugin = None # Plugin built during the previous stage.
for stage in 'stage1', 'stage2', 'stage3':
# Build and install GCC.
gcc_obj_dir = 'gcc.obj.%s' % stage
if clean:
f.addStep(
ShellCommand(name='rm-%s' % gcc_obj_dir,
command=['rm', '-rf', gcc_obj_dir],
haltOnFailure=True,
description=['rm build dir', 'gcc', stage],
workdir='.',
env=cur_env))
f.addStep(
Configure(name='configure.gcc.%s' % stage,
command=([
'../' + gcc_src_dir + '/configure',
WithProperties(
'--prefix=%%(builddir)s/%s' % gcc_install_dir)
] + gcc_configure_args +
getCCSetting(prev_gcc, prev_gxx)),
haltOnFailure=True,
description=['configure', 'gcc', stage],
workdir=gcc_obj_dir,
env=cur_env))
f.addStep(
WarningCountingShellCommand(name='compile.gcc.%s' % stage,
command=[
'nice', '-n', '10', 'make',
WithProperties('-j%s' % jobs)
],
haltOnFailure=True,
description=['compile', 'gcc', stage],
workdir=gcc_obj_dir,
env=cur_env,
timeout=timeout * 60))
f.addStep(
WarningCountingShellCommand(
name='install.gcc.%s' % stage,
command=['nice', '-n', '10', 'make', 'install'],
haltOnFailure=True,
description=['install', 'gcc', stage],
workdir=gcc_obj_dir,
env=cur_env))
# From this point on build everything using the just built GCC.
prev_gcc = '%(builddir)s/' + gcc_install_dir + '/bin/gcc'
prev_gxx = '%(builddir)s/' + gcc_install_dir + '/bin/g++'
# The built libstdc++ and libgcc may well be more recent than the system
# versions. Set the library path so that programs compiled with the just
# built GCC will start successfully, rather than failing due to missing
# shared library dependencies.
f.addStep(
buildbot.steps.shell.SetProperty(
name='gcc.search.paths.%s' % stage,
command=[WithProperties(prev_gcc), '-print-search-dirs'],
extract_fn=extractSearchPaths,
haltOnFailure=True,
description=['gcc', 'search paths', stage],
env=cur_env))
cur_env = cur_env.copy()
if 'LIBRARY_PATH' in env:
cur_env['LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s' +
':' + env['LIBRARY_PATH'])
else:
cur_env['LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s')
if 'LD_LIBRARY_PATH' in env:
cur_env['LD_LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s' +
':' +
env['LD_LIBRARY_PATH'])
else:
cur_env['LD_LIBRARY_PATH'] = WithProperties('%(gcc_libraries)s')
# Build everything using the DragonEgg plugin from the previous stage.
if prev_plugin is not None:
prev_gcc += ' -fplugin=' + prev_plugin
prev_gxx += ' -fplugin=' + prev_plugin
# Build LLVM with the just built GCC and install it.
llvm_obj_dir = 'llvm.obj.%s' % stage
if clean:
f.addStep(
ShellCommand(name='rm-%s' % llvm_obj_dir,
command=['rm', '-rf', llvm_obj_dir],
haltOnFailure=True,
description=['rm build dir', 'llvm', stage],
workdir='.',
env=cur_env))
f.addStep(
Configure(name='configure.llvm.%s' % stage,
command=([
'../' + llvm_src_dir + '/configure',
WithProperties(
'--prefix=%%(builddir)s/%s' % llvm_install_dir)
] + llvm_configure_args +
getCCSetting(prev_gcc, prev_gxx)),
haltOnFailure=True,
description=['configure', 'llvm', stage],
workdir=llvm_obj_dir,
env=cur_env))
f.addStep(
WarningCountingShellCommand(name='compile.llvm.%s' % stage,
command=[
'nice', '-n', '10', 'make',
WithProperties('-j%s' % jobs)
],
haltOnFailure=True,
description=['compile', 'llvm', stage],
workdir=llvm_obj_dir,
env=cur_env,
timeout=timeout * 60))
# Optionally run the LLVM testsuite.
if check_llvm:
f.addStep(
LitTestCommand(name='check.llvm.%s' % stage,
command=[
'nice', '-n', '10', 'make',
WithProperties('LIT_ARGS=-v -j%s' % jobs),
'check-all'
],
description=['test', 'llvm', stage],
haltOnFailure=True,
workdir=llvm_obj_dir,
env=cur_env,
timeout=timeout * 60))
f.addStep(
WarningCountingShellCommand(
name='install.llvm.%s' % stage,
command=['nice', '-n', '10', 'make', 'install'],
haltOnFailure=True,
description=['install', 'llvm', stage],
workdir=llvm_obj_dir,
env=cur_env))
# Build dragonegg with the just built LLVM and GCC.
dragonegg_pre_obj_dir = 'dragonegg.obj.pre.%s' % stage
if clean:
f.addStep(
ShellCommand(
name='rm-%s' % dragonegg_pre_obj_dir,
command=['rm', '-rf', dragonegg_pre_obj_dir],
description=['rm build dir', 'dragonegg pre', stage],
haltOnFailure=True,
workdir='.',
env=cur_env))
f.addStep(
WarningCountingShellCommand(
name='compile.dragonegg.pre.%s' % stage,
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('-j%s' % jobs),
WithProperties('GCC=%(builddir)s/' + gcc_install_dir +
'/bin/gcc'),
WithProperties('LLVM_CONFIG=%(builddir)s/' +
llvm_install_dir + '/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir)
] + getCCSetting(prev_gcc, prev_gxx),
haltOnFailure=True,
description=['compile', 'dragonegg pre', stage],
workdir=dragonegg_pre_obj_dir,
env=cur_env,
timeout=timeout * 60))
prev_gcc = '%(builddir)s/' + gcc_install_dir + '/bin/gcc -fplugin=%(builddir)s/' + dragonegg_pre_obj_dir + '/dragonegg.so'
prev_gxx = '%(builddir)s/' + gcc_install_dir + '/bin/g++ -fplugin=%(builddir)s/' + dragonegg_pre_obj_dir + '/dragonegg.so'
# Now build dragonegg again using the just built dragonegg. The build is
# always done in the same directory to ensure that object files built by
# the different stages should be the same (the build directory name ends
# up in object files via debug info), then moved to a per-stage directory.
dragonegg_bld_dir = 'dragonegg.obj' # The common build directory.
dragonegg_obj_dir = 'dragonegg.obj.%s' % stage # The per-stage directory.
f.addStep(
ShellCommand(
name='rm-%s' % dragonegg_obj_dir,
command=['rm', '-rf', dragonegg_bld_dir, dragonegg_obj_dir],
description=['rm build dirs', 'dragonegg', stage],
haltOnFailure=True,
workdir='.',
env=cur_env))
f.addStep(
WarningCountingShellCommand(
name='compile.dragonegg.%s' % stage,
command=[
'nice', '-n', '10', 'make', '-f', '../' +
dragonegg_src_dir + '/Makefile', 'DISABLE_VERSION_CHECK=1',
WithProperties('-j%s' % jobs),
WithProperties('GCC=%(builddir)s/' + gcc_install_dir +
'/bin/gcc'),
WithProperties('LLVM_CONFIG=%(builddir)s/' +
llvm_install_dir + '/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir)
] + getCCSetting(prev_gcc, prev_gxx),
description=['compile', 'dragonegg', stage],
haltOnFailure=True,
workdir=dragonegg_bld_dir,
env=cur_env,
timeout=timeout * 60))
f.addStep(
ShellCommand(name='mv-%s' % dragonegg_obj_dir,
command=['mv', dragonegg_bld_dir, dragonegg_obj_dir],
description=['mv build dir', 'dragonegg', stage],
haltOnFailure=True,
workdir='.',
env=cur_env))
# Optionally run the dragonegg testsuite.
if check_dragonegg:
f.addStep(
LitTestCommand(
name='check.dragonegg.%s' % stage,
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('GCC=%(builddir)s/' + gcc_install_dir +
'/bin/gcc'),
WithProperties('LLVM_CONFIG=%(builddir)s/' +
llvm_install_dir + '/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' +
dragonegg_src_dir),
WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check'
],
description=['test', 'dragonegg', stage],
haltOnFailure=True,
workdir=dragonegg_obj_dir,
env=cur_env,
timeout=timeout * 60))
# Ensure that the following stages use the just built plugin.
prev_plugin = '%(builddir)s/' + dragonegg_obj_dir + '/dragonegg.so'
prev_gcc = '%(builddir)s/' + gcc_install_dir + '/bin/gcc -fplugin=' + prev_plugin
prev_gxx = '%(builddir)s/' + gcc_install_dir + '/bin/g++ -fplugin=' + prev_plugin
# Check that the dragonegg objects didn't change between stages 2 and 3.
f.addStep(
ShellCommand(name='compare.stages',
command=[
'sh', '-c', 'for O in *.o ; do cmp ' +
'../dragonegg.obj.stage2/$O ' +
'../dragonegg.obj.stage3/$O || exit 1 ; ' + 'done'
],
haltOnFailure=True,
description='compare stages 2 and 3',
workdir='dragonegg.obj.stage3',
env=cur_env))
return f
def getLLDBCMakeBuildFactory(
clean=False,
jobs="%(jobs)s",
# Source directory containing a built python
python_source_dir=None,
# Default values for VS devenv and build configuration
vs=None,
config='Release',
target_arch='x86',
extra_cmake_args=None,
test=False,
testTimeout=2400,
install=False):
############# PREPARING
# Global configurations
build_dir = 'build'
f = LLVMBuildFactory(is_legacy_mode=False,
depends_on_projects=["llvm", "clang", "lldb", "lld"],
obj_dir=build_dir)
# Determine Slave Environment and Set MSVC environment.
if vs:
f.addStep(
SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
extract_fn=extractSlaveEnvironment))
f.addGetSourcecodeSteps()
build_cmd = ['ninja']
install_cmd = ['ninja', 'install']
test_cmd = ['ninja', 'check-lldb']
if jobs:
build_cmd.append(WithProperties("-j%s" % jobs))
install_cmd.append(WithProperties("-j%s" % jobs))
test_cmd.append(WithProperties("-j%s" % jobs))
############# CLEANING
cleanBuildRequested = lambda step: clean or step.build.getProperty(
"clean", default=step.build.getProperty("clean_obj"))
f.addStep(
RemoveDirectory(name='clean ' + build_dir,
dir=build_dir,
haltOnFailure=False,
flunkOnFailure=False,
doStepIf=cleanBuildRequested))
rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
f.obj_dir)
cmake_options = [
"-G",
"Ninja",
"-DCMAKE_BUILD_TYPE=" + config,
"-DCMAKE_INSTALL_PREFIX=../install",
"-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
]
if python_source_dir:
cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
if extra_cmake_args:
cmake_options += extra_cmake_args
f.addStep(
CmakeCommand(name="cmake-configure",
description=["cmake configure"],
haltOnFailure=True,
options=cmake_options,
path=rel_src_dir,
env=Property('slave_env'),
workdir=build_dir))
f.addStep(
WarningCountingShellCommand(name='build',
command=build_cmd,
haltOnFailure=True,
description='ninja build',
workdir=build_dir,
env=Property('slave_env')))
ignoreInstallFail = bool(install != 'ignoreFail')
f.addStep(
ShellCommand(name='install',
command=install_cmd,
flunkOnFailure=ignoreInstallFail,
description='ninja install',
workdir=build_dir,
doStepIf=bool(install),
env=Property('slave_env')))
ignoreTestFail = bool(test != 'ignoreFail')
f.addStep(
ShellCommand(name='test',
command=test_cmd,
flunkOnFailure=ignoreTestFail,
timeout=testTimeout,
description='ninja test',
workdir=build_dir,
doStepIf=bool(test),
env=Property('slave_env')))
return f
def getDragonEggNightlyTestBuildFactory(gcc='gcc',
gxx='g++',
llvm_configure_args=[],
testsuite_configure_args=[],
xfails=[],
clean=True,
env={},
jobs='%(jobs)s',
timeout=20):
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.',
env=env))
# Checkout LLVM sources.
llvm_src_dir = 'llvm.src'
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_src_dir,
env=env))
# Checkout DragonEgg sources.
dragonegg_src_dir = 'dragonegg.src'
f.addStep(
SVN(name='svn-dragonegg',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/dragonegg/',
defaultBranch='trunk',
workdir=dragonegg_src_dir,
env=env))
# Checkout the test-suite sources.
testsuite_src_dir = 'test-suite.src'
f.addStep(
SVN(name='svn-test-suite',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/test-suite/',
defaultBranch='trunk',
workdir=testsuite_src_dir,
env=env))
# Build and install LLVM.
llvm_obj_dir = 'llvm.obj'
llvm_install_dir = 'llvm.install'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % llvm_obj_dir,
command=['rm', '-rf', llvm_obj_dir, llvm_install_dir],
description='rm build dir llvm',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
Configure(
name='configure.llvm',
command=([
'../' + llvm_src_dir + '/configure',
WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir)
] + llvm_configure_args),
description='configuring llvm',
descriptionDone='configure llvm',
haltOnFailure=True,
workdir=llvm_obj_dir,
env=env))
f.addStep(
WarningCountingShellCommand(name='compile.llvm',
command=[
'nice', '-n', '10', 'make',
WithProperties('-j%s' % jobs)
],
haltOnFailure=True,
description='compiling llvm',
descriptionDone='compile llvm',
workdir=llvm_obj_dir,
env=env,
timeout=timeout * 60))
f.addStep(
WarningCountingShellCommand(
name='install.llvm',
command=['nice', '-n', '10', 'make', 'install'],
haltOnFailure=True,
description='installing llvm',
descriptionDone='install llvm',
workdir=llvm_obj_dir,
env=env))
# Build dragonegg with the just built LLVM.
dragonegg_obj_dir = 'dragonegg.obj'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % dragonegg_obj_dir,
command=['rm', '-rf', dragonegg_obj_dir],
description='rm build dir dragonegg',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
WarningCountingShellCommand(
name='compile.dragonegg',
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('-j%s' % jobs),
WithProperties('GCC=' + gcc),
WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir +
'/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir)
],
haltOnFailure=True,
description='compiling dragonegg',
descriptionDone='compile dragonegg',
workdir=dragonegg_obj_dir,
env=env,
timeout=timeout * 60))
# Pretend that DragonEgg is llvm-gcc by creating llvm-gcc and llvm-g++
# scripts that dispatch to gcc with dragonegg and g++ with dragonegg.
if clean:
f.addStep(
ShellCommand(name='rm-bin',
command=['rm', '-rf', 'bin'],
description='rm bin dir',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
ShellCommand(
name='create.llvm-gcc.script',
command=WithProperties('echo "#!/bin/sh" > llvm-gcc'
'; echo "exec ' + gcc +
' -fplugin=%(builddir)s/' +
dragonegg_obj_dir +
'/dragonegg.so \\"\$@\\"" >> llvm-gcc'
'; chmod a+x llvm-gcc'),
description='create llvm-gcc script',
haltOnFailure=True,
workdir='bin',
env=env))
f.addStep(
ShellCommand(
name='create.llvm-g++.script',
command=WithProperties('echo "#!/bin/sh" > llvm-g++'
'; echo "exec ' + gxx +
' -fplugin=%(builddir)s/' +
dragonegg_obj_dir +
'/dragonegg.so \\"\$@\\"" >> llvm-g++'
'; chmod a+x llvm-g++'),
description='create llvm-g++ script',
haltOnFailure=True,
workdir='bin',
env=env))
# Configure the test-suite.
testsuite_obj_dir = 'test-suite.obj'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % testsuite_obj_dir,
command=['rm', '-rf', testsuite_obj_dir],
description='rm test-suite build dir',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
Configure(
name='configure.test-suite',
command=[
'../' + testsuite_src_dir + '/configure',
WithProperties('--with-llvmsrc=%(builddir)s/' + llvm_src_dir),
WithProperties('--with-llvmobj=%(builddir)s/' + llvm_obj_dir),
WithProperties('--with-llvmgccdir=%(builddir)s/'),
'--with-llvmcc=llvm-gcc', 'CC=' + gcc, 'CXX=' + gxx
] + testsuite_configure_args,
description='configuring nightly test-suite',
descriptionDone='configure nightly test-suite',
haltOnFailure=True,
workdir=testsuite_obj_dir,
env=env))
# Build and test.
f.addStep(
ShellCommand(name='rm.test-suite.report',
command=[
'rm', '-rf', testsuite_obj_dir + '/report',
testsuite_obj_dir + '/report.nightly.raw.out',
testsuite_obj_dir + '/report.nightly.txt'
],
description='rm test-suite report',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
NightlyTestCommand(name='make.test-suite',
command=[
'make',
WithProperties('-j%s' % jobs),
'ENABLE_PARALLEL_REPORT=1', 'DISABLE_CBE=1',
'DISABLE_JIT=1',
'RUNTIMELIMIT=%s' % (timeout * 60),
'TEST=nightly', 'report'
],
logfiles={'report': 'report.nightly.txt'},
description='running nightly test-suite',
descriptionDone='run nightly test-suite',
haltOnFailure=True,
xfails=xfails,
timeout=timeout * 60,
workdir=testsuite_obj_dir,
env=env))
return f
def __init__(self,
architecture=None,
distribution=None,
basetgz=None,
mirror=None,
extrapackages=None,
keyring=None,
components=None,
**kwargs):
"""
Creates the DebPbuilder object.
@type architecture: str
@param architecture: the name of the architecture to build
@type distribution: str
@param distribution: the man of the distribution to use
@type basetgz: str
@param basetgz: the path or path template of the basetgz
@type mirror: str
@param mirror: the mirror for building basetgz
@type extrapackages: list
@param extrapackages: adds packages specified to buildroot
@type keyring: str
@param keyring: keyring file to use for verification
@type components: str
@param components: components to use for chroot creation
@type kwargs: dict
@param kwargs: All further keyword arguments.
"""
WarningCountingShellCommand.__init__(self, **kwargs)
if architecture:
self.architecture = architecture
if distribution:
self.distribution = distribution
if mirror:
self.mirror = mirror
if extrapackages:
self.extrapackages = extrapackages
if keyring:
self.keyring = keyring
if components:
self.components = components
if self.architecture:
kwargs['architecture'] = self.architecture
else:
kwargs['architecture'] = 'local'
kwargs['distribution'] = self.distribution
if basetgz:
self.basetgz = basetgz % kwargs
else:
self.basetgz = self.basetgz % kwargs
if not self.distribution:
config.error("You must specify a distribution.")
self.command = ['pdebuild', '--buildresult', '.', '--pbuilder', self.pbuilder]
if self.architecture:
self.command += ['--architecture', self.architecture]
self.command += ['--', '--buildresult', '.', self.baseOption, self.basetgz]
if self.extrapackages:
self.command += ['--extrapackages', " ".join(self.extrapackages)]
self.suppressions.append((None, re.compile(r"\.pbuilderrc does not exist"), None, None))
self.addLogObserver(
'stdio', logobserver.LineConsumerLogObserver(self.logConsumer))
def getDragonEggTestBuildFactory(gcc='gcc',
svn_testsuites=[],
llvm_configure_args=[],
xfails=[],
clean=True,
env={},
jobs='%(jobs)s',
timeout=20):
f = buildbot.process.factory.BuildFactory()
# Determine the build directory.
f.addStep(
buildbot.steps.shell.SetProperty(name='get_builddir',
command=['pwd'],
property='builddir',
description='set build dir',
workdir='.',
env=env))
# Checkout LLVM sources.
llvm_src_dir = 'llvm.src'
f.addStep(
SVN(name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=llvm_src_dir,
env=env))
# Checkout DragonEgg sources.
dragonegg_src_dir = 'dragonegg.src'
f.addStep(
SVN(name='svn-dragonegg',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/dragonegg/',
defaultBranch='trunk',
workdir=dragonegg_src_dir,
env=env))
# Checkout victims for the compilator.
compilator_dir = dragonegg_src_dir + '/test/compilator'
# Checkout testsuites held in subversion repositories. These are usually
# specified using a known good revision (supplied by appending @revision to
# the URL). The SVN step can't handle that. As it provides no mechanism at
# all for checking out a specific revision, run the command directly here.
for svn_testsuite in svn_testsuites:
url = svn_testsuite[0]
name = svn_testsuite[1]
if url is None:
f.addStep(
ShellCommand(name='rm-%s' % name,
command=['rm', '-rf', name],
description=['rm', name],
haltOnFailure=True,
workdir=compilator_dir,
env=env))
else:
svn_co = ['svn', 'checkout', url, name]
f.addStep(
ShellCommand(name=name,
command=svn_co,
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Checkout miscellaneous testsuites.
# LAPACK.
f.addStep(
ShellCommand(
name='wget.lapack',
command='wget -N http://www.netlib.org/lapack/lapack-3.4.0.tgz',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.lapack',
command='tar xzf lapack-3.4.0.tgz',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Nist Fortran 77 test suite.
f.addStep(
ShellCommand(
name='wget.nist',
command=
'wget -N http://www.itl.nist.gov/div897/ctg/suites/fcvs21.tar.Z',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.nist',
command='tar xzf ../fcvs21.tar.Z',
haltOnFailure=True,
workdir=compilator_dir + '/fcvs21/',
env=env))
# Polyhedron.
f.addStep(
ShellCommand(
name='wget.polyhedron',
command=
'wget -N http://www.polyhedron.com/web_images/documents/pb11.zip',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.polyhedron',
command='unzip -o pb11.zip',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Large single compilation-unit C programs.
f.addStep(
ShellCommand(
name='wget.bzip2',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/bzip2.c',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(
name='wget.gzip',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gzip.c',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='fix.gzip',
command='sed -i "s/^static char \*$/char */" gzip.c',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(
name='wget.oggenc',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/oggenc.c',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(
name='wget.gcc',
command=
'wget -N http://people.csail.mit.edu/smcc/projects/single-file-programs/gcc.c.bz2',
haltOnFailure=False,
workdir=compilator_dir,
env=env))
f.addStep(
ShellCommand(name='unpack.gcc',
command='bunzip2 -f -k gcc.c.bz2',
haltOnFailure=True,
workdir=compilator_dir,
env=env))
# Build and install LLVM.
llvm_obj_dir = 'llvm.obj'
llvm_install_dir = 'llvm.install'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % llvm_obj_dir,
command=['rm', '-rf', llvm_obj_dir, llvm_install_dir],
description='rm build dir llvm',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
Configure(
name='configure.llvm',
command=([
'../' + llvm_src_dir + '/configure',
WithProperties('--prefix=%%(builddir)s/%s' % llvm_install_dir)
] + llvm_configure_args),
description='configuring llvm',
descriptionDone='configure llvm',
haltOnFailure=True,
workdir=llvm_obj_dir,
env=env))
f.addStep(
WarningCountingShellCommand(name='compile.llvm',
command=[
'nice', '-n', '10', 'make',
WithProperties('-j%s' % jobs)
],
haltOnFailure=True,
description='compiling llvm',
descriptionDone='compile llvm',
workdir=llvm_obj_dir,
env=env,
timeout=timeout * 60))
f.addStep(
WarningCountingShellCommand(
name='install.llvm',
command=['nice', '-n', '10', 'make', 'install'],
haltOnFailure=True,
description='installing llvm',
descriptionDone='install llvm',
workdir=llvm_obj_dir,
env=env))
# Build dragonegg with the just built LLVM.
dragonegg_obj_dir = 'dragonegg.obj'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % dragonegg_obj_dir,
command=['rm', '-rf', dragonegg_obj_dir],
description='rm build dir dragonegg',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
WarningCountingShellCommand(
name='compile.dragonegg',
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('-j%s' % jobs),
WithProperties('GCC=' + gcc),
WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir +
'/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir)
],
haltOnFailure=True,
description='compiling dragonegg',
descriptionDone='compile dragonegg',
workdir=dragonegg_obj_dir,
env=env,
timeout=timeout * 60))
# Run the dragonegg testsuite.
testsuite_output_dir = dragonegg_obj_dir + '/test/Output'
if clean:
f.addStep(
ShellCommand(name='rm-%s' % testsuite_output_dir,
command=['rm', '-rf', testsuite_output_dir],
description='rm test-suite output directory',
haltOnFailure=True,
workdir='.',
env=env))
f.addStep(
LitTestCommand(
name='make.check',
command=[
'nice', '-n', '10', 'make', '-f',
'../' + dragonegg_src_dir + '/Makefile',
WithProperties('GCC=' + gcc),
WithProperties('LLVM_CONFIG=%(builddir)s/' + llvm_install_dir +
'/bin/llvm-config'),
WithProperties('TOP_DIR=%(builddir)s/' + dragonegg_src_dir),
WithProperties('LIT_ARGS=-v -j%s' % jobs), 'check'
],
description='running test-suite',
descriptionDone='run test-suite',
haltOnFailure=True,
workdir=dragonegg_obj_dir,
env=env,
timeout=timeout * 60))
return f
