def core_factory(repourl, usedocs=False, configure=None, audit=False,
clean=False, nocheck = False, configure_opts=None):
f = factory.BuildFactory()
mode = "update"
if clean:
#f.addStep(MakeWritable, workdir='.')
f.addStep(shell.ShellCommand(command = ["chmod", "-R", "+w", "."], workdir='.'))
mode = "clobber"
f.addStep(source.Git(repourl=repourl, mode=mode))
if audit:
f.addStep(shell.ShellCommand(command = ["python", 'audit.py'], workdir='build/xapian-maintainer-tools'))
f.addStep(shell.ShellCommand(command = ["chmod", '644', 'copyright.csv', 'fixmes.csv'], workdir='build/xapian-maintainer-tools'))
f.addStep(shell.ShellCommand(command = ["mv", 'copyright.csv', 'fixmes.csv', '/var/www/'], workdir='build/xapian-maintainer-tools'))
f.addStep(Bootstrap())
if configure:
f.addStep(shell.Configure(command=configure))
else:
if configure_opts is None:
configure_opts = []
if not usedocs:
configure_opts.append("--disable-documentation")
if configure_opts:
f.addStep(shell.Configure(command=["sh", "configure"] + configure_opts))
else:
f.addStep(shell.Configure())
f.addStep(shell.Compile())
if not nocheck:
f.addStep(shell.Test(name="check", command=["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
return f
def gen_tarball_updated_factory(rooturl, nocheck=False, omega=True, configure_opts=[]):
"""
Make a factory for doing builds from tarballs.
"""
configure_cmd = ["sh", "configure", ] + configure_opts
f = factory.BuildFactory()
f.addStep(shell.ShellCommand(command = ["python", "-c", "try: import urllib2 as u\nexcept: import urllib.request as u\nopen('get_tarballs.py', 'wb').write(u.urlopen('%s').read())" %
'http://trac.xapian.org/export/HEAD/trunk/xapian-maintainer-tools/buildbot/scripts/get_tarballs.py'], workdir='.', haltOnFailure=True))
f.addStep(shell.ShellCommand(command = ["python", 'get_tarballs.py', rooturl], workdir='.', haltOnFailure=True))
f.addStep(shell.Configure(workdir='build/xapian-core', command=configure_cmd))
f.addStep(shell.Compile(workdir='build/xapian-core'))
if not nocheck:
f.addStep(shell.Test(workdir='build/xapian-core', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
if omega:
f.addStep(shell.Configure(workdir='build/xapian-omega', command = ["./configure", xapian_config_arg] + configure_opts))
f.addStep(shell.Compile(workdir='build/xapian-omega'))
if not nocheck:
f.addStep(shell.Test(workdir='build/xapian-omega', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
f.addStep(shell.Configure(workdir='build/xapian-bindings', command = ["./configure", xapian_config_arg] + configure_opts))
f.addStep(shell.Compile(workdir='build/xapian-bindings', command = ["make"]))
if not nocheck:
f.addStep(shell.Test(workdir='build/xapian-bindings', name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
# If everything passed, there's not much point keeping the build - we'd
# delete the old build tree and download new tarballs next time anyway.
f.addStep(slave.RemoveDirectory('build'))
return f
def gen_tarball_updated_factory(rooturl, configure_opts=[]):
"""
Make a factory for doing builds from tarballs.
"""
configure_cmd = [
"sh",
"configure",
] + configure_opts
f = factory.BuildFactory()
f.addStep(
shell.ShellCommand(command=[
"python", "-c",
"import urllib2;open('get_tarballs.py', 'wb').write(urllib2.urlopen('%s').read())"
%
'http://trac.xapian.org/export/HEAD/trunk/xapian-maintainer-tools/buildbot/scripts/get_tarballs.py'
],
workdir='.',
haltOnFailure=True))
f.addStep(
shell.ShellCommand(command=["python", 'get_tarballs.py'],
workdir='.',
haltOnFailure=True))
f.addStep(
shell.Configure(workdir='build/xapian-core', command=configure_cmd))
f.addStep(shell.Compile(workdir='build/xapian-core'))
f.addStep(
shell.Test(workdir='build/xapian-core',
name="check",
command=[
"make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain",
"VALGRIND="
]))
f.addStep(
shell.Configure(workdir='build/xapian-omega',
command=["./configure", xapian_config_arg] +
configure_opts))
f.addStep(shell.Compile(workdir='build/xapian-omega'))
f.addStep(
shell.Test(workdir='build/xapian-omega',
name="check",
command=[
"make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain",
"VALGRIND="
]))
f.addStep(
shell.Configure(workdir='build/xapian-bindings',
command=["./configure", xapian_config_arg] +
configure_opts))
f.addStep(shell.Compile(workdir='build/xapian-bindings', command=["make"]))
f.addStep(
shell.Test(workdir='build/xapian-bindings',
name="check",
command=[
"make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain",
"VALGRIND="
]))
return f
def gen_svn_updated_lcov_factory(baseURL, configure_opts=[]):
"""
Factory for doing HEAD build from SVN, without cleaning first, and using
lcov to generate a coverage report. This one is much more expensive, so
should be run with a higher stable time.
"""
f = factory.BuildFactory()
f.addStep(source.SVN(baseURL=baseURL, defaultBranch='trunk',
mode="update"))
f.addStep(Bootstrap())
f.addStep(
shell.Configure(command=[
"sh", "configure", "--disable-shared", "CXXFLAGS=-O0 --coverage"
] + configure_opts,
workdir="build/xapian-core"))
f.addStep(shell.Compile(workdir="build/xapian-core"))
f.addStep(
shell.ShellCommand(
command=["make", "coverage-check", "GENHTML_ARGS=--html-gzip"],
workdir="build/xapian-core",
haltOnFailure=True))
f.addStep(
shell.ShellCommand(command=["chmod", "-R", "a+rX", "lcov"],
workdir="build/xapian-core",
haltOnFailure=True))
f.addStep(
shell.ShellCommand(
command=
'NOW=`date -u +%Y-%m-%d`; cp -a lcov/. /var/www/"$NOW" && ln -sfT "$NOW" /var/www/latest',
workdir="build/xapian-core",
haltOnFailure=True))
return f
def gen_svn_updated_valgrind_factory(baseURL, configure_opts=[]):
"""
Factory for doing HEAD build from SVN, without cleaning first, and using
valgrind to check. This one is much more expensive, so should be run with
a higher stable time.
"""
f = factory.BuildFactory()
f.addStep(source.SVN(baseURL=baseURL, defaultBranch='trunk',
mode="update"))
f.addStep(Bootstrap())
f.addStep(
shell.Configure(command=["sh", "configure", "CXXFLAGS=-O0 -g"] +
configure_opts))
f.addStep(shell.Compile())
f.addStep(
shell.Test(name="check",
command=["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain"],
workdir='build/xapian-core'))
#for target in ("check-none", "check-inmemory", "check-remoteprog",
# "check-chert"):
# f.addStep(shell.Test(name=target, command = ["make", target, "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=/home/olly/install/bin/valgrind"], workdir='build/xapian-core'))
#
## Currently, valgrind incorrectly reports leaked memory for the remotetcp
## backend, so check that one without using valgrind.
#f.addStep(shell.Test(name="check-remotetcp", command = ["make", "check-remotetcp", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND=/home/olly/install/bin/valgrind"], workdir='build/xapian-core'))
return f
def gen_git_updated_lcov_factory(repourl, configure_opts=[]):
"""
Factory for doing build from git master, without cleaning first, and using
lcov to generate a coverage report. This one is much more expensive, so
should be run with a higher stable time.
"""
f = factory.BuildFactory()
f.addStep(source.Git(repourl=repourl, mode="update"))
f.addStep(Bootstrap())
f.addStep(
shell.Configure(command=[
"sh", "configure", "--enable-maintainer-mode", "--disable-shared",
"--disable-documentation", "CXXFLAGS=-O0 --coverage", "VALGRIND=",
"CCACHE_DISABLE=1"
] + configure_opts,
workdir="build/xapian-core"))
f.addStep(shell.Compile(workdir="build/xapian-core"))
f.addStep(
shell.ShellCommand(
command=["make", "coverage-check", "GENHTML_ARGS=--html-gzip"],
workdir="build/xapian-core",
haltOnFailure=True))
f.addStep(
shell.ShellCommand(command=["chmod", "-R", "a+rX", "lcov"],
workdir="build/xapian-core",
haltOnFailure=True))
f.addStep(
shell.ShellCommand(
command=
'NOW=`date -u +%Y-%m-%d`; cp -a lcov/. /var/www/"$NOW" && ln -sfT "$NOW" /var/www/latest',
workdir="build/xapian-core",
haltOnFailure=True))
return f
def test_run(self):
self.setup_step(shell.Configure())
self.expect_commands(
ExpectShell(workdir='wkdir', command=["./configure"]).exit(0))
self.expect_outcome(result=SUCCESS)
return self.run_step()
def gen_svn_updated_valgrind_factory(baseURL, configure_opts=[]):
"""
Factory for doing HEAD build from SVN, without cleaning first, and using
valgrind to check. This one is much more expensive, so should be run with
a higher stable time.
"""
f = factory.BuildFactory()
f.addStep(source.SVN(baseURL=baseURL, defaultBranch='trunk',
mode="update"))
f.addStep(Bootstrap())
configure_opts.append("--disable-documentation")
f.addStep(
shell.Configure(command=["sh", "configure", "CXXFLAGS=-O0 -g"] +
configure_opts))
f.addStep(
QuietenLibtool([
'xapian-core/libtool', 'xapian-applications/omega/libtool',
'xapian-bindings/libtool'
]))
f.addStep(shell.Compile())
f.addStep(
shell.Test(name="check",
command=["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain"],
workdir='build/xapian-core'))
return f
def gen_svn_debug_updated_factory(baseURL, opts, nocheck=False):
"""
Make a factory for doing a debug HEAD build from SVN, but without cleaning
first. This build is intended to catch commonly made mistakes quickly.
"""
f = factory.BuildFactory()
f.addStep(source.SVN(baseURL=baseURL, defaultBranch='trunk',
mode="update"))
f.addStep(Bootstrap())
opts.append("--disable-documentation")
f.addStep(shell.Configure(command=[
"sh",
"configure",
] + opts))
f.addStep(
QuietenLibtool([
'xapian-core/libtool', 'xapian-applications/omega/libtool',
'xapian-bindings/libtool'
]))
f.addStep(shell.Compile())
if not nocheck:
f.addStep(
shell.Test(name="check",
command=[
"make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain",
"VALGRIND="
]))
return f
def test_run(self):
self.setupStep(shell.Configure())
self.expectCommands(
ExpectShell(workdir='wkdir', command=["./configure"]) + 0)
self.expectOutcome(result=SUCCESS)
return self.runStep()
def gen_git_debug_updated_factory(repourl, opts, nocheck=False):
"""
Make a factory for doing a debug build from git master, but without cleaning
first. This build is intended to catch commonly made mistakes quickly.
"""
f = factory.BuildFactory()
f.addStep(source.Git(repourl=repourl, mode="update"))
f.addStep(Bootstrap())
opts.append("--disable-documentation")
f.addStep(shell.Configure(command = ["sh", "configure", ] + opts))
f.addStep(shell.Compile())
if not nocheck:
f.addStep(shell.Test(name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain", "VALGRIND="]))
return f
def gen_git_updated_valgrind_factory(repourl, configure_opts=[]):
"""
Factory for doing build from git master, without cleaning first, and using
valgrind to check. This one is much more expensive, so should be run with
a higher stable time.
"""
f = factory.BuildFactory()
f.addStep(source.Git(repourl=repourl, mode="update"))
f.addStep(Bootstrap())
configure_opts.append("--disable-documentation")
f.addStep(shell.Configure(command = ["sh", "configure", "CXXFLAGS=-O0 -g"] + configure_opts))
f.addStep(shell.Compile())
f.addStep(shell.Test(name="check", command = ["make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain"], workdir='build/xapian-core'))
return f
def gen_svn_debug_updated_factory(baseURL, opts):
"""
Make a factory for doing a debug HEAD build from SVN, but without cleaning
first. This build is intended to catch commonly made mistakes quickly.
"""
f = factory.BuildFactory()
f.addStep(source.SVN(baseURL=baseURL, defaultBranch='trunk',
mode="update"))
f.addStep(Bootstrap())
f.addStep(shell.Configure(command=[
"sh",
"configure",
] + opts))
f.addStep(shell.Compile())
f.addStep(
shell.Test(name="check",
command=[
"make", "check", "XAPIAN_TESTSUITE_OUTPUT=plain",
"VALGRIND="
]))
return f
def testAllSteps(self):
# make sure that steps can be created from the factories that they
# return
for s in (
dummy.Dummy(),
dummy.FailingDummy(),
dummy.RemoteDummy(),
maxq.MaxQ("testdir"),
python.BuildEPYDoc(),
python.PyFlakes(),
python_twisted.HLint(),
python_twisted.Trial(testpath=None, tests="tests"),
python_twisted.ProcessDocs(),
python_twisted.BuildDebs(),
python_twisted.RemovePYCs(),
shell.ShellCommand(),
shell.TreeSize(),
shell.Configure(),
shell.Compile(),
shell.Test(),
source.CVS("cvsroot", "module"),
source.SVN("svnurl"),
source.Darcs("repourl"),
source.Git("repourl"),
source.Arch("url", "version"),
source.Bazaar("url", "version", "archive"),
source.Bzr("repourl"),
source.Mercurial("repourl"),
source.P4("p4base"),
source.P4Sync(1234, "p4user", "passwd", "client", mode="copy"),
source.Monotone("server", "branch"),
transfer.FileUpload("src", "dest"),
transfer.FileDownload("src", "dest"),
):
try:
self._loop(s)
except:
print "error checking %s" % s
raise
def test_class_attrs(self):
step = shell.Configure()
self.assertEqual(step.command, ['./configure'])
def test_class_attrs(self):
# nothing too exciting here, but at least make sure the class is
# present
step = shell.Configure()
self.assertEqual(step.command, ['./configure'])
def __init__ ( self, baseURL, useBuildType=None, arguments=[]):
"""Factory for CMake out-of-source builds with extra buildbot code.
Uses the following builder properties (strings) when generating a build:
- generator: CMake generator
- arguments: build-specific CMake arguments
@cvar baseURL: url of the svn repository
@cvar useBuildType: for multi-configuration generators (see CMAKE_BUILD_TYPE)
@cvar arguments: extra CMake arguments
"""
assert baseURL is not None
assert type(arguments) == type([])
from buildbot.steps import source, slave, shell
from buildbot.process.properties import WithProperties, Property
BuildFactory.__init__(self)
if useBuildType is None:
buildTypeArgument=[]
else:
buildTypeArgument=["--config", useBuildType]
# update svn
self.addStep(source.SVN(
workdir=self.sourcedir,
mode='update',
baseURL=baseURL,
defaultBranch='trunk',
retry=(30,2) ))
# recreate build dir
self.addStep(slave.RemoveDirectory(
dir=self.workdir,
haltOnFailure=False,
flunkOnFailure=False))
self.addStep(slave.MakeDirectory(
dir=self.workdir))
# configure
self.addStep(shell.Configure(
command=["cmake", "../source", WithProperties("-G%s", 'generator'), arguments, Property('extra_arguments', default=[])],
logEnviron=False))
# compile - the install target builds and then copies files to the
# production directory (default is subdirectory 'install') and removes
# full paths from library dependencies so it works when you copy the
# production files elsewhere
self.addStep(shell.Compile(
command=["cmake", "--build", ".", "--target", "install"] + buildTypeArgument,
logEnviron=False))
# package - the package target creates an installer/package/archive
# that contains the production files; the target is only available if
# CPack and a package generator are available
self.addStep(shell.SetProperty(
haltOnFailure = True,
flunkOnFailure = True,
extract_fn=lambda rc, stdout, stderr: {"WITH_CPACK": stdout.find("WITH_CPACK:BOOL=ON") != -1},
command=["cmake", "-N", "-LA"],
logEnviron=False))
self.addStep(shell.ShellCommand(
name = "Package",
description = ["packaging"],
descriptionDone = ["package"],
haltOnFailure = False,
flunkOnFailure = False,
doStepIf=lambda step: step.build.getProperty("WITH_CPACK"),
command=["cmake", "--build", ".", "--target", "package"] + buildTypeArgument,
logEnviron=False))
