def compile(self, builddir='build', config='release', target=None, useClean=False, desc=None, doStepIf=True,
warningPattern=None, suppressionFile=None, suppressions=None,
runParallel=True, **kwargs):
@renderer
def compileCommand(props):
command = '%s cmake --build . --config %s' % (self.envCmd, config)
if not target is None:
command += ' --target %s' % target
if useClean:
command += ' --clean-first'
if runParallel:
cpus = props.getProperty('CPUs')
if not cpus:
cpus = 1
if self.compiler and self.compiler.startswith('vc'):
command += ' -- /maxcpucount:%s /consoleloggerparameters:NoSummary' % cpus
else:
command += ' -- -j%s' % cpus
return command
if desc is None:
desc = 'compile %s' % config
if warningPattern is None:
warningPattern = self.r_warning_pattern
step = \
Compile(command=compileCommand, workdir=builddir, env=self.env,
name=desc, descriptionDone=desc, description=desc, doStepIf=doStepIf,
warningPattern=warningPattern,
warnOnWarnings=True, haltOnFailure=True,
suppressionFile=suppressionFile, **kwargs)
if suppressions or self.suppressions:
step.addSuppression((suppressions or []) + (self.suppressions or []))
yield self.processStep(step)
def createSummary(self, log):
Compile.createSummary(self, log)
if not self.warningPattern:
return
wre = self.warningPattern
if isinstance(wre, str):
import re
wre = re.compile(wre)
interested = True
if self.partialwarningStart:
interested = False
warnings = []
for line in log.getText().split("\n"):
if line == self.partialwarningStart:
interested = True
if line == self.partialwarningEnd:
interested = False
if interested and wre.match(line):
warnings.append(line)
if warnings:
self.addCompleteLog("partialwarnings", "\n".join(warnings) + "\n")
if self.partialwarningProperty is not None:
old_count = self.getProperty(self.partialwarningProperty, 0)
self.setProperty(self.partialwarningProperty, old_count + len(warnings), "PartialWarningsCompile")
else:
old_count = self.getProperty("partial-warnings-count", 0)
self.setProperty("partial-warnings-count", old_count + len(warnings), "PartialWarningsCompile")
def start(self):
if self.getProperty('release'):
self.command = ['make', self.target or self.name]
else:
self.command = ['make', self.target or self.name, 'TCREV=%s' % self.getProperty('got_revision')]
Compile.start(self)
def start(self):
try:
self.getProperty('release')
except KeyError:
self.setProperty('release', False)
Compile.start(self)
def __init__(self, partialwarningStart=None, partialwarningEnd=None, partialwarningProperty=None, **kwargs):
Compile.__init__(self, **kwargs)
self.partialwarningStart = partialwarningStart
self.partialwarningEnd = partialwarningEnd
self.partialwarningProperty = partialwarningProperty
self.addFactoryArguments(partialwarningStart=self.partialwarningStart)
self.addFactoryArguments(partialwarningEnd=self.partialwarningEnd)
self.addFactoryArguments(partialwarningProperty=self.partialwarningProperty)
def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
if branch == "3.x":
branch = master_branch_version
installed_python = "./target/bin/python%s" % branch
self.addStep(
Configure(command=["./configure", "--prefix", "$(PWD)/target"] +
self.configureFlags))
compile = ["make", self.makeTarget]
install = ["make", self.installTarget]
testopts = self.defaultTestOpts[:]
# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
if parallel:
compile = ["make", parallel, self.makeTarget]
install = ["make", parallel, self.installTarget]
testopts = testopts + [parallel]
test = [installed_python] + self.interpreterFlags
test += ["-m", "test.regrtest"] + testopts
self.addStep(Compile(command=compile))
self.addStep(Install(command=install))
self.addStep(LockInstall())
self.addStep(
Test(command=test, timeout=self.test_timeout,
usePTY=test_with_PTY))
self.addStep(Uninstall())
self.addStep(Clean())
def setup(self, parallel, test_with_PTY=False, **kwargs):
self.addStep(
Configure(command=["./configure", "--prefix", "$(PWD)/target"] +
self.configureFlags))
compile = ["make", self.makeTarget]
testopts = self.testFlags
# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
faulthandler_timeout = self.test_timeout - 5 * 60
if parallel:
compile = ["make", parallel, self.makeTarget]
testopts = testopts + " " + parallel
if "-j" not in testopts:
testopts = "-j2 " + testopts
test = [
"make",
"buildbottest",
"TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
"TESTPYTHONOPTS=" + self.interpreterFlags,
"TESTTIMEOUT=" + str(faulthandler_timeout),
]
self.addStep(Compile(command=compile))
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=["make", "pythoninfo"],
warnOnFailure=True,
))
self.addStep(
Test(command=test, timeout=self.test_timeout,
usePTY=test_with_PTY))
self.addStep(Clean())
def get_cppcheck_steps():
from buildbot.process.properties import Interpolate
from buildbot.steps.shell import Compile
return [
Compile(
name='cppcheck',
description=['cppcheck'],
descriptionDone=['cppcheck'],
command=[
'cppcheck',
Interpolate('-j %(prop:parallel)s'), '--enable=all',
'--inconclusive', '--suppress=unusedFunction',
'--suppress=functionStatic', '--suppress=functionConst',
'--suppress=noConstructor', '--suppress=noExplicitConstructor',
'--suppress=missingInclude', '--force', '-q',
'--template={file}:{line}: warning: ({severity}) {message}',
'-DSFML_SYSTEM_API=', '-DSFML_NETWORK_API=',
'-DSFML_AUDIO_API=', '-DSFML_WINDOW_API=',
'-DSFML_GRAPHICS_API=', '-I', 'include', '-I', 'src', 'src',
'examples'
],
want_stdout=True,
want_stderr=True,
logEnviron=False)
]
def __init__(self,
source,
buildpython=["python"],
trialpython=[],
trial=None,
testpath=".",
randomly=None,
recurse=None,
tests=None,
useTestCaseNames=False,
env=None):
BuildFactory.__init__(self, [source])
assert tests or useTestCaseNames, "must use one or the other"
if trial is not None:
self.trial = trial
if randomly is not None:
self.randomly = randomly
if recurse is not None:
self.recurse = recurse
from buildbot.steps.python_twisted import Trial
buildcommand = buildpython + ["./setup.py", "build"]
self.addStep(Compile(command=buildcommand, env=env))
self.addStep(
Trial(
python=trialpython,
trial=self.trial,
testpath=testpath,
tests=tests,
testChanges=useTestCaseNames,
randomly=self.randomly,
recurse=self.recurse,
env=env,
))
def __init__(self,
source,
configure="./configure",
configureEnv={},
configureFlags=[],
compile=["make", "all"],
test=["make", "check"]):
BuildFactory.__init__(self, [source])
if configure is not None:
# we either need to wind up with a string (which will be
# space-split), or with a list of strings (which will not). The
# list of strings is the preferred form.
if type(configure) is str:
if configureFlags:
assert not " " in configure # please use list instead
command = [configure] + configureFlags
else:
command = configure
else:
assert isinstance(configure, (list, tuple))
command = configure + configureFlags
self.addStep(Configure(command=command, env=configureEnv))
if compile is not None:
self.addStep(Compile(command=compile))
if test is not None:
self.addStep(Test(command=test))
def make_arm():
f = BuildFactory()
f.addStep(
GitNoBranch(repourl="https://github.com/dolphin-emu/dolphin.git",
progress=True,
mode="incremental"))
f.addStep(
ShellCommand(command=["mkdir", "-p", "build"],
logEnviron=False,
description="mkbuilddir",
descriptionDone="mkbuilddir"))
f.addStep(
ShellCommand(command="cmake -DENABLE_QT=OFF -GNinja ..",
workdir="build/build",
description="configuring",
descriptionDone="configure",
haltOnFailure=True))
f.addStep(
Compile(command=["ninja"],
workdir="build/build",
description="building",
descriptionDone="build",
haltOnFailure=True))
f.addStep(
Test(command=["ninja", "unittests"],
workdir="build/build",
description="testing",
descriptionDone="test",
haltOnFailure=True))
return f
def commandComplete(self, cmd):
log = cmd.logs['stdio']
for line in log.readlines():
mt = self.filename_rx.search(line)
if mt:
filename = mt.group(1).strip()
if self.filesuffix is not None:
filename += self.filesuffix
if self.fileprefix is not None:
filename = self.fileprefix + filename
self.setProperty('filename', filename)
self.setProperty('basefilename', filename[filename.rfind(self.sep) + 1:])
break
Compile.commandComplete(self, cmd)
def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
self.addStep(
Configure(
command=["./configure", "--prefix", "$(PWD)/target"]
+ self.configureFlags
)
)
compile = ["make", self.makeTarget]
testopts = self.testFlags
if "-R" not in self.testFlags:
testopts += " --junit-xml test-results.xml"
# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
faulthandler_timeout = self.test_timeout - 5 * 60
if parallel:
compile = ["make", parallel, self.makeTarget]
testopts = testopts + " " + parallel
if "-j" not in testopts:
testopts = "-j2 " + testopts
cleantest = [
"make",
"cleantest",
"TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
"TESTPYTHONOPTS=" + self.interpreterFlags,
"TESTTIMEOUT=" + str(faulthandler_timeout),
]
test = [
"make",
"buildbottest",
"TESTOPTS=" + testopts + " ${BUILDBOT_TESTOPTS}",
"TESTPYTHONOPTS=" + self.interpreterFlags,
"TESTTIMEOUT=" + str(faulthandler_timeout),
]
self.addStep(Compile(command=compile, env=self.compile_environ))
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=["make", "pythoninfo"],
warnOnFailure=True,
env=self.test_environ,
)
)
# FIXME: https://bugs.python.org/issue37359#msg346686
# if regrtest_has_cleanup(branch):
# self.addStep(CleanupTest(command=cleantest))
self.addStep(
Test(
command=test,
timeout=self.test_timeout,
usePTY=test_with_PTY,
env=self.test_environ,
)
)
if branch not in ("3",) and "-R" not in self.testFlags:
self.addStep(UploadTestResults(branch))
self.addStep(Clean())
def make_dolphin_debian_build(mode="normal"):
f = BuildFactory()
mode = mode.split(",")
debug = "debug" in mode
pr = "pr" in mode
fifoci_golden = "fifoci_golden" in mode
f.addStep(
GitNoBranch(repourl="https://github.com/dolphin-emu/dolphin.git",
progress=True,
mode="incremental"))
f.addStep(
ShellCommand(command=["mkdir", "-p", "build"],
logEnviron=False,
description="mkbuilddir",
descriptionDone="mkbuilddir"))
cmake_cmd = ["cmake", "..", "-GNinja"]
if debug:
cmake_cmd.append("-DFASTLOG=ON")
cmake_cmd.append("-DDISTRIBUTOR=dolphin-emu.org")
f.addStep(
ShellCommand(command=cmake_cmd,
workdir="build/build",
description="configuring",
descriptionDone="configure",
haltOnFailure=True))
f.addStep(
Compile(command=["ninja"],
workdir="build/build",
description="building",
descriptionDone="build",
haltOnFailure=True))
f.addStep(
Test(command=["ninja", "unittests"],
workdir="build/build",
description="testing",
descriptionDone="test",
haltOnFailure=True))
if fifoci_golden:
if pr:
f.addStep(
Trigger(schedulerNames=["pr-fifoci-lin"],
copy_properties=[
"pr_id", "repo", "headrev", "branchname",
"shortrev"
]))
else:
f.addStep(
TriggerIfBranch(schedulerNames=["fifoci-lin"],
branchList=["master"],
copy_properties=["shortrev"]))
return f
def describe(self, done=False):
d = Compile.describe(self, done)
if isinstance(d, (str, unicode)):
return "%s: %s" % (self.locale, d)
d = list(d)
d.insert(0, "%s:" % self.locale)
return d
def describe(self, done=False):
d = Compile.describe(self, done)
if isinstance(d, (str, unicode)):
return "%s: %s" % (self.locale, d)
d = list(d)
d.insert(0, "%s:" % self.locale)
return d
def newBuild(self, requests):
"""Create a list of steps to build these possibly coalesced requests.
After initial steps, loop over the locales which need building
and insert steps for each locale."""
locales = getLocalesForRequests(requests)
steps = []
if len(locales) > 0:
steps.append(SetLocalesStep(locales=locales))
steps.append(
NonLocaleMercurial(repourl=self.mainRepoURL,
mainBranch=self.mainBranch))
steps.append(
ShellCommand(command=['autoconf-2.13'], haltOnFailure=True))
steps.append(
ShellCommand(command=['rm', '-rf', 'obj'],
flunkOnFailure=False))
steps.append(
ShellCommand(command=['mkdir', '-p', 'l10n', 'obj/dist'],
flunkOnFailure=False))
configureArgs = [
'../configure', '--enable-application=browser',
'--disable-compile-environment', '--with-l10n-base=../l10n'
]
configureArgs.extend(self.extraConfigureArgs)
steps.append(
Compile(command=configureArgs,
workdir='build/obj',
haltOnFailure=True))
steps.append(
ShellCommand(command=[
'wget', '-O',
'obj/dist/%s' % self.repackFile,
'%s/%s' % (self.repackLocation, self.repackFile)
],
haltOnFailure=True))
for locale in locales:
steps.append(
LocaleMercurial(locale=locale,
localesBranch=self.localesBranch,
workdir='build/l10n/%s' % locale,
repourl=self.localesRepoURL %
{'locale': locale}))
steps.append(
LocaleCompile(locale=locale,
command=[
'make', '-C', 'obj/browser/locales',
'installers-%s' % locale
]))
b = self.buildClass(requests)
b.useProgress = False
b.setStepFactories([step.getStepFactory() for step in steps])
return b
def __init__(
self,
build_targets = Interpolate("%(prop:build_targets)s"),
build_mode = Interpolate("%(prop:build_mode)s"),
build_cat = Interpolate("%(prop:build_cat)s"),
build_extras = Interpolate("%(prop:build_extras)s"),
jobs = None,
**kwargs):
Compile.__init__(
self,
command=SconsCommandRenderer(
build_targets,
build_mode,
build_cat,
build_extras,
jobs),
**kwargs)
def start(self):
multi = self.getProperties().getProperty('cc', '').startswith("vc")
command = ["cmake", "--build", "." if multi else self.config]
if multi:
command.append("--config")
command.append(self.config)
if self.target is not None:
command.append("--target")
command.append(self.target)
self.setCommand(command)
return Compile.start(self)
def start(self):
multi = self.getProperties().getProperty('cc','').startswith("vc")
command = ["cmake", "--build", "." if multi else self.config]
if multi:
command.append("--config")
command.append(self.config)
if self.target is not None:
command.append("--target")
command.append(self.target)
self.setCommand(command)
return Compile.start(self)
def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
if branch == "3.x":
branch = master_branch_version
elif branch == "custom":
branch = "3"
installed_python = "./target/bin/python%s" % branch
self.addStep(
Configure(
command=["./configure", "--prefix", "$(PWD)/target"]
+ self.configureFlags
)
)
compile = ["make", self.makeTarget]
install = ["make", self.installTarget]
testopts = self.defaultTestOpts[:]
# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
if branch != "2.7":
faulthandler_timeout = self.test_timeout - 5 * 60
testopts += [f"--timeout={faulthandler_timeout}"]
if parallel:
compile = ["make", parallel, self.makeTarget]
install = ["make", parallel, self.installTarget]
testopts = testopts + [parallel]
test = [installed_python] + self.interpreterFlags
test += ["-m", "test.regrtest"] + testopts
cleantest = test + ["--cleanup"]
self.addStep(Compile(command=compile))
self.addStep(Install(command=install))
self.addStep(LockInstall())
# FIXME: https://bugs.python.org/issue37359#msg346686
# if regrtest_has_cleanup(branch):
# self.addStep(CleanupTest(command=cleantest))
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=[installed_python, "-m", "test.pythoninfo"],
warnOnFailure=True,
)
)
self.addStep(
Test(command=test, timeout=self.test_timeout, usePTY=test_with_PTY)
)
self.addStep(Uninstall())
self.addStep(Clean())
def __init__(self,
source,
configure="./configure",
configureEnv=None,
configureFlags=None,
reconf=None,
compile=None,
test=None,
distcheck=None):
if configureEnv is None:
configureEnv = {}
if configureFlags is None:
configureFlags = []
if compile is None:
compile = ["make", "all"]
if test is None:
test = ["make", "check"]
if distcheck is None:
distcheck = ["make", "distcheck"]
BuildFactory.__init__(self, [source])
if reconf is True:
reconf = ["autoreconf", "-si"]
if reconf is not None:
self.addStep(
ShellCommand(name="autoreconf",
command=reconf,
env=configureEnv))
if configure is not None:
# we either need to wind up with a string (which will be
# space-split), or with a list of strings (which will not). The
# list of strings is the preferred form.
if isinstance(configure, str):
if configureFlags:
assert " " not in configure # please use list instead
command = [configure] + configureFlags
else:
command = configure
else:
assert isinstance(configure, (list, tuple))
command = configure + configureFlags
self.addStep(Configure(command=command, env=configureEnv))
if compile is not None:
self.addStep(Compile(command=compile, env=configureEnv))
if test is not None:
self.addStep(Test(command=test, env=configureEnv))
if distcheck is not None:
self.addStep(Test(command=distcheck, env=configureEnv))
def setup(self, parallel, branch, **kwargs):
build_command = self.build_command + self.buildFlags
test_command = self.test_command + self.testFlags
if not (branch == "2.7" or "-R" in self.testFlags):
test_command += [r"--junit-xml", r"test-results.xml"]
clean_command = self.clean_command + self.cleanFlags
if parallel:
test_command.append(parallel)
self.addStep(Compile(command=build_command))
if self.remoteTest:
# deploy
self.addStep(
ShellCommand(
name="remotedeploy",
description="remotedeploy",
command=self.remote_deploy_command + self.remoteDeployFlags,
warnOnFailure=True,
)
)
# pythonInfo
self.addStep(
ShellCommand(
name="remotepythoninfo",
description="remotepythoninfo",
command=self.remote_pythonInfo_command + self.remotePythonInfoFlags,
warnOnFailure=True,
)
)
else:
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=self.python_command + ["-m", "test.pythoninfo"],
warnOnFailure=True,
)
)
timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT
if branch != "2.7":
test_command += ["--timeout", timeout - (5 * 60)]
# FIXME: https://bugs.python.org/issue37359#msg346686
# if regrtest_has_cleanup(branch):
# cleantest = test_command + ["--cleanup"]
# self.addStep(CleanupTest(command=cleantest))
self.addStep(Test(command=test_command, timeout=timeout))
if branch not in ("2.7", "3") and "-R" not in self.testFlags:
self.addStep(UploadTestResults(branch))
self.addStep(Clean(command=clean_command))
def setup(self, **kwargs):
self.addStep(
Configure(command=["./configure", "--prefix", "$(PWD)/target"]))
self.addStep(Compile(command=["make"]))
self.addStep(
ShellCommand(name="install",
description="installing",
command=["make", "install"]))
self.addStep(
Test(command=[
"make",
"-C",
"Tools/freeze/test",
"PYTHON=../../../target/bin/python3",
"OUTDIR=../../../target/freezetest",
]))
def setup(self, parallel, branch, **kwargs):
build_command = self.build_command + self.buildFlags
test_command = self.test_command + self.testFlags
clean_command = self.clean_command + self.cleanFlags
if parallel:
test_command.append(parallel)
self.addStep(Compile(command=build_command))
if self.remoteTest:
# deploy
self.addStep(
ShellCommand(
name="remotedeploy",
description="remotedeploy",
command=self.remote_deploy_command +
self.remoteDeployFlags,
warnOnFailure=True,
))
# pythonInfo
self.addStep(
ShellCommand(
name="remotepythoninfo",
description="remotepythoninfo",
command=self.remote_pythonInfo_command +
self.remotePythonInfoFlags,
warnOnFailure=True,
))
else:
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=self.python_command + ["-m", "test.pythoninfo"],
warnOnFailure=True,
))
# timeout is a bit more than the regrtest default timeout
if self.test_timeout:
timeout = self.test_timeout
else:
timeout = TEST_TIMEOUT
if branch != "2.7":
test_command += ["--timeout", timeout - (5 * 60)]
# FIXME: https://bugs.python.org/issue37359#msg346686
#if regrtest_has_cleanup(branch):
# cleantest = test_command + ["--cleanup"]
# self.addStep(CleanupTest(command=cleantest))
self.addStep(Test(command=test_command, timeout=timeout))
self.addStep(Clean(command=clean_command))
def MakeWinLegacyBuilder():
f = BuildFactory()
f.addSteps(svn_co_legacy)
f.addStep(
ShellCommand(
name='revision',
command=['cmd', '/c', 'revision.cmd'],
workdir='build/os/win32',
env={
'SCHAT_VERSION': SCHAT_VERSION_LEGACY,
'SCHAT_REVISION': Property('got_revision')
},
haltOnFailure=True,
))
f.addStep(
ShellCommand(
name='qmake',
command=['qmake', '-r'],
haltOnFailure=True,
))
f.addStep(Compile(
command=['jom', '-j3', '/NOLOGO'],
haltOnFailure=True,
))
f.addStep(
ShellCommand(
name='nsis',
command=['cmd', '/c', 'nsis.cmd'],
workdir='build/os/win32',
env={
'SCHAT_SIGN_FILE': schat_passwords.SIGN_FILE,
'SCHAT_SIGN_PASSWORD': schat_passwords.SIGN_PASSWORD,
'SCHAT_VERSION': SCHAT_VERSION_LEGACY,
'SCHAT_REVISION': Property('got_revision')
},
haltOnFailure=True,
logEnviron=False,
))
f.addStep(
FileUpload(
mode=0644,
slavesrc=WithProperties(
'os/win32/out/schat-%(version_legacy)s.%(got_revision)s.exe'),
masterdest=UploadFileNameLegacy(
'schat-%(version_legacy)s.%(got_revision)s.exe'),
))
def MakeWinBuilder():
f = BuildFactory()
f.addSteps(svn_co)
f.addStep(
ShellCommand(
name='revision',
command=['cmd', '/c', 'revision.cmd'],
workdir='build/os/win32',
env={'SCHAT_REVISION': Property('got_revision')},
haltOnFailure=True,
))
f.addStep(
ShellCommand(
name='qmake',
command=['qmake', '-r'],
haltOnFailure=True,
))
f.addStep(Compile(
command=['jom', '-j3', '/NOLOGO'],
haltOnFailure=True,
))
f.addStep(
ShellCommand(
name='nsis',
command=['cmd', '/c', 'nsis.cmd'],
workdir='build/os/win32',
env={
'SCHAT_SIGN_FILE': schat_passwords.SIGN_FILE,
'SCHAT_SIGN_PASSWORD': schat_passwords.SIGN_PASSWORD,
'SCHAT_VERSION': SCHAT_VERSION,
'SCHAT_REVISION': Property('got_revision')
},
haltOnFailure=True,
logEnviron=False,
))
f.addStep(
FileUpload(
mode=0644,
slavesrc=WithProperties('os/win32/out/schat2-%(version)s.exe'),
masterdest=UploadFileName('schat2-%(version)s%(suffix)s.exe'),
url=WithProperties(
'https://download.schat.me/schat2/snapshots/%(version)s/r%(got_revision)s/schat2-%(version)s%(suffix)s.exe'
),
))
def setup(self, parallel, branch, **kwargs):
build_command = self.build_command + self.buildFlags
test_command = self.test_command + self.testFlags
if "-R" not in self.testFlags:
test_command += [r"--junit-xml", r"test-results.xml"]
clean_command = self.clean_command + self.cleanFlags
if parallel:
test_command.append(parallel)
self.addStep(Compile(command=build_command))
if self.remoteTest:
# deploy
self.addStep(
ShellCommand(
name="remotedeploy",
description="remotedeploy",
command=self.remote_deploy_command + self.remoteDeployFlags,
warnOnFailure=True,
)
)
# pythonInfo
self.addStep(
ShellCommand(
name="remotepythoninfo",
description="remotepythoninfo",
command=self.remote_pythonInfo_command + self.remotePythonInfoFlags,
warnOnFailure=True,
)
)
else:
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=self.python_command + ["-m", "test.pythoninfo"],
warnOnFailure=True,
)
)
timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT
test_command += ["--timeout", timeout - (5 * 60)]
self.addStep(Test(command=test_command, timeout=timeout))
if branch not in ("3",) and "-R" not in self.testFlags:
self.addStep(UploadTestResults(branch))
self.addStep(Clean(command=clean_command))
def stepCompile(self):
@renderer
def compile_command(props):
build_cpus = props.getProperty('parallel', 2)
build_type = props.getProperty('build_type', 'release')
command = 'cmake --build . --config %s -- -j%d' % (build_type,
build_cpus)
return command
@renderer
def compile_desc(props):
return 'compile ' + props.getProperty('build_type', 'release')
self.addStep(
Compile(name=compile_desc,
descriptionDone=compile_desc,
description=compile_desc,
command=compile_command,
warnOnWarnings=True,
haltOnFailure=True))
def make_dolphin_freebsd_build(mode="normal"):
f = BuildFactory()
mode = mode.split(",")
f.addStep(
GitNoBranch(repourl="https://github.com/dolphin-emu/dolphin.git",
progress=True,
mode="incremental"))
f.addStep(
ShellCommand(command=["mkdir", "-p", "build"],
logEnviron=False,
description="mkbuilddir",
descriptionDone="mkbuilddir"))
cmake_cmd = ["cmake", "..", "-GNinja"]
cmake_cmd.append("-DDISTRIBUTOR=dolphin-emu.org")
f.addStep(
ShellCommand(command=cmake_cmd,
workdir="build/build",
description="configuring",
descriptionDone="configure",
haltOnFailure=True,
env={"WX_CONFIG": "wxgtk2u-3.0-config"}))
f.addStep(
Compile(command=["ninja"],
workdir="build/build",
description="building",
descriptionDone="build",
haltOnFailure=True))
f.addStep(
Test(command=["ninja", "unittests"],
workdir="build/build",
description="testing",
descriptionDone="test",
haltOnFailure=True))
return f
def get_cppcheck_steps():
from buildbot.steps.shell import Compile
return [
Compile(
name='cppcheck',
description=['cppcheck'],
descriptionDone=['cppcheck'],
command=[
'cppcheck', '--std=c++03', '--enable=all', '--inconclusive',
'--suppress=unusedFunction', '--suppress=functionStatic',
'--suppress=functionConst', '--suppress=noExplicitConstructor',
'--suppress=variableHidingTypedef',
'--suppress=missingInclude', '--force', '-q',
'--template={file}:{line}: warning: ({severity}) {message}',
'-I', 'include', '-I', 'src', 'src', 'examples'
],
want_stdout=True,
want_stderr=True,
logEnviron=False)
]
def get_build_step(abi):
command = [
"/usr/local/bin/python%s.%s" % (abi[2], abi[3]),
"makepanda/makepanda.py",
"--everything",
"--outputdir", outputdir,
common_flags, universal_flag,
"--osxtarget", Property("osxtarget"),
"--no-gles", "--no-gles2", "--no-egl",
"--version", Property("version"),
]
do_step = True
if abi in ('cp27-cp27m', 'cp34-cp34m', 'cp35-cp35m'):
do_step = is_branch('release/1.10.x')
# Run makepanda - give it enough timeout (1h)
s = Compile(name='compile '+abi, command=command, timeout=1*60*60,
env={"MAKEPANDA_THIRDPARTY": "/Users/buildbot/thirdparty",
"MAKEPANDA_SDKS": "/Users/buildbot/sdks"},
haltOnFailure=True, doStepIf=do_step)
return s
def setup(self, parallel, branch, **kwargs):
build_command = self.build_command + self.buildFlags
test_command = self.test_command + self.testFlags
clean_command = self.clean_command + self.cleanFlags
if parallel:
test_command.append(parallel)
self.addStep(Compile(command=build_command))
self.addStep(
ShellCommand(
name="pythoninfo",
description="pythoninfo",
command=self.python_command + ["-m", "test.pythoninfo"],
warnOnFailure=True,
))
# timeout is a bit more than the regrtest default timeout
if self.test_timeout:
timeout = self.test_timeout
else:
timeout = TEST_TIMEOUT
if branch != "2.7":
test_command += ["--timeout", timeout - (5 * 60)]
self.addStep(Test(command=test_command, timeout=timeout))
self.addStep(Clean(command=clean_command))
def __init__(self, **kwargs):
self.variant = kwargs.pop('variant', None)
Compile.__init__(self, **kwargs)
def createSummary(self, log):
Compile.createSummary(self, log)
self.addURL('coverage',
'http://www.fraca7.net/TaskCoach-coverage/%s/index.html' % (self.getProperty('buildername')))
def createSummary(self, log):
Compile.createSummary(self, log)
self.addURL('Documentation',
'http://www.fraca7.net/TaskCoach-doc/%s/index.html' % (self.getProperty('buildername')))
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'integrationtests']
Compile.__init__(self, **kwargs)
def __init__(self, **kwargs):
name = kwargs.pop('name')
kwargs['command'] = ['./uploadppa.sh', name]
Compile.__init__(self, **kwargs)
self.addFactoryArguments(name=name)
def __init__(self, **kwargs):
kwargs['command'] = ['make', self.name,
WithProperties('TCVERSION=r%s', 'got_revision')]
Compile.__init__(self, **kwargs)
def __init__(self, **kwargs):
name = kwargs.pop('name')
kwargs['command'] = ['make', 'ppa-' + name]
Compile.__init__(self, **kwargs)
self.addFactoryArguments(name=name)
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'revision', WithProperties('TCREV=%s',
'got_revision')]
Compile.__init__(self, **kwargs)
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'unittests']
# On several distros, some tests fail randomly. Just make sure
# we still build the package and so on.
kwargs['haltOnFailure'] = False
Compile.__init__(self, **kwargs)
def __init__(self, config, target = None, **kwargs):
self.config = config
self.target = target
Compile.__init__(self, **kwargs)
def __init__(self, locale, **kwargs):
Compile.__init__(self, **kwargs)
self.addFactoryArguments(locale=locale)
self.locale = locale
def createPoclFactory( environ={},
repository='https://github.com/pocl/pocl.git',
branch='master',
buildICD=True,
llvm_dir='/usr/',
icd_dir='/usr/',
tests_dir=None,
config_opts='',
pedantic=True,
tcedir='',
f=None,
cmake=False
):
"""
Create a buildbot factory object that builds pocl.
environ Dictionary: The environment variables to append to the build. PATH and
LD_LIBRARY_PATH will be added from llvm_dir (if given).
repository String: the repo to build from. defaults to pocl on github
branch String: the branch in 'repository' to build from. default to master
buildICD Bool: if false, the ICD extension is not built.
llvm_dir String: LLVM installation dir. I.e. without the 'bin/' or 'lib/'.
icd_dir String: ICD loader installation dir. We expect here to be a ICD loader that
understand the OCL_ICD_VENDORS parameter, i.e. ocl-icd or patched
Khronos loader.
tests_dir String: Path where the extenral testsuite packages can be copied from.
('cp' is used, so they need to be on the same filesystem).
NOTE: currently only a placeholder - not tested on the public buildbot
config_opts String: extra options to pass to ./configure
cmake String: use CMake instead of autotools to build pocl
"""
#multiple slaves that pend on lock seem to pend after they modified environ.
myenviron = environ.copy()
if 'PATH' in myenviron.keys():
myenviron['PATH'] = llvm_dir+"/bin/:"+myenviron['PATH']+":${PATH}"
else:
myenviron['PATH'] = llvm_dir+"/bin/:${PATH}"
if 'LD_LIBRARY_PATH' in myenviron.keys():
myenviron['LD_LIBRARY_PATH'] = llvm_dir+"/lib/:"+myenviron['PATH']+":${LD_LIBRARY_PATH}"
else:
myenviron['LD_LIBRARY_PATH'] = llvm_dir+"/lib/:${LD_LIBRARY_PATH}"
if tcedir:
myenviron['PATH'] = tcedir+"/bin/:"+myenviron['PATH']
myenviron['LD_LIBRARY_PATH'] = tcedir+"/lib/:"+myenviron['LD_LIBRARY_PATH']
if f==None:
f = factory.BuildFactory()
f.addStep(
Git(
repourl=repository,
mode=Property('git_mode'),
branch=branch )
)
if not cmake:
f.addStep(ShellCommand(
command=["./autogen.sh"],
haltOnFailure=True,
name="autoconfig",
env=myenviron,
description="autoconfiging",
descriptionDone="autoconf"))
if tests_dir!=None:
f.addStep(ShellCommand(
haltOnFailure=True,
command=["cp", "-u", tests_dir+AMD_test_pkg,
"examples/AMD/"+AMD_test_pkg],
name="copy AMD",
description="copying",
descriptionDone="copied AMD",
#kludge around 'cp' always complaining if source is missing
decodeRC={0:SUCCESS,1:SUCCESS}
))
f.addStep(ShellCommand(
haltOnFailure=False,
command=["cp", "-u", tests_dir+ViennaCL_test_pkg,
"examples/ViennaCL/"+ViennaCL_test_pkg],
name="copy ViennaCL",
description="copying",
descriptionDone="copied ViennaCL",
decodeRC={0:SUCCESS,1:SUCCESS}
))
if cmake:
f.addStep(
ShellCommand(
command=["cmake", "."],
env=myenviron,
haltOnFailure=True,
name="CMake",
description="cmaking",
descriptionDone="cmade"))
else:
configOpts=config_opts.split(' ')
if pedantic==True:
configOpts = configOpts + ['--enable-pedantic']
if buildICD==False:
configOpts = configOpts + ['--disable-icd']
f.addStep(ShellCommand(
command=["./configure"] + configOpts,
haltOnFailure=True,
name="configure pocl",
env=myenviron,
description="configureing",
descriptionDone="configure"))
f.addStep(Compile(env=myenviron ))
if tests_dir!=None and not cmake:
f.addStep(ShellCommand(command=["make", "prepare-examples"],
haltOnFailure=True,
name="prepare examples",
env=myenviron,
description="preparing",
descriptionDone="prepare"))
if tcedir:
f.addStep(ShellCommand(command=["./tools/scripts/run_tta_tests"],
haltOnFailure=True,
name="checks",
env=myenviron,
description="testing",
descriptionDone="tests",
logfiles={"test.log": "tests/testsuite.log"},
timeout=60*60))
else:
f.addStep(ShellCommand(command=["make", "check"],
haltOnFailure=True,
name="checks",
env=myenviron,
description="testing",
descriptionDone="tests",
logfiles={"test.log": "tests/testsuite.log"},
#blas3 alone takes 15-20 min.
timeout=60*60))
return f
#######
## LLVM/clang builder
##
# srcdir - LLVM source diectory
# builddir - LLVM build dir
# installdir - final LLVM install directory
# test_install_dir - the LLVM install dir pocl_build tests against
def createLLVMFactory(srcdir, builddir, installdir, test_install_dir):
f = factory.BuildFactory()
f.addStep(
SVN(
name='svn-llvm',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/llvm/',
defaultBranch='trunk',
workdir=srcdir))
f.addStep(
SVN(
name='svn-clang',
mode='update',
baseURL='http://llvm.org/svn/llvm-project/cfe/',
defaultBranch='trunk',
workdir='%s/tools/clang' % srcdir))
f.addStep(
ShellCommand(
command=[
'%s/configure' % srcdir,
'--prefix=' + installdir,
'--enable-optimized',
'--enable-targets=host',
'--enable-shared'],
workdir=builddir,
haltOnFailure=True,
name="configure",
descriptionDone='configure',
description='configuring'))
f.addStep(
ShellCommand(
command=['make', '-j', '4'],
workdir=builddir,
haltOnFailure=True,
name = "compile",
descriptionDone = 'compile',
description='compiling'))
f.addStep(
ShellCommand(
command=['make', 'check'],
workdir=builddir,
name='check',
descriptionDone='check',
haltOnFailure=True,
description='checking'))
f.addStep(
ShellCommand(
command=['make', 'install'],
env={'DESTDIR':test_install_dir},
workdir=builddir,
haltOnFailure=True,
name = 'install',
descriptionDone='install',
description='installing'))
f=createPoclFactory(
llvm_dir=test_install_dir+installdir,
f=f)
f.addStep(
ShellCommand(
command=['make', 'install'],
workdir=builddir,
haltOnFailure=True,
name = 'install',
descriptionDone='install',
description='installing'))
return f
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'nuke']
Compile.__init__(self, **kwargs)
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'ppa-' + kwargs.pop('name')]
Compile.__init__(self, **kwargs)
c['builders'] = []
f = BuildFactory()
# Check out the source.
f.addStep(Git(repourl="git://github.com/Forkk/forkk.net", mode="incremental"))
# f.addStep(ShellCommand(name="init-sandbox",
# command=["cabal", "sandbox", "init"],
# description=["initializing", "sandbox"],
# descriptionDone=["initialize", "sandbox"]))
# Build the site.
f.addStep(Compile(
name="compile",
command=["ghc", "--make", "-threaded", "site.hs"],
description=["compiling", "site", "builder"],
descriptionDone=["compile", "site", "builder"]
))
f.addStep(Compile(
name="build-site",
command=["./site", "rebuild"],
description=["building", "site"],
descriptionDone=["build", "site"]
))
f.addStep(DirectoryUpload(
name="deploy",
slavesrc="_site",
masterdest="/var/www/forkk.net",
description=["deploying", "site"],
descriptionDone=["deploy", "site"]
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'coverage']
# Same remark as for UnitTests
kwargs['haltOnFailure'] = False
Compile.__init__(self, **kwargs)
# These steps fill in properties used to determine upstream_version.
] + whl_version_steps + [
# Download the Dockerfile for this distribution.
FileDownload(mastersrc=Interpolate("dockerfiles/%(prop:suite)s-%(prop:arch)s"), workerdest="Dockerfile", workdir="context"),
# Make sure the base distribution is up-to-date.
ShellCommand(command=cloudimg_cmd, workdir="context"),
# Build the Docker image.
ShellCommand(name="setup", command=setup_cmd, workdir="context", haltOnFailure=True),
# Invoke makepanda.
Compile(name="compile py2",
command=get_build_command(2),
haltOnFailure=True,
doStepIf=is_branch("release/1.10.x")),
Compile(name="compile py3",
command=get_build_command(3),
haltOnFailure=True),
# Run the test suite.
Test(name="test py2", command=get_test_command(2), haltOnFailure=True, doStepIf=is_branch("release/1.10.x")),
Test(name="test py3", command=get_test_command(3), haltOnFailure=True),
# Build the installer.
ShellCommand(name="package", command=package_cmd, haltOnFailure=True,
doStepIf=lambda step:not step.getProperty("optimize", False)),
# And the test scripts for deploy-ng.
#Test(name="build_samples", command=test_deployng_cmd, doStepIf=is_branch("deploy-ng"), haltOnFailure=True),
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'disttests']
Compile.__init__(self, **kwargs)
def createSummary(self, log):
if not self.ignoreWarnings:
Compile.createSummary(self, log)
def __init__(self, **kwargs):
kwargs['command'] = ['make', 'languagetests']
Compile.__init__(self, **kwargs)
