def masterConfig(bigfilename):
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(name="force", builderNames=["testy"])
]
f = BuildFactory()
# do a bunch of transfer to exercise the protocol
f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt"))
f.addStep(steps.StringDownload("filecontent2", workerdest="dir/file2.txt"))
# create 8 MB file
with open(bigfilename, 'w') as o:
buf = "xxxxxxxx" * 1024
for i in range(1000):
o.write(buf)
f.addStep(
steps.FileDownload(mastersrc=bigfilename, workerdest="bigfile.txt"))
f.addStep(
steps.FileUpload(workersrc="dir/file2.txt", masterdest="master.txt"))
f.addStep(
steps.FileDownload(mastersrc="master.txt", workerdest="dir/file3.txt"))
f.addStep(steps.DirectoryUpload(workersrc="dir", masterdest="dir"))
c['builders'] = [
BuilderConfig(name="testy", workernames=["local1"], factory=f)
]
return c
def _addPreBuildSteps(self):
self.addSteps([
steps.FileDownload(
mastersrc="/home/{{ buildbot_master_user }}/zcash-librustzcash-alloc.diff",
workerdest="zcash-librustzcash-alloc.diff"),
steps.FileDownload(
mastersrc="/home/{{ buildbot_master_user }}/librustzcash-alloc.diff",
workerdest="depends/patches/librustzcash/librustzcash-alloc.diff"),
steps.ShellCommand(
command=['patch', '-p1', '-i', 'zcash-librustzcash-alloc.diff'],
name='Apply librustzcash alloc patch',
haltOnFailure=True,
)
])
def downloadAndRunScript(scriptName, extraFiles=(), args=(), **kwargs):
"""
Downloads the script to remote location and executes it
:param: scriptName name of the local script to execute
:param: extraFiles name of extra files that should be transferred to the remote host
:param: args list of arguments to pass to the remote script
:param: kwargs parameters of the executeStep
"""
taskSteps = []
allFiles = list(extraFiles)
allFiles.append(scriptName)
for file in allFiles:
taskSteps.append(
steps.FileDownload(
name="Transferring {} to worker".format(file),
mastersrc="maxscale/builders/support/scripts/{}".format(file),
workerdest=util.Interpolate(
"%(prop:builddir)s/scripts/{}".format(file)),
hideStepIf=True,
alwaysRun=True,
mode=0o755,
))
remoteScriptName = util.Interpolate(
"%(prop:builddir)s/scripts/{}".format(scriptName))
taskSteps.append(
steps.ShellCommand(command=[remoteScriptName, *args],
timeout=1800,
**kwargs))
return taskSteps
def masterConfig():
c = {}
from buildbot.config import BuilderConfig
from buildbot.process.factory import BuildFactory
from buildbot.plugins import steps, schedulers
c['schedulers'] = [
schedulers.ForceScheduler(
name="force",
builderNames=["testy"])]
f = BuildFactory()
# do a bunch of transfer to exercise the protocol
f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt"))
f.addStep(steps.StringDownload("filecontent2", workerdest="dir/file2.txt"))
f.addStep(
steps.FileUpload(workersrc="dir/file2.txt", masterdest="master.txt"))
f.addStep(
steps.FileDownload(mastersrc="master.txt", workerdest="dir/file3.txt"))
f.addStep(steps.DirectoryUpload(workersrc="dir", masterdest="dir"))
c['builders'] = [
BuilderConfig(name="testy",
workernames=["local1"],
factory=f)
]
return c
def mkdocsfactory():
f = util.BuildFactory()
f.addSteps([
gitStep,
steps.FileDownload(mastersrc="virtualenv.py",
slavedest="virtualenv.py",
flunkOnFailure=True),
# run docs tools in their own virtualenv, otherwise we end up
# documenting the version of Buildbot running the metabuildbot!
VirtualenvSetup(name='virtualenv setup',
no_site_packages=True,
virtualenv_packages=[
'sphinx==1.2.2', '--editable=master',
'--editable=slave'
],
virtualenv_dir='sandbox',
haltOnFailure=True),
# manual
steps.ShellCommand(command=util.Interpolate(
textwrap.dedent("""\
source sandbox/bin/activate &&
make docs
""")),
name="create docs"),
steps.ShellCommand(command=textwrap.dedent("""\
export VERSION=latest &&
tar -C /home/buildbot/www/buildbot.net/buildbot/docs -zvxf master/docs/docs.tgz &&
chmod -R a+rx /home/buildbot/www/buildbot.net/buildbot/docs/latest &&
find /home/buildbot/www/buildbot.net/buildbot/docs/latest -name '*.html' | xargs python /home/buildbot/www/buildbot.net/buildbot/add-tracking.py
"""),
name="docs to web",
flunkOnFailure=True,
haltOnFailure=True)
])
return f
def steps_download(target):
st = []
mastersrc_dir = u'bins-deploy/%(prop:trigger_builder)s/%(prop:revision)s'
# Everything going to /srv should be group-writeable so I can update it manually
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/zImage'),
workerdest=u'/srv/tftp/zImage',
haltOnFailure=True, mode=0o0664, name='Download zImage'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos5420-arndale-octa.dtb'),
workerdest=u'/srv/tftp/exynos5420-arndale-octa.dtb',
haltOnFailure=True, mode=0o0664, name='Download Arndale Octa DTB'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos5422-odroidxu3-lite.dtb'),
workerdest=u'/srv/tftp/exynos5422-odroidxu3-lite.dtb',
haltOnFailure=True, mode=0o0664, name='Download Odroid XU3 DTB'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos4412-odroidu3.dtb'),
workerdest=u'/srv/tftp/exynos4412-odroidu3.dtb',
haltOnFailure=True, mode=0o0664, name='Download Odroid U3 DTB'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos4412-odroidx2.dtb'),
workerdest=u'/srv/tftp/exynos4412-odroidx2.dtb',
haltOnFailure=True, mode=0o0664, name='Download Odroid X2 DTB'))
# XU, XU4 and HC1 might be missing for older kernels
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos5410-odroidxu.dtb'),
workerdest=u'/srv/tftp/exynos5410-odroidxu.dtb',
haltOnFailure=False, warnOnFailure=True, flunkOnFailure=False,
mode=0o0664, name='Download Odroid XU DTB'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos5422-odroidxu4.dtb'),
workerdest=u'/srv/tftp/exynos5422-odroidxu4.dtb',
# In case of failure do not halt, do not fail and mark build as warning.
# flunkOnFailure is by default True.
haltOnFailure=False, warnOnFailure=True, flunkOnFailure=False,
mode=0o0664, name='Download Odroid XU4 DTB'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/exynos5422-odroidhc1.dtb'),
workerdest=u'/srv/tftp/exynos5422-odroidhc1.dtb',
haltOnFailure=False, warnOnFailure=True, flunkOnFailure=False,
mode=0o0664, name='Download Odroid HC1 DTB'))
st.append(steps.FileDownload(
mastersrc=util.Interpolate(mastersrc_dir + '/modules-out.tar.gz'),
workerdest='deploy-modules-out.tar.gz',
haltOnFailure=True, mode=0o0644, name='Download modules'))
return st
def downloadScript(scriptName, **kwargs):
"""Downloads script with the given name from scripts directory to the current worker"""
return [
steps.FileDownload(
name="Transferring {} to worker".format(scriptName),
mastersrc="maxscale/builders/support/scripts/{}".format(
scriptName),
workerdest=util.Interpolate(
"%(prop:builddir)s/scripts/{}".format(scriptName)),
mode=0o755,
**kwargs)
]
def _addCaCertsForHttpsGitRemotes(self):
"""Download in the workers the expected CA certificate files and set
them accordingly in the current user Git configuration file.
.. warning::
As ``git-lfs`` has reimplemented its own procedure to parse the
".gitconfig" file, you need to pay attention to the features
supported by the ``git-lfs`` binary in addition to the ones
supported by ``git``.
For instance, as of November 2018, ``git-lfs`` does neither support
wildcards in the URLs patterns in the ``http`` sections of the Git
configuration file nor file paths to CA certificates beginning by
``~`` (to indicate the home directory) contrary to ``git``.
"""
if not self.buildmaster_setup.private_settings_addendum.additional_git_https_cacerts:
raise ValueError(line(
"""Could not declare custom CA certificates for Git HTTPS
remotes if the Buildbot setup does not provide any."""))
for url, cacert_filepath in (self.buildmaster_setup.private_settings_addendum
.additional_git_https_cacerts
.items()):
sanitizedurl = re.sub(r'[^a-zA-Z0-9\.\-\_\:]+', "_", url)
cacert_filename_in_worker = "cacert-{}.pem".format(sanitizedurl)
self.addStep(steps.FileDownload(
name="get custom cacert",
description="download custom CA certificate for HTTPS git remotes",
haltOnFailure=True,
mastersrc=cacert_filepath,
workerdest="~/.git-cacerts/{}".format(cacert_filename_in_worker),
))
self.addStep(steps.ShellCommand(
name="setup custom cacert for git",
description="set up the fetched CA certificate for HTTPS git remotes",
haltOnFailure=True,
command=textwrap.dedent(
r"""
git config --global "http.${url}.sslCAInfo" \
"${HOME}/.git-cacerts/${cacert_filename_in_worker}"
""").strip(),
env={
"url": url,
"cacert_filename_in_worker": cacert_filename_in_worker,
},
))
def __init__(self):
ZcashBaseFactory.__init__(self)
self.addSteps([
steps.FileDownload(mastersrc="/home/{{ buildbot_master_user }}/block-107134.tar.xz", workerdest="block-107134.tar.xz"),
valgrind('sleep'),
valgrind('parameterloading'),
valgrind('createjoinsplit'),
valgrind('verifyjoinsplit'),
valgrind('solveequihash'),
valgrind('solveequihash', 2, name='valgrind solveequihash 2 threads'),
valgrind('verifyequihash'),
valgrind('connectblockslow'),
])
def run(self):
def does_state_tar_exist(step):
return os.path.exists('flatpak/state-dir.tar')
def does_repo_tar_exist(step):
return os.path.exists('flatpak/repo.tar')
def does_exist(step):
return does_state_tar_exist(step) and does_repo_tar_exist(step)
self.build.addStepsAfterCurrentStep([
steps.FileDownload(
name='download state-dir.tar',
haltOnFailure=True,
doStepIf=does_state_tar_exist,
mastersrc='flatpak/state-dir.tar',
workerdest='state-dir.tar',
),
steps.FileDownload(
name='download repo.tar',
haltOnFailure=True,
doStepIf=does_repo_tar_exist,
mastersrc='flatpak/repo.tar',
workerdest='repo.tar',
),
steps.ShellSequence(
name='unpack archives and clean up',
haltOnFailure=True,
doStepIf=does_exist,
logEnviron=False,
commands=[
util.ShellArg(command=['tar', 'xf', 'state-dir.tar', 'repo.tar'], logfile='stdio'),
util.ShellArg(command=['rm', '-f', 'state-dir.tar', 'repo.tar'], logfile='stdio'),
],
),
])
return buildbot.process.results.SUCCESS
def build_coverity():
remove_build = steps.RemoveDirectory("build")
remove_src = steps.RemoveDirectory("src")
create_build = steps.MakeDirectory("build")
download_src_archive = steps.FileDownload(
mastersrc=util.Property("src_archive"),
workerdest="src.tar.xz",
workdir="src")
extract_src_archive = steps.ShellCommand(
name="Extract source archive",
command=["tar", "xJf", "src.tar.xz"],
workdir="src")
cmake_step = steps.CMake(path="../src/",
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
make_step = steps.Compile(command=[
"cov-build", "--dir", "cov-int", "make", "-j", "16", "-l", "32"
],
workdir="build",
env=env)
compress = steps.ShellCommand(
command=["tar", "czvf", "gnuradio.tgz", "cov-int"], workdir="build")
upload = steps.ShellCommand(command=[
"curl", "--form", "token=" + tokens.coverityToken, "--form",
"[email protected]", "--form", "[email protected]", "--form",
util.Interpolate("version=%(prop:revision)s"), "--form",
util.Interpolate(
"description=\"Weekly Buildbot submission for %(prop:branch)s branch \""
), "https://scan.coverity.com/builds?project=GNURadio"
],
workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(remove_src)
factory.addStep(create_build)
factory.addStep(download_src_archive)
factory.addStep(extract_src_archive)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(compress)
factory.addStep(upload)
return factory
def build_and_test():
remove_build = steps.RemoveDirectory("build")
remove_src = steps.RemoveDirectory("src")
create_build = steps.MakeDirectory("build")
download_src_archive = steps.FileDownload(
mastersrc=util.Property("src_archive"),
workerdest="src.tar.xz",
workdir="src")
extract_src_archive = steps.ShellCommand(
name="Extract source archive",
command=["tar", "xJf", "src.tar.xz"],
workdir="src")
cmake_step = steps.CMake(path="../src/",
definitions=util.Property("cmake_defs", {}),
options=util.Property("cmake_opts", []),
workdir="build",
env=env)
@util.renderer
def join_make_opts(props):
make_opts = props.getProperty("make_opts", [])
return ["make"] + make_opts
make_step = steps.Compile(command=join_make_opts, workdir="build", env=env)
@util.renderer
def parse_test_excludes(props):
command = ["ctest", "--output-on-failure", "--timeout", "120"]
excludes = props.getProperty('test_excludes', [])
excludes.append("qtgui")
if excludes is not None:
command += ["-E", "|".join(excludes)]
return command
test_step = steps.Test(command=parse_test_excludes, workdir="build")
factory = util.BuildFactory()
factory.addStep(remove_build)
factory.addStep(remove_src)
factory.addStep(create_build)
factory.addStep(download_src_archive)
factory.addStep(extract_src_archive)
factory.addStep(cmake_step)
factory.addStep(make_step)
factory.addStep(test_step)
return factory
def downloadAndRunScript(scriptName, args=(), **kwargs):
"""
Downloads the script to remote location and executes it
:param: scriptName name of the local script to execute
"""
remoteScriptName = util.Interpolate(
"%(prop:builddir)s/scripts/{}".format(scriptName))
downloadStep = steps.FileDownload(
name="Transferring {} to worker".format(scriptName),
mastersrc="maxscale/builders/support/scripts/{}".format(scriptName),
workerdest=remoteScriptName,
mode=0o755)
executeStep = steps.ShellCommand(
name="Execute script: {}".format(scriptName),
command=[remoteScriptName, *args],
timeout=1800,
**kwargs)
return [downloadStep, executeStep]
def __init__(self):
ZcashBaseFactory.__init__(self)
self.addSteps([
steps.FileDownload(mastersrc="/home/{{ buildbot_master_user }}/block-107134.tar.xz", workerdest="block-107134.tar.xz"),
sh('wget', '-N', 'https://z.cash/downloads/benchmarks/benchmark-200k-UTXOs.tar.xz', name='download benchmark-200k-UTXOs.tar.xz'),
time('sleep'),
time('parameterloading'),
time('createjoinsplit'),
time('verifyjoinsplit'),
time('solveequihash'),
time('solveequihash', 2, name='time solveequihash 2 threads'),
time('verifyequihash'),
time('validatelargetx'),
time('connectblockslow'),
time('sendtoaddress', '200k-recv', 0.0009, name='time-sendtoaddress-200k-recv-1'),
time('sendtoaddress', '200k-recv', 0.0099, name='time-sendtoaddress-200k-recv-10'),
time('sendtoaddress', '200k-recv', 0.0999, name='time-sendtoaddress-200k-recv-100'),
time('sendtoaddress', '200k-send', 0.0009, name='time-sendtoaddress-200k-send-1'),
time('sendtoaddress', '200k-send', 0.0099, name='time-sendtoaddress-200k-send-10'),
time('sendtoaddress', '200k-send', 0.0999, name='time-sendtoaddress-200k-send-100'),
time('loadwallet', '200k-recv', name='time-loadwallet-200k-recv'),
time('listunspent', '200k-recv', name='time-listunspent-200k-recv'),
memory('sleep'),
memory('parameterloading'),
memory('createjoinsplit'),
memory('verifyjoinsplit'),
memory('solveequihash'),
memory('solveequihash', 2, name='memory solveequihash 2 threads'),
memory('verifyequihash'),
memory('validatelargetx'),
memory('connectblockslow'),
memory('sendtoaddress', '200k-recv', 0.0999, name='memory-sendtoaddress-200k-recv-100'),
memory('loadwallet', '200k-recv', name='memory-loadwallet-200k-recv'),
memory('listunspent', '200k-recv', name='memory-listunspent-200k-recv'),
steps.POST(urllib.parse.urljoin(CODESPEED_URL, '/result/add/json/'),
data={'json': getPerfJson},
auth=('buildbot', CODESPEED_PASS),
verify=CODESPEED_CERT,
doStepIf=lambda s: s.getProperty('publish', False),
hideStepIf=lambda results, s: results==util.SKIPPED,
),
])
def run(self):
self.build.addStepsAfterCurrentStep([
steps.FileDownload(
name='download gpg archive',
haltOnFailure=True,
mastersrc='flatpak/flatpak-gpg.tar.gz',
workerdest='flatpak-gpg.tar.gz',
),
steps.ShellSequence(
name='expand gpg archive',
haltOnFailure=True,
logEnviron=False,
commands=[
util.ShellArg(command=['tar', 'xf', 'flatpak-gpg.tar.gz'], logfile='stdio'),
util.ShellArg(command=['rm', '-f', 'flatpak-gpg.tar.gz'], logfile='stdio'),
util.ShellArg(command=['gpg2', '--homedir', 'flatpak-gpg', '--list-keys'], logfile='stdio'),
],
),
])
return buildbot.process.results.SUCCESS
from typing import List, Optional, Any, Dict
from buildbot import config
from buildbot.plugins import steps, util
from buildbot.process import buildstep, logobserver
from buildbot.process.results import FAILURE, worst_status
from twisted.internet import defer
from .build_factory import master_file, worker_file
download_step = steps.FileDownload(
mastersrc=master_file,
workerdest=worker_file,
mode=755,
)
class ScummVMTest(steps.Test):
"""Test steps that treats a specific warning as errors"""
treat_as_error: str = "###################### LINGO: syntax error"
def has_error(self, line):
return self.treat_as_error in line
def evaluateCommand(self, cmd):
result = super().evaluateCommand(cmd)
if any(self.has_error(line) for line in self.loggedWarnings):
return worst_status(FAILURE, result)
return result
def __init__(self, pkgbuilddir: str, group: str, pkg_base: str,
properties: dict):
super().__init__()
gpg_sign = properties["gpg_sign"]
sshdir = properties["sshdir"]
workdir = f"{pkgbuilddir}/{group}/{pkg_base}"
if group in ("community", "packages"):
workdir += "/trunk"
# set initial properties
self.addStep(
steps.SetProperties(name="set properties from srcinfo",
properties=properties))
# find dependencies
depends = properties["depends"]
if depends is not None:
for depends_name in depends:
self.addSteps([
steps.SetProperty(
name=f"set depends_name to {depends_name}",
property="depends_name",
value=depends_name,
hideStepIf=True,
),
FindDependency(name=f"find {depends_name}"),
])
# download build files
self.addStep(
steps.FileDownload(
name="download PKGBUILD",
mastersrc=f"{workdir}/PKGBUILD",
workerdest="PKGBUILD",
))
for src_name in properties["src_names"]:
self.addStep(
steps.FileDownload(
name=f"download {src_name}",
mastersrc=f"{workdir}/{src_name}",
workerdest=src_name,
))
install = properties["install"]
if install:
self.addStep(
steps.FileDownload(
name=f"download {install}",
mastersrc=f"{workdir}/{install}",
workerdest=install,
))
# update pkgver, pkgrel
self.addSteps([SetPkgver(), SetPkgrel(), Updpkgsums()])
# update git tag revision
if properties["git_tag"]:
self.addStep(SetTagRevision())
# update git commit revision
if properties["git_revision"]:
self.addStep(SetCommitRevision())
# build
self.addStep(ArchBuild())
# update properties
self.addSteps([
Srcinfo(),
steps.FileUpload(
name="upload updated PKGBUILD",
workersrc="PKGBUILD",
masterdest=f"{workdir}/PKGBUILD",
),
steps.FileUpload(
name="upload updated .SRCINFO",
workersrc=".SRCINFO",
masterdest=f"{workdir}/.SRCINFO",
),
steps.SetProperties(
name="refresh properties from srcinfo",
properties=ArchBuildUtil.srcinfo,
),
])
# upload and optionally sign packages
for pkg_name in properties["pkg_names"]:
self.addSteps([
steps.SetProperty(
name=f"set pkg_name to {pkg_name}",
property="pkg_name",
value=pkg_name,
hideStepIf=True,
),
steps.FileUpload(
name=f"upload {pkg_name}",
workersrc=ArchBuildUtil.pkg,
masterdest=ArchBuildUtil.pkg_masterdest,
),
MovePackage(name=f"move {pkg_name}"),
])
if gpg_sign:
self.addSteps([
GpgSign(name=f"sign {pkg_name}"),
steps.FileDownload(
name=f"download {pkg_name} sig",
mastersrc=ArchBuildUtil.sig_mastersrc,
workerdest=ArchBuildUtil.sig_workerdest,
),
])
# update repository
self.addStep(RepoAdd(name=f"add {pkg_name}"))
# synchronize repository
if sshdir:
self.addStep(CreateSshfsDirectory())
self.addStep(MountPkgbuildCom(env=ArchBuildUtil.ssh_agent))
self.addStep(RepoSync(env=ArchBuildUtil.ssh_agent))
self.addStep(UnmountPkgbuildCom())
# cleanup
self.addStep(Cleanup())
