def masterConfig(bigfilename): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = BuildFactory() # do a bunch of transfer to exercise the protocol f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt")) f.addStep(steps.StringDownload("filecontent2", workerdest="dir/file2.txt")) # create 8 MB file with open(bigfilename, 'w') as o: buf = "xxxxxxxx" * 1024 for i in range(1000): o.write(buf) f.addStep( steps.FileDownload(mastersrc=bigfilename, workerdest="bigfile.txt")) f.addStep( steps.FileUpload(workersrc="dir/file2.txt", masterdest="master.txt")) f.addStep( steps.FileDownload(mastersrc="master.txt", workerdest="dir/file3.txt")) f.addStep(steps.DirectoryUpload(workersrc="dir", masterdest="dir")) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def _addPreBuildSteps(self): self.addSteps([ steps.FileDownload( mastersrc="/home/{{ buildbot_master_user }}/zcash-librustzcash-alloc.diff", workerdest="zcash-librustzcash-alloc.diff"), steps.FileDownload( mastersrc="/home/{{ buildbot_master_user }}/librustzcash-alloc.diff", workerdest="depends/patches/librustzcash/librustzcash-alloc.diff"), steps.ShellCommand( command=['patch', '-p1', '-i', 'zcash-librustzcash-alloc.diff'], name='Apply librustzcash alloc patch', haltOnFailure=True, ) ])
def downloadAndRunScript(scriptName, extraFiles=(), args=(), **kwargs): """ Downloads the script to remote location and executes it :param: scriptName name of the local script to execute :param: extraFiles name of extra files that should be transferred to the remote host :param: args list of arguments to pass to the remote script :param: kwargs parameters of the executeStep """ taskSteps = [] allFiles = list(extraFiles) allFiles.append(scriptName) for file in allFiles: taskSteps.append( steps.FileDownload( name="Transferring {} to worker".format(file), mastersrc="maxscale/builders/support/scripts/{}".format(file), workerdest=util.Interpolate( "%(prop:builddir)s/scripts/{}".format(file)), hideStepIf=True, alwaysRun=True, mode=0o755, )) remoteScriptName = util.Interpolate( "%(prop:builddir)s/scripts/{}".format(scriptName)) taskSteps.append( steps.ShellCommand(command=[remoteScriptName, *args], timeout=1800, **kwargs)) return taskSteps
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] f = BuildFactory() # do a bunch of transfer to exercise the protocol f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt")) f.addStep(steps.StringDownload("filecontent2", workerdest="dir/file2.txt")) f.addStep( steps.FileUpload(workersrc="dir/file2.txt", masterdest="master.txt")) f.addStep( steps.FileDownload(mastersrc="master.txt", workerdest="dir/file3.txt")) f.addStep(steps.DirectoryUpload(workersrc="dir", masterdest="dir")) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def mkdocsfactory(): f = util.BuildFactory() f.addSteps([ gitStep, steps.FileDownload(mastersrc="virtualenv.py", slavedest="virtualenv.py", flunkOnFailure=True), # run docs tools in their own virtualenv, otherwise we end up # documenting the version of Buildbot running the metabuildbot! VirtualenvSetup(name='virtualenv setup', no_site_packages=True, virtualenv_packages=[ 'sphinx==1.2.2', '--editable=master', '--editable=slave' ], virtualenv_dir='sandbox', haltOnFailure=True), # manual steps.ShellCommand(command=util.Interpolate( textwrap.dedent("""\ source sandbox/bin/activate && make docs """)), name="create docs"), steps.ShellCommand(command=textwrap.dedent("""\ export VERSION=latest && tar -C /home/buildbot/www/buildbot.net/buildbot/docs -zvxf master/docs/docs.tgz && chmod -R a+rx /home/buildbot/www/buildbot.net/buildbot/docs/latest && find /home/buildbot/www/buildbot.net/buildbot/docs/latest -name '*.html' | xargs python /home/buildbot/www/buildbot.net/buildbot/add-tracking.py """), name="docs to web", flunkOnFailure=True, haltOnFailure=True) ]) return f
def steps_download(target): st = [] mastersrc_dir = u'bins-deploy/%(prop:trigger_builder)s/%(prop:revision)s' # Everything going to /srv should be group-writeable so I can update it manually st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/zImage'), workerdest=u'/srv/tftp/zImage', haltOnFailure=True, mode=0o0664, name='Download zImage')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos5420-arndale-octa.dtb'), workerdest=u'/srv/tftp/exynos5420-arndale-octa.dtb', haltOnFailure=True, mode=0o0664, name='Download Arndale Octa DTB')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos5422-odroidxu3-lite.dtb'), workerdest=u'/srv/tftp/exynos5422-odroidxu3-lite.dtb', haltOnFailure=True, mode=0o0664, name='Download Odroid XU3 DTB')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos4412-odroidu3.dtb'), workerdest=u'/srv/tftp/exynos4412-odroidu3.dtb', haltOnFailure=True, mode=0o0664, name='Download Odroid U3 DTB')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos4412-odroidx2.dtb'), workerdest=u'/srv/tftp/exynos4412-odroidx2.dtb', haltOnFailure=True, mode=0o0664, name='Download Odroid X2 DTB')) # XU, XU4 and HC1 might be missing for older kernels st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos5410-odroidxu.dtb'), workerdest=u'/srv/tftp/exynos5410-odroidxu.dtb', haltOnFailure=False, warnOnFailure=True, flunkOnFailure=False, mode=0o0664, name='Download Odroid XU DTB')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos5422-odroidxu4.dtb'), workerdest=u'/srv/tftp/exynos5422-odroidxu4.dtb', # In case of failure do not halt, do not fail and mark build as warning. # flunkOnFailure is by default True. haltOnFailure=False, warnOnFailure=True, flunkOnFailure=False, mode=0o0664, name='Download Odroid XU4 DTB')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/exynos5422-odroidhc1.dtb'), workerdest=u'/srv/tftp/exynos5422-odroidhc1.dtb', haltOnFailure=False, warnOnFailure=True, flunkOnFailure=False, mode=0o0664, name='Download Odroid HC1 DTB')) st.append(steps.FileDownload( mastersrc=util.Interpolate(mastersrc_dir + '/modules-out.tar.gz'), workerdest='deploy-modules-out.tar.gz', haltOnFailure=True, mode=0o0644, name='Download modules')) return st
def downloadScript(scriptName, **kwargs): """Downloads script with the given name from scripts directory to the current worker""" return [ steps.FileDownload( name="Transferring {} to worker".format(scriptName), mastersrc="maxscale/builders/support/scripts/{}".format( scriptName), workerdest=util.Interpolate( "%(prop:builddir)s/scripts/{}".format(scriptName)), mode=0o755, **kwargs) ]
def _addCaCertsForHttpsGitRemotes(self): """Download in the workers the expected CA certificate files and set them accordingly in the current user Git configuration file. .. warning:: As ``git-lfs`` has reimplemented its own procedure to parse the ".gitconfig" file, you need to pay attention to the features supported by the ``git-lfs`` binary in addition to the ones supported by ``git``. For instance, as of November 2018, ``git-lfs`` does neither support wildcards in the URLs patterns in the ``http`` sections of the Git configuration file nor file paths to CA certificates beginning by ``~`` (to indicate the home directory) contrary to ``git``. """ if not self.buildmaster_setup.private_settings_addendum.additional_git_https_cacerts: raise ValueError(line( """Could not declare custom CA certificates for Git HTTPS remotes if the Buildbot setup does not provide any.""")) for url, cacert_filepath in (self.buildmaster_setup.private_settings_addendum .additional_git_https_cacerts .items()): sanitizedurl = re.sub(r'[^a-zA-Z0-9\.\-\_\:]+', "_", url) cacert_filename_in_worker = "cacert-{}.pem".format(sanitizedurl) self.addStep(steps.FileDownload( name="get custom cacert", description="download custom CA certificate for HTTPS git remotes", haltOnFailure=True, mastersrc=cacert_filepath, workerdest="~/.git-cacerts/{}".format(cacert_filename_in_worker), )) self.addStep(steps.ShellCommand( name="setup custom cacert for git", description="set up the fetched CA certificate for HTTPS git remotes", haltOnFailure=True, command=textwrap.dedent( r""" git config --global "http.${url}.sslCAInfo" \ "${HOME}/.git-cacerts/${cacert_filename_in_worker}" """).strip(), env={ "url": url, "cacert_filename_in_worker": cacert_filename_in_worker, }, ))
def __init__(self): ZcashBaseFactory.__init__(self) self.addSteps([ steps.FileDownload(mastersrc="/home/{{ buildbot_master_user }}/block-107134.tar.xz", workerdest="block-107134.tar.xz"), valgrind('sleep'), valgrind('parameterloading'), valgrind('createjoinsplit'), valgrind('verifyjoinsplit'), valgrind('solveequihash'), valgrind('solveequihash', 2, name='valgrind solveequihash 2 threads'), valgrind('verifyequihash'), valgrind('connectblockslow'), ])
def run(self): def does_state_tar_exist(step): return os.path.exists('flatpak/state-dir.tar') def does_repo_tar_exist(step): return os.path.exists('flatpak/repo.tar') def does_exist(step): return does_state_tar_exist(step) and does_repo_tar_exist(step) self.build.addStepsAfterCurrentStep([ steps.FileDownload( name='download state-dir.tar', haltOnFailure=True, doStepIf=does_state_tar_exist, mastersrc='flatpak/state-dir.tar', workerdest='state-dir.tar', ), steps.FileDownload( name='download repo.tar', haltOnFailure=True, doStepIf=does_repo_tar_exist, mastersrc='flatpak/repo.tar', workerdest='repo.tar', ), steps.ShellSequence( name='unpack archives and clean up', haltOnFailure=True, doStepIf=does_exist, logEnviron=False, commands=[ util.ShellArg(command=['tar', 'xf', 'state-dir.tar', 'repo.tar'], logfile='stdio'), util.ShellArg(command=['rm', '-f', 'state-dir.tar', 'repo.tar'], logfile='stdio'), ], ), ]) return buildbot.process.results.SUCCESS
def build_coverity(): remove_build = steps.RemoveDirectory("build") remove_src = steps.RemoveDirectory("src") create_build = steps.MakeDirectory("build") download_src_archive = steps.FileDownload( mastersrc=util.Property("src_archive"), workerdest="src.tar.xz", workdir="src") extract_src_archive = steps.ShellCommand( name="Extract source archive", command=["tar", "xJf", "src.tar.xz"], workdir="src") cmake_step = steps.CMake(path="../src/", definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) make_step = steps.Compile(command=[ "cov-build", "--dir", "cov-int", "make", "-j", "16", "-l", "32" ], workdir="build", env=env) compress = steps.ShellCommand( command=["tar", "czvf", "gnuradio.tgz", "cov-int"], workdir="build") upload = steps.ShellCommand(command=[ "curl", "--form", "token=" + tokens.coverityToken, "--form", "[email protected]", "--form", "[email protected]", "--form", util.Interpolate("version=%(prop:revision)s"), "--form", util.Interpolate( "description=\"Weekly Buildbot submission for %(prop:branch)s branch \"" ), "https://scan.coverity.com/builds?project=GNURadio" ], workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(remove_src) factory.addStep(create_build) factory.addStep(download_src_archive) factory.addStep(extract_src_archive) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(compress) factory.addStep(upload) return factory
def build_and_test(): remove_build = steps.RemoveDirectory("build") remove_src = steps.RemoveDirectory("src") create_build = steps.MakeDirectory("build") download_src_archive = steps.FileDownload( mastersrc=util.Property("src_archive"), workerdest="src.tar.xz", workdir="src") extract_src_archive = steps.ShellCommand( name="Extract source archive", command=["tar", "xJf", "src.tar.xz"], workdir="src") cmake_step = steps.CMake(path="../src/", definitions=util.Property("cmake_defs", {}), options=util.Property("cmake_opts", []), workdir="build", env=env) @util.renderer def join_make_opts(props): make_opts = props.getProperty("make_opts", []) return ["make"] + make_opts make_step = steps.Compile(command=join_make_opts, workdir="build", env=env) @util.renderer def parse_test_excludes(props): command = ["ctest", "--output-on-failure", "--timeout", "120"] excludes = props.getProperty('test_excludes', []) excludes.append("qtgui") if excludes is not None: command += ["-E", "|".join(excludes)] return command test_step = steps.Test(command=parse_test_excludes, workdir="build") factory = util.BuildFactory() factory.addStep(remove_build) factory.addStep(remove_src) factory.addStep(create_build) factory.addStep(download_src_archive) factory.addStep(extract_src_archive) factory.addStep(cmake_step) factory.addStep(make_step) factory.addStep(test_step) return factory
def downloadAndRunScript(scriptName, args=(), **kwargs): """ Downloads the script to remote location and executes it :param: scriptName name of the local script to execute """ remoteScriptName = util.Interpolate( "%(prop:builddir)s/scripts/{}".format(scriptName)) downloadStep = steps.FileDownload( name="Transferring {} to worker".format(scriptName), mastersrc="maxscale/builders/support/scripts/{}".format(scriptName), workerdest=remoteScriptName, mode=0o755) executeStep = steps.ShellCommand( name="Execute script: {}".format(scriptName), command=[remoteScriptName, *args], timeout=1800, **kwargs) return [downloadStep, executeStep]
def __init__(self): ZcashBaseFactory.__init__(self) self.addSteps([ steps.FileDownload(mastersrc="/home/{{ buildbot_master_user }}/block-107134.tar.xz", workerdest="block-107134.tar.xz"), sh('wget', '-N', 'https://z.cash/downloads/benchmarks/benchmark-200k-UTXOs.tar.xz', name='download benchmark-200k-UTXOs.tar.xz'), time('sleep'), time('parameterloading'), time('createjoinsplit'), time('verifyjoinsplit'), time('solveequihash'), time('solveequihash', 2, name='time solveequihash 2 threads'), time('verifyequihash'), time('validatelargetx'), time('connectblockslow'), time('sendtoaddress', '200k-recv', 0.0009, name='time-sendtoaddress-200k-recv-1'), time('sendtoaddress', '200k-recv', 0.0099, name='time-sendtoaddress-200k-recv-10'), time('sendtoaddress', '200k-recv', 0.0999, name='time-sendtoaddress-200k-recv-100'), time('sendtoaddress', '200k-send', 0.0009, name='time-sendtoaddress-200k-send-1'), time('sendtoaddress', '200k-send', 0.0099, name='time-sendtoaddress-200k-send-10'), time('sendtoaddress', '200k-send', 0.0999, name='time-sendtoaddress-200k-send-100'), time('loadwallet', '200k-recv', name='time-loadwallet-200k-recv'), time('listunspent', '200k-recv', name='time-listunspent-200k-recv'), memory('sleep'), memory('parameterloading'), memory('createjoinsplit'), memory('verifyjoinsplit'), memory('solveequihash'), memory('solveequihash', 2, name='memory solveequihash 2 threads'), memory('verifyequihash'), memory('validatelargetx'), memory('connectblockslow'), memory('sendtoaddress', '200k-recv', 0.0999, name='memory-sendtoaddress-200k-recv-100'), memory('loadwallet', '200k-recv', name='memory-loadwallet-200k-recv'), memory('listunspent', '200k-recv', name='memory-listunspent-200k-recv'), steps.POST(urllib.parse.urljoin(CODESPEED_URL, '/result/add/json/'), data={'json': getPerfJson}, auth=('buildbot', CODESPEED_PASS), verify=CODESPEED_CERT, doStepIf=lambda s: s.getProperty('publish', False), hideStepIf=lambda results, s: results==util.SKIPPED, ), ])
def run(self): self.build.addStepsAfterCurrentStep([ steps.FileDownload( name='download gpg archive', haltOnFailure=True, mastersrc='flatpak/flatpak-gpg.tar.gz', workerdest='flatpak-gpg.tar.gz', ), steps.ShellSequence( name='expand gpg archive', haltOnFailure=True, logEnviron=False, commands=[ util.ShellArg(command=['tar', 'xf', 'flatpak-gpg.tar.gz'], logfile='stdio'), util.ShellArg(command=['rm', '-f', 'flatpak-gpg.tar.gz'], logfile='stdio'), util.ShellArg(command=['gpg2', '--homedir', 'flatpak-gpg', '--list-keys'], logfile='stdio'), ], ), ]) return buildbot.process.results.SUCCESS
from typing import List, Optional, Any, Dict from buildbot import config from buildbot.plugins import steps, util from buildbot.process import buildstep, logobserver from buildbot.process.results import FAILURE, worst_status from twisted.internet import defer from .build_factory import master_file, worker_file download_step = steps.FileDownload( mastersrc=master_file, workerdest=worker_file, mode=755, ) class ScummVMTest(steps.Test): """Test steps that treats a specific warning as errors""" treat_as_error: str = "###################### LINGO: syntax error" def has_error(self, line): return self.treat_as_error in line def evaluateCommand(self, cmd): result = super().evaluateCommand(cmd) if any(self.has_error(line) for line in self.loggedWarnings): return worst_status(FAILURE, result) return result
def __init__(self, pkgbuilddir: str, group: str, pkg_base: str, properties: dict): super().__init__() gpg_sign = properties["gpg_sign"] sshdir = properties["sshdir"] workdir = f"{pkgbuilddir}/{group}/{pkg_base}" if group in ("community", "packages"): workdir += "/trunk" # set initial properties self.addStep( steps.SetProperties(name="set properties from srcinfo", properties=properties)) # find dependencies depends = properties["depends"] if depends is not None: for depends_name in depends: self.addSteps([ steps.SetProperty( name=f"set depends_name to {depends_name}", property="depends_name", value=depends_name, hideStepIf=True, ), FindDependency(name=f"find {depends_name}"), ]) # download build files self.addStep( steps.FileDownload( name="download PKGBUILD", mastersrc=f"{workdir}/PKGBUILD", workerdest="PKGBUILD", )) for src_name in properties["src_names"]: self.addStep( steps.FileDownload( name=f"download {src_name}", mastersrc=f"{workdir}/{src_name}", workerdest=src_name, )) install = properties["install"] if install: self.addStep( steps.FileDownload( name=f"download {install}", mastersrc=f"{workdir}/{install}", workerdest=install, )) # update pkgver, pkgrel self.addSteps([SetPkgver(), SetPkgrel(), Updpkgsums()]) # update git tag revision if properties["git_tag"]: self.addStep(SetTagRevision()) # update git commit revision if properties["git_revision"]: self.addStep(SetCommitRevision()) # build self.addStep(ArchBuild()) # update properties self.addSteps([ Srcinfo(), steps.FileUpload( name="upload updated PKGBUILD", workersrc="PKGBUILD", masterdest=f"{workdir}/PKGBUILD", ), steps.FileUpload( name="upload updated .SRCINFO", workersrc=".SRCINFO", masterdest=f"{workdir}/.SRCINFO", ), steps.SetProperties( name="refresh properties from srcinfo", properties=ArchBuildUtil.srcinfo, ), ]) # upload and optionally sign packages for pkg_name in properties["pkg_names"]: self.addSteps([ steps.SetProperty( name=f"set pkg_name to {pkg_name}", property="pkg_name", value=pkg_name, hideStepIf=True, ), steps.FileUpload( name=f"upload {pkg_name}", workersrc=ArchBuildUtil.pkg, masterdest=ArchBuildUtil.pkg_masterdest, ), MovePackage(name=f"move {pkg_name}"), ]) if gpg_sign: self.addSteps([ GpgSign(name=f"sign {pkg_name}"), steps.FileDownload( name=f"download {pkg_name} sig", mastersrc=ArchBuildUtil.sig_mastersrc, workerdest=ArchBuildUtil.sig_workerdest, ), ]) # update repository self.addStep(RepoAdd(name=f"add {pkg_name}")) # synchronize repository if sshdir: self.addStep(CreateSshfsDirectory()) self.addStep(MountPkgbuildCom(env=ArchBuildUtil.ssh_agent)) self.addStep(RepoSync(env=ArchBuildUtil.ssh_agent)) self.addStep(UnmountPkgbuildCom()) # cleanup self.addStep(Cleanup())