def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] f = BuildFactory() # do a bunch of transfer to exercise the protocol f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt")) f.addStep(steps.StringDownload("filecontent2", workerdest="dir/file2.txt")) f.addStep( steps.FileUpload(workersrc="dir/file2.txt", masterdest="master.txt")) f.addStep( steps.FileDownload(mastersrc="master.txt", workerdest="dir/file3.txt")) f.addStep(steps.DirectoryUpload(workersrc="dir", masterdest="dir")) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def MakeWinBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep(ShellCommand( name = 'revision', command = ['cmd', '/c', 'revision.cmd'], workdir = 'build/os/win32', env = { 'SCHAT_REVISION': Property('got_revision') }, haltOnFailure = True, )) f.addStep(ShellCommand( name = 'qmake', command = ['qmake', '-r'], haltOnFailure = True, )) f.addStep(Compile( command = ['jom', '-j3', '/NOLOGO'], haltOnFailure = True, )) f.addStep(ShellCommand( name = 'nsis', command = ['cmd', '/c', 'nsis.cmd'], workdir = 'build/os/win32', env = { 'SCHAT_SIGN_FILE': schat_passwords.SIGN_FILE, 'SCHAT_SIGN_PASSWORD': schat_passwords.SIGN_PASSWORD, 'SCHAT_VERSION': SCHAT_VERSION, 'SCHAT_REVISION': Property('got_revision') }, haltOnFailure = True, logEnviron = False, )) f.addStep(FileUpload( mode = 0644, slavesrc = WithProperties('os/win32/out/schat2-%(version)s.exe'), masterdest = UploadFileName('schat2-%(version)s%(suffix)s.exe'), url = WithProperties('https://download.schat.me/schat2/snapshots/%(version)s/r%(got_revision)s/schat2-%(version)s%(suffix)s.exe'), ))
def create_factory(): factory = BuildFactory() factory.addSteps(common.initTargetProperty()) factory.addStep( steps.Trigger(name="Call the 'build' scheduler", schedulerNames=['build'], waitForFinish=True, haltOnFailure=True, copy_properties=COMMON_PROPERTIES, set_properties={ 'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'), 'box': 'ubuntu_bionic_libvirt', 'try_already_running': 'no', })) factory.addStep( steps.Trigger(name="Call the 'run_performance_test' scheduler", schedulerNames=['run_performance_test_trigger'], waitForFinish=True, copy_properties=COMMON_PROPERTIES, set_properties={ 'test_branch': util.Property('branch'), 'host': 'max-tst-01', })) return factory
def makeCleanOldBuildsFactory(): """ Remove build results older than 14 days. """ # FIXME we shouldn't hard code this (DRY) basedir = r'/srv/buildmaster/data' path = os.path.join(basedir, "private_html") factory = BuildFactory() factory.addStep( MasterShellCommand([ 'find', path, '-type', 'f', '-mtime', '+14', '-exec', 'unlink', '{}', ';', '-print' ], description=['Removing', 'old', 'results'], descriptionDone=['Remove', 'old', 'results'], name='remove-old-results')) # Vagrant tutorial boxes are created on the Vagrant slave, and # uploaded to S3. However, the boxes must be kept on the slave # for running subsequent tests # (flocker/acceptance/vagrant/centos-7/zfs and # flocker/installed-package/vagrant/centos-7). This means there is # not an obvious place to remove the boxes. So, we periodically # cleanup old boxes here. "Old" is the number of days passed as # parameter to script. factory.addStep( ShellCommand( command=['python', '/home/buildslave/remove-old-boxes.py', '14'], description=['Removing', 'old', 'boxes'], descriptionDone=['Remove', 'old', 'boxes'], name='remove-old-boxes')) return factory
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, reporters c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] notifier = reporters.PushoverNotifier('1234', 'abcd', mode="all", watchedWorkers=['local1'], messageFormatter=MessageFormatter(template='This is a message.'), messageFormatterMissingWorker=MessageFormatterMissingWorker( template='No worker.')) c['services'] = [ reporters.MailNotifier("*****@*****.**", mode="all"), notifier ] return c
def clean_chroot(cls, slaves): base_dir = "../../../" factory = BuildFactory() step_clean_chroot = ShellCommand( command = ["scripts/chroot-cleaner", WithProperties('%(branch)s') ], workdir=base_dir, haltOnFailure = True, flunkOnFailure = True, description = "cleanup chroots", descriptionDone = "cleaned chroots", name = "cleanup chroots", interruptSignal="TERM") factory.addStep(step_clean_chroot) builder = BuilderConfig( name = "clean_chroot", category = "clean", slavenames = [ i.slavename for i in slaves ], properties = { "pkgname": "clean_chroot" }, factory = factory, builddir = "builders/clean_chroot", slavebuilddir = "builders/clean_chroot", ) return builder
def triggeredBuildIsNotCreated(): f2 = BuildFactory() f2.addStep(steps.ShellCommand(command="echo 'hello'")) def nextBuild(*args, **kwargs): return defer.succeed(None) return setupTriggerConfiguration(f2, nextBuild=nextBuild)
def masterConfig(use_with=False): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] c['secretsProviders'] = [ FakeSecretStorage(secretdict={ "foo": "bar", "something": "more" }) ] f = BuildFactory() if use_with: secrets_list = [("pathA", Interpolate('%(secret:something)s'))] with f.withSecrets(secrets_list): f.addStep( steps.ShellCommand(command=Interpolate('echo %(secret:foo)s'))) else: f.addSteps( [steps.ShellCommand(command=Interpolate('echo %(secret:foo)s'))], withSecrets=[("pathA", Interpolate('%(secret:something)s'))]) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def MakeDebBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep( ShellCommand( name='revision', command=['bash', 'revision.sh'], workdir='build/os/ubuntu', haltOnFailure=True, env={'SCHAT_REVISION': Property('got_revision')}, )) f.addStep( ShellCommand( name='deb', command=['bash', 'build.sh'], workdir='build/os/ubuntu', env={'SCHAT_VERSION': SCHAT_VERSION}, haltOnFailure=True, )) f.addStep( FileUpload( mode=0644, slavesrc=WithProperties( 'os/ubuntu/deb/schat2_%(version)s-1~%(codename)s_%(arch)s.deb' ), masterdest=UploadFileName( 'schat2_%(version)s-1~%(codename)s%(suffix)s_%(arch)s.deb'), url=WithProperties( 'https://download.schat.me/schat2/snapshots/%(version)s/r%(got_revision)s/schat2_%(version)s-1~%(codename)s%(suffix)s_%(arch)s.deb' ), ))
def MakeMacBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep( ShellCommand( name='revision', command=['bash', 'revision.sh'], workdir='build/os/macosx', haltOnFailure=True, env={'SCHAT_REVISION': Property('got_revision')}, )) f.addStep( ShellCommand( name='dmg', command=['bash', 'deploy.sh'], workdir='build/os/macosx', haltOnFailure=True, )) f.addStep( FileUpload( mode=0644, slavesrc=WithProperties( 'os/macosx/dmg/SimpleChat2-%(version)s.dmg'), masterdest=UploadFileName('SimpleChat2-%(version)s%(suffix)s.dmg'), url=WithProperties( 'https://download.schat.me/schat2/snapshots/%(version)s/r%(got_revision)s/SimpleChat2-%(version)s%(suffix)s.dmg' ), ))
def masterConfig(extra_config): c = {} from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config # each time c.update(extra_config) return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] # note that as of December 2018, the vault docker image default to kv # version 2 to be enabled by default c['secretsProviders'] = [HashiCorpVaultSecretProvider( vaultToken='my_vaulttoken', vaultServer="http://localhost:8200", apiVersion=2 )] f = BuildFactory() f.addStep(ShellCommand(command=[Interpolate('echo %(secret:key)s')])) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def masterConfig(use_interpolation): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps, util c['services'] = [FakeSecretReporter('http://example.com/hook', auth=('user', Interpolate('%(secret:httppasswd)s')))] c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] c['secretsProviders'] = [FakeSecretStorage( secretdict={"foo": "bar", "something": "more", 'httppasswd': 'myhttppasswd'})] f = BuildFactory() if use_interpolation: if os.name == "posix": # on posix we can also check whether the password was passed to the command command = Interpolate('echo %(secret:foo)s | sed "s/bar/The password was there/"') else: command = Interpolate('echo %(secret:foo)s') else: command = ['echo', util.Secret('foo')] f.addStep(steps.ShellCommand(command=command)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def masterConfig(num_concurrent, extra_steps=None): if extra_steps is None: extra_steps = [] c = {} c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] triggereables = [] for i in range(num_concurrent): c['schedulers'].append( schedulers.Triggerable( name="trigsched" + str(i), builderNames=["build"])) triggereables.append("trigsched" + str(i)) f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) f.addStep(steps.Trigger(schedulerNames=triggereables, waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) f2 = BuildFactory() f2.addStep(steps.ShellCommand(command='echo ola')) for step in extra_steps: f2.addStep(step) c['builders'] = [ BuilderConfig(name="testy", workernames=["marathon0"], factory=f), BuilderConfig(name="build", workernames=["marathon" + str(i) for i in range(num_concurrent)], factory=f2)] url = os.environ.get('BBTEST_MARATHON_URL') creds = os.environ.get('BBTEST_MARATHON_CREDS') if creds is not None: user, password = creds.split(":") else: user = password = None masterFQDN = os.environ.get('masterFQDN') marathon_extra_config = { } c['workers'] = [ MarathonLatentWorker('marathon' + str(i), url, user, password, 'buildbot/buildbot-worker:master', marathon_extra_config=marathon_extra_config, masterFQDN=masterFQDN) for i in range(num_concurrent) ] # un comment for debugging what happens if things looks locked. # c['www'] = {'port': 8080} # if the masterFQDN is forced (proxy case), then we use 9989 default port # else, we try to find a free port if masterFQDN is not None: c['protocols'] = {"pb": {"port": "tcp:9989"}} else: c['protocols'] = {"pb": {"port": "tcp:0"}} return c
def masterConfig(): c = {} from buildbot.worker import Worker from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['workers'] = [Worker("local1", "localpw")] c['protocols'] = {'pb': {'port': 'tcp:0'}} c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['status'] = [] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['db'] = {'db_url': "sqlite:///state.sqlite"} c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config each time c.update(BuildmasterConfig) return c
def createFactory(): factory = BuildFactory() factory.addSteps(common.initTargetProperty()) factory.addStep(steps.Trigger( name="Call the 'build' scheduler. Build CentOS", schedulerNames=['build'], waitForFinish=True, haltOnFailure=True, copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES, set_properties={ "box": util.Property("box"), 'try_already_running': 'yes', "target": util.Property("target"), 'virtual_builder_name': util.Interpolate('Build for %(prop:box)s'), } )) factory.addStep(steps.Trigger( name="Call the 'run_test_snapshot' scheduler. Run functional tests", schedulerNames=['run_test_snapshot'], waitForFinish=True, copy_properties=COMMON_BUILD_AND_TEST_SNAPSHOT_PROPERTIES, set_properties={ "box": util.Property("box"), "target": util.Property("target"), "test_branch": util.Property("branch"), "test_set": common.renderTestSet, "backend_ssl": util.Property("backend_ssl"), "use_valgrind": util.Property("use_valgrind"), } )) return factory
def triggeredBuildIsNotCreated(): f2 = BuildFactory() f2.addStep(steps.ShellCommand(command="echo 'hello'")) def nextBuild(*args, **kwargs): return defer.succeed(None) return setupTriggerConfiguration(f2, nextBuild=nextBuild)
def makeCleanOldBuildsFactory(): """ Remove build results older than 14 days. """ # FIXME we shouldn't hard code this (DRY) basedir = r'/srv/buildmaster/data' path = os.path.join(basedir, "private_html") factory = BuildFactory() factory.addStep(MasterShellCommand( ['find', path, '-type', 'f', '-mtime', '+14', '-exec', 'unlink', '{}', ';', '-print'], description=['Removing', 'old', 'results'], descriptionDone=['Remove', 'old', 'results'], name='remove-old-results')) # Vagrant tutorial boxes are created on the Vagrant slave, and # uploaded to S3. However, the boxes must be kept on the slave # for running subsequent tests # (flocker/acceptance/vagrant/centos-7/zfs and # flocker/installed-package/vagrant/centos-7). This means there is # not an obvious place to remove the boxes. So, we periodically # cleanup old boxes here. "Old" is the number of days passed as # parameter to script. factory.addStep(ShellCommand( command=['python', '/home/buildslave/remove-old-boxes.py', '14'], description=['Removing', 'old', 'boxes'], descriptionDone=['Remove', 'old', 'boxes'], name='remove-old-boxes')) return factory
def setupTriggerConfiguration(triggeredFactory, nextBuild=None): c = {} c['schedulers'] = [ schedulers.Triggerable(name="trigsched", builderNames=["triggered"]), schedulers.AnyBranchScheduler(name="sched", builderNames=["main"]) ] f = BuildFactory() f.addStep( steps.Trigger(schedulerNames=['trigsched'], waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) mainBuilder = BuilderConfig(name="main", workernames=["local1"], factory=f) triggeredBuilderKwargs = { 'name': "triggered", 'workernames': ["local1"], 'factory': triggeredFactory } if nextBuild is not None: triggeredBuilderKwargs['nextBuild'] = nextBuild triggeredBuilder = BuilderConfig(**triggeredBuilderKwargs) c['builders'] = [mainBuilder, triggeredBuilder] return c
def masterConfig(extra_config): c = {} from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config # each time c.update(extra_config) return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, reporters c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] notifier = reporters.PushoverNotifier('1234', 'abcd', mode="all", watchedWorkers=['local1'], messageFormatter=MessageFormatter(template='This is a message.'), messageFormatterMissingWorker=MessageFormatterMissingWorker( template='No worker.')) c['services'] = [ reporters.MailNotifier("*****@*****.**", mode="all"), notifier ] return c
def MakeWinLegacyBuilder(): f = BuildFactory() f.addSteps(svn_co_legacy) f.addStep(ShellCommand( name = 'revision', command = ['cmd', '/c', 'revision.cmd'], workdir = 'build/os/win32', env = { 'SCHAT_VERSION': SCHAT_VERSION_LEGACY, 'SCHAT_REVISION': Property('got_revision') }, haltOnFailure = True, )) f.addStep(ShellCommand( name = 'qmake', command = ['qmake', '-r'], haltOnFailure = True, )) f.addStep(Compile( command = ['jom', '-j3', '/NOLOGO'], haltOnFailure = True, )) f.addStep(ShellCommand( name = 'nsis', command = ['cmd', '/c', 'nsis.cmd'], workdir = 'build/os/win32', env = { 'SCHAT_SIGN_FILE': schat_passwords.SIGN_FILE, 'SCHAT_SIGN_PASSWORD': schat_passwords.SIGN_PASSWORD, 'SCHAT_VERSION': SCHAT_VERSION_LEGACY, 'SCHAT_REVISION': Property('got_revision') }, haltOnFailure = True, logEnviron = False, )) f.addStep(FileUpload( mode = 0644, slavesrc = WithProperties('os/win32/out/schat-%(version_legacy)s.%(got_revision)s.exe'), masterdest = UploadFileNameLegacy('schat-%(version_legacy)s.%(got_revision)s.exe'), ))
def masterGlobConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers from buildbot.steps.worker import CompositeStepMixin class CustomStep(steps.BuildStep, CompositeStepMixin): @defer.inlineCallbacks def run(self): content = yield self.getFileContentFromWorker( "dir/file1.txt", abandonOnFailure=True) assert content == "filecontent" return SUCCESS c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt")) f.addStep( steps.StringDownload("filecontent2", workerdest="dir/notafile1.txt")) f.addStep(steps.StringDownload("filecontent2", workerdest="dir/only1.txt")) f.addStep( steps.MultipleFileUpload( workersrcs=["dir/file*.txt", "dir/not*.txt", "dir/only?.txt"], masterdest="dest/", glob=True)) f.addStep(CustomStep()) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers from buildbot.plugins import util lock = util.MasterLock("lock") c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy", "testy2", "testy3"]), ] f = BuildFactory() lockstep = LockedStep(locks=[lock.access('exclusive')]) f.addStep(lockstep) # assert lockstep._factory.buildStep() == lockstep._factory.buildStep() c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f), BuilderConfig(name="testy2", workernames=["local1"], factory=f), BuilderConfig(name="testy3", workernames=["local1"], factory=f)] return c
def create_factory(config, mode="incremental"): factory = BuildFactory() factory.addStep( Git(repourl=config["repo"], alwaysUseLatest=True, mode=mode)) return factory
def make_arm(): f = BuildFactory() f.addStep( GitNoBranch(repourl="https://github.com/dolphin-emu/dolphin.git", progress=True, mode="incremental")) f.addStep( ShellCommand(command=["mkdir", "-p", "build"], logEnviron=False, description="mkbuilddir", descriptionDone="mkbuilddir")) f.addStep( ShellCommand(command="cmake -DENABLE_QT=OFF -GNinja ..", workdir="build/build", description="configuring", descriptionDone="configure", haltOnFailure=True)) f.addStep( Compile(command=["ninja"], workdir="build/build", description="building", descriptionDone="build", haltOnFailure=True)) f.addStep( Test(command=["ninja", "unittests"], workdir="build/build", description="testing", descriptionDone="test", haltOnFailure=True)) return f
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] f = BuildFactory() # do a bunch of transfer to exercise the protocol f.addStep(steps.StringDownload("filecontent", workerdest="dir/file1.txt")) f.addStep(steps.StringDownload("filecontent2", workerdest="dir/file2.txt")) f.addStep( steps.FileUpload(workersrc="dir/file2.txt", masterdest="master.txt")) f.addStep( steps.FileDownload(mastersrc="master.txt", workerdest="dir/file3.txt")) f.addStep(steps.DirectoryUpload(workersrc="dir", masterdest="dir")) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def masterConfig(addFailure=False): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.Triggerable(name="trigsched", builderNames=["build"]), schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) f.addStep( steps.Trigger(schedulerNames=['trigsched'], waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) f2 = BuildFactory() f2.addStep(steps.ShellCommand(command='echo ola')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f), BuilderConfig(name="build", workernames=["local1"], factory=f2) ] if addFailure: f3 = BuildFactory() f3.addStep(steps.ShellCommand(command='false')) c['builders'].append( BuilderConfig(name="build2", workernames=["local1"], factory=f3)) c['builders'].append( BuilderConfig(name="build3", workernames=["local1"], factory=f2)) c['schedulers'][0] = schedulers.Triggerable( name="trigsched", builderNames=["build", "build2", "build3"]) return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers from buildbot.plugins import util lock = util.MasterLock("lock") c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy", "testy2", "testy3"]), ] f = BuildFactory() lockstep = LockedStep(locks=[lock.access('exclusive')]) f.addStep(lockstep) # assert lockstep._factory.buildStep() == lockstep._factory.buildStep() c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f), BuilderConfig(name="testy2", workernames=["local1"], factory=f), BuilderConfig(name="testy3", workernames=["local1"], factory=f)] return c
def setupTriggerConfiguration(triggeredFactory, nextBuild=None): c = {} c['schedulers'] = [ schedulers.Triggerable( name="trigsched", builderNames=["triggered"]), schedulers.AnyBranchScheduler( name="sched", builderNames=["main"])] f = BuildFactory() f.addStep(steps.Trigger(schedulerNames=['trigsched'], waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) mainBuilder = BuilderConfig(name="main", workernames=["local1"], factory=f) triggeredBuilderKwargs = {'name': "triggered", 'workernames': ["local1"], 'factory': triggeredFactory} if nextBuild is not None: triggeredBuilderKwargs['nextBuild'] = nextBuild triggeredBuilder = BuilderConfig(**triggeredBuilderKwargs) c['builders'] = [mainBuilder, triggeredBuilder] return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] c['secretsProviders'] = [FakeSecretStorage( secretdict={"foo": "bar", "something": "more"})] f = BuildFactory() if os.name == "posix": f.addStep(steps.ShellCommand(command=Interpolate( 'echo %(secret:foo)s | sed "s/bar/The password was there/"'))) else: f.addStep(steps.ShellCommand(command=Interpolate( 'echo %(secret:foo)s'))) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def masterConfig(num_concurrent, extra_steps=None): if extra_steps is None: extra_steps = [] c = {} c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] triggereables = [] for i in range(num_concurrent): c['schedulers'].append( schedulers.Triggerable( name="trigsched" + str(i), builderNames=["build"])) triggereables.append("trigsched" + str(i)) f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) f.addStep(steps.Trigger(schedulerNames=triggereables, waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) f2 = BuildFactory() f2.addStep(steps.ShellCommand(command='echo ola')) for step in extra_steps: f2.addStep(step) c['builders'] = [ BuilderConfig(name="testy", workernames=["marathon0"], factory=f), BuilderConfig(name="build", workernames=["marathon" + str(i) for i in range(num_concurrent)], factory=f2)] url = os.environ.get('BBTEST_MARATHON_URL') creds = os.environ.get('BBTEST_MARATHON_CREDS') if creds is not None: user, password = creds.split(":") else: user = password = None masterFQDN = os.environ.get('masterFQDN') marathon_extra_config = { } c['workers'] = [ MarathonLatentWorker('marathon' + str(i), url, user, password, 'buildbot/buildbot-worker:master', marathon_extra_config=marathon_extra_config, masterFQDN=masterFQDN) for i in range(num_concurrent) ] # un comment for debugging what happens if things looks locked. # c['www'] = {'port': 8080} # if the masterFQDN is forced (proxy case), then we use 9989 default port # else, we try to find a free port if masterFQDN is not None: c['protocols'] = {"pb": {"port": "tcp:9989"}} else: c['protocols'] = {"pb": {"port": "tcp:0"}} return c
def masterConfig(use_with=False): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] c['secretsProviders'] = [FakeSecretStorage( secretdict={"foo": "bar", "something": "more"})] f = BuildFactory() if use_with: secrets_list = [("pathA", Interpolate('%(secret:something)s'))] with f.withSecrets(secrets_list): f.addStep(steps.ShellCommand(command=Interpolate('echo %(secret:foo)s'))) else: f.addSteps([steps.ShellCommand(command=Interpolate('echo %(secret:foo)s'))], withSecrets=[("pathA", Interpolate('%(secret:something)s'))]) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def masterConfig(secret_specifier): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] # note that as of December 2018, the vault docker image default to kv # version 2 to be enabled by default c['secretsProviders'] = [ HashiCorpVaultSecretProvider(vaultToken='my_vaulttoken', vaultServer="http://localhost:8200", apiVersion=2) ] f = BuildFactory() f.addStep( ShellCommand( command=Interpolate('echo {} | base64'.format(secret_specifier)))) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def masterConfig(): global num_reconfig num_reconfig += 1 c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.schedulers.forcesched import ForceScheduler from buildbot.steps.shell import ShellCommand from buildbot.util.service import BuildbotService class MyShellCommand(ShellCommand): def getResultSummary(self): service = self.master.service_manager.namedServices['myService'] return dict(step=u"num reconfig: %d" % (service.num_reconfig, )) class MyService(BuildbotService): name = "myService" def reconfigService(self, num_reconfig): self.num_reconfig = num_reconfig return defer.succeed(None) c['schedulers'] = [ForceScheduler(name="force", builderNames=["testy"])] f = BuildFactory() f.addStep(MyShellCommand(command='echo hei')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] c['services'] = [MyService(num_reconfig=num_reconfig)] if num_reconfig == 3: c['services'].append( MyService(name="myService2", num_reconfig=num_reconfig)) return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] c['secretsProviders'] = [ FakeSecretStorage(secretdict={ "foo": "bar", "something": "more" }) ] f = BuildFactory() if os.name == "posix": f.addStep( steps.ShellCommand(command=Interpolate( 'echo %(secret:foo)s | sed "s/bar/The password was there/"'))) else: f.addStep( steps.ShellCommand(command=Interpolate('echo %(secret:foo)s'))) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def masterConfig(): c = {} from buildbot.worker import Worker from buildbot.config import BuilderConfig from buildbot.process.buildstep import BuildStep from buildbot.process.factory import BuildFactory from buildbot.process import results class MyBuildStep(BuildStep): def start(self): self.finished(results.SUCCESS) c['workers'] = [Worker("local1", "localpw")] c['protocols'] = {'pb': {'port': 'tcp:0'}} c['change_source'] = [] c['schedulers'] = [] # filled in above f1 = BuildFactory() f1.addStep(MyBuildStep(name='one')) f1.addStep(MyBuildStep(name='two')) c['builders'] = [ BuilderConfig(name="a", workernames=["local1"], factory=f1), ] c['status'] = [] c['title'] = "test" c['titleURL'] = "test" c['buildbotURL'] = "http://localhost:8010/" c['db'] = {'db_url': "sqlite:///state.sqlite"} c['mq'] = {'debug': True} # test wants to influence the config, but we still return a new config each time c.update(BuildmasterConfig) return c
def triggerRunsForever(): f2 = BuildFactory() f2.addStep( steps.ShellCommand(command="\n".join( ['while :', 'do', ' echo "sleeping";', ' sleep 1;' 'done']))) return setupTriggerConfiguration(f2)
def getFactory(self, branch, combo): factory = BuildFactory() factory.addStep( Git(repourl=self._project_git_repo, mode='copy', branch=branch)) factory.addStep( EmacsCompile(command=["make", "clean", "all"])) return factory
def masterConfig(num_concurrent, extra_steps=None): if extra_steps is None: extra_steps = [] c = {} c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] triggereables = [] for i in range(num_concurrent): c['schedulers'].append( schedulers.Triggerable( name="trigsched" + str(i), builderNames=["build"])) triggereables.append("trigsched" + str(i)) f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) f.addStep(steps.Trigger(schedulerNames=triggereables, waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) f2 = BuildFactory() f2.addStep(steps.ShellCommand(command='echo ola')) for step in extra_steps: f2.addStep(step) c['builders'] = [ BuilderConfig(name="testy", workernames=["hyper0"], factory=f), BuilderConfig(name="build", workernames=["hyper" + str(i) for i in range(num_concurrent)], factory=f2)] hyperconfig = workerhyper.Hyper.guess_config() if isinstance(hyperconfig, string_types): hyperconfig = json.load(open(hyperconfig)) hyperhost, hyperconfig = hyperconfig['clouds'].items()[0] masterFQDN = os.environ.get('masterFQDN') c['workers'] = [ HyperLatentWorker('hyper' + str(i), 'passwd', hyperhost, hyperconfig['accesskey'], hyperconfig[ 'secretkey'], 'buildbot/buildbot-worker:master', masterFQDN=masterFQDN) for i in range(num_concurrent) ] # un comment for debugging what happens if things looks locked. # c['www'] = {'port': 8080} # if the masterFQDN is forced (proxy case), then we use 9989 default port # else, we try to find a free port if masterFQDN is not None: c['protocols'] = {"pb": {"port": "tcp:9989"}} else: c['protocols'] = {"pb": {"port": "tcp:0"}} return c
def masterConfig(num_concurrent, extra_steps=None): if extra_steps is None: extra_steps = [] c = {} c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] triggereables = [] for i in range(num_concurrent): c['schedulers'].append( schedulers.Triggerable(name="trigsched" + str(i), builderNames=["build"])) triggereables.append("trigsched" + str(i)) f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) f.addStep( steps.Trigger(schedulerNames=triggereables, waitForFinish=True, updateSourceStamp=True)) f.addStep(steps.ShellCommand(command='echo world')) f2 = BuildFactory() f2.addStep(steps.ShellCommand(command='echo ola')) for step in extra_steps: f2.addStep(step) c['builders'] = [ BuilderConfig(name="testy", workernames=["hyper0"], factory=f), BuilderConfig( name="build", workernames=["hyper" + str(i) for i in range(num_concurrent)], factory=f2) ] hyperconfig = workerhyper.Hyper.guess_config() if isinstance(hyperconfig, string_types): hyperconfig = json.load(open(hyperconfig)) hyperhost, hyperconfig = hyperconfig['clouds'].items()[0] masterFQDN = os.environ.get('masterFQDN') c['workers'] = [ HyperLatentWorker('hyper' + str(i), 'passwd', hyperhost, hyperconfig['accesskey'], hyperconfig['secretkey'], 'buildbot/buildbot-worker:master', masterFQDN=masterFQDN) for i in range(num_concurrent) ] # un comment for debugging what happens if things looks locked. # c['www'] = {'port': 8080} # if the masterFQDN is forced (proxy case), then we use 9989 default port # else, we try to find a free port if masterFQDN is not None: c['protocols'] = {"pb": {"port": "tcp:9989"}} else: c['protocols'] = {"pb": {"port": "tcp:0"}} return c
def getFactory(self): factory = BuildFactory() for step in self.factorySteps: if type(step) is list: raise Exception('Error: build step is list') if step is None: raise Exception('Error: build step is None') factory.addStep(step) return factory
def triggerRunsForever(): f2 = BuildFactory() f2.addStep(steps.ShellCommand(command="\n".join(['while :', 'do', ' echo "sleeping";', ' sleep 1;' 'done']))) return setupTriggerConfiguration(f2)
def makeCleanOldResources(): """ Remove cloud instances (from acceptance and client tests) and cloud volumes more than two hours old. """ factory = BuildFactory() factory.addStep(CleanAcceptanceInstances(lag=timedelta(hours=2))) factory.addStep(CleanVolumes(lag=timedelta(minutes=30))) return factory
def makeCleanOldResources(): """ Remove cloud instances (from acceptance and client tests) and cloud volumes more than two hours old. """ factory = BuildFactory() factory.addStep(CleanAcceptanceInstances(lag=timedelta(hours=2))) factory.addStep(CleanVolumes(lag=timedelta(minutes=30))) return factory
def MakeObsBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep(ShellCommand( name = 'upload', command = ['bash', 'os/obs/obs-upload.sh', Property('project'), WithProperties('%(version)s.%(got_revision)s')], haltOnFailure = True, logEnviron = False )) return f
def get_buildsteps(working_dir): build_steps = BuildFactory() repo = Git(repourl="https://github.com/mlakewood/Buildbot-rollout.git", branch='master') virt_env = working_dir + '/virt/lib' slave_python = working_dir + '/virt/bin/python' env = {"LD_LIBRARY_PATH": virt_env} build_steps.addStep(repo) # # Remove the Virtual Environment from the last run # command = "rm -Rf %s/virt" % (working_dir) # build_steps.addStep(ShellCommand(workdir=working_dir, description="Clear Virtualenv", command=command.split(" "))) # # Create the virtual environment for the build # command = "virtualenv virt" # build_steps.addStep(ShellCommand(workdir=working_dir, description="Create Virtualenv", command=command.split(" "))) # # Pip install the python packages from requirements.txt # command = '%s/virt/bin/pip install -r requirements.txt' % working_dir # build_steps.addStep(ShellCommand(workdir=working_dir, description="Install packages", command=command.split(" "))) # Run the tests through coverage to get test coverage at the same time command = "../virt/bin/coverage run --include=src -m unittest discover -vf tests" build_steps.addStep( ShellCommand(workdir=working_dir + '/rollout', description="rollout Unit Tests", command=command.split(" "))) # Output the coverage report command = "virt/bin/coverage report --omit=*tests* -m" build_steps.addStep( ShellCommand(workdir=working_dir, description="API Unit Test Coverage Report", command=command.split(" "))) # Run pylint. P command = "pylint %s/rollout --rcfile=rollout/.pylintrc" % (working_dir) build_steps.addStep( PyLint(workdir=working_dir, description="API pylint", command=command.split(" "))) command = "./jslint js/*" build_steps.addStep( ShellCommand(workdir=working_dir + '/rollout', description="Insight JSLint code", command=command)) return build_steps
def test_addStep_deprecated(self): """ Passing keyword arguments to L{BuildFactory.addStep} is deprecated, but pass the arguments to the first argument, to construct a step. """ factory = BuildFactory() factory.addStep(BuildStep) self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) warnings = self.flushWarnings([self.test_addStep_deprecated]) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0]['category'], DeprecationWarning)
def test_addStep_deprecated(self): """ Passing keyword arguments to L{BuildFactory.addStep} is deprecated, but pass the arguments to the first argument, to construct a step. """ factory = BuildFactory() factory.addStep(BuildStep) self.assertEqual(factory.steps, [_BuildStepFactory(BuildStep)]) warnings = self.flushWarnings([self.test_addStep_deprecated]) self.assertEqual(len(warnings), 1) self.assertEqual(warnings[0]['category'], DeprecationWarning)
def build_factory(): factory = BuildFactory() # Step 1: check out the source factory.addStep(Git(repourl='git://github.com/buildbot/pyflakes.git', mode='copy')) # Step 2: run the tests # (note that this will require that 'trial' is installed) factory.addStep(ShellCommand(command=['trial', 'pyflakes'])) return factory
def ros_debbuild(c, job_name, packages, url, distro, arch, rosdistro, version, machines, othermirror, keys, trigger_pkgs = None): gbp_args = ['-uc', '-us', '--git-ignore-branch', '--git-ignore-new', '--git-verbose', '--git-dist='+distro, '--git-arch='+arch, '-j4'] f = BuildFactory() # Remove the build directory. f.addStep( RemoveDirectory( name = job_name+'-clean', dir = Interpolate('%(prop:workdir)s'), hideStepIf = success, ) ) # Check out the repository master branch, since releases are tagged and not branched f.addStep( Git( repourl = url, branch = 'master', alwaysUseLatest = True, # this avoids broken builds when schedulers send wrong tag/rev mode = 'full' # clean out old versions ) ) # Update the cowbuilder f.addStep( ShellCommand( command = ['cowbuilder-update.py', distro, arch] + keys, hideStepIf = success ) ) # Need to build each package in order for package in packages: debian_pkg = 'ros-'+rosdistro+'-'+package.replace('_','-') # debian package name (ros-groovy-foo) branch_name = 'debian/'+debian_pkg+'_%(prop:release_version)s_'+distro # release branch from bloom deb_name = debian_pkg+'_%(prop:release_version)s'+distro final_name = debian_pkg+'_%(prop:release_version)s-%(prop:datestamp)s'+distro+'_'+arch+'.deb' # Check out the proper tag. Use --force to delete changes from previous deb stamping f.addStep( ShellCommand( haltOnFailure = True, name = package+'-checkout', command = ['git', 'checkout', Interpolate(branch_name), '--force'], hideStepIf = success ) ) # Download script for building the source deb f.addStep( FileDownload( name = job_name+'-grab-build-source-deb-script', mastersrc = 'scripts/build_source_deb.py', slavedest = Interpolate('%(prop:workdir)s/build_source_deb.py'), mode = 0755, hideStepIf = success ) )
def getBasicFactory(self, branch): factory = BuildFactory() _emacs_prop = 'EMACS=%(slave/binaries/emacs:-emacs)s' factory.addStep( Git(repourl=self._project_git_repo, mode='copy', branch=branch, logEnviron=False)) factory.addStep( EmacsCompile(command=["make", WithProperties(_emacs_prop), "clean", "all"], logEnviron=False)) return factory
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c["schedulers"] = [schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"])] f = BuildFactory() f.addStep(steps.MasterShellCommand(command="echo hello")) c["builders"] = [BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
class Job(object): """The base class for all job types. It doesn't do much by itself, except for creating a new Builder and a new Force Scheduler so that users can forcibly trigger a new build. This class can be used as a context manager although, by default, it does nothing. """ def __init__(self, name, workers): self.name = utils.name(name) self.workers = workers self.build = BuildFactory() BuildmasterConfig['builders'].append(BuilderConfig( name=self.name, factory=self.build, slavenames=self.workers)) BuildmasterConfig['schedulers'].append(ForceScheduler( name=scheduler_name(self, 'force'), builderNames=[self.name], branch=FixedParameter(name="reason", default=""), reason=FixedParameter(name="reason", default="Forced build"), revision=FixedParameter(name="revision", default=""), repository=FixedParameter(name="repository", default=""), project=FixedParameter(name="project", default=""), properties=[], username=FixedParameter(name="username", default="WebUI"))) def __enter__(self): return self def __exit__(self, type, value, traceback): pass def add_step(self, step): """Adds a build step on this Job.""" self.build.addStep(step) def trigger(self, job_or_jobs): """Adds a build step which triggers execution of another job.""" if type(job_or_jobs) is list: self.add_step(Trigger( schedulerNames=[scheduler_name(j, 'trigger') for j in job_or_jobs], waitForFinish=True)) else: self.add_step(Trigger( schedulerNames=[scheduler_name(job_or_jobs, 'trigger')], waitForFinish=True))
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, util c["schedulers"] = [schedulers.ForceScheduler(name="force", builderNames=["testy"])] f = BuildFactory() # do a bunch of transfer to exercise the protocol f.addStep(steps.ShellCommand(command=["echo", util.URLForBuild])) c["builders"] = [BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def config_for_master_command(self, **kwargs): c = {} c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.MasterShellCommand(**kwargs)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def MakeObsBuilder(): f = BuildFactory() f.addSteps(svn_co) f.addStep( ShellCommand(name='upload', command=[ 'bash', 'os/obs/obs-upload.sh', Property('project'), WithProperties('%(version)s.%(got_revision)s') ], haltOnFailure=True, logEnviron=False)) return f
def testGoodStep(self): f = BuildFactory() f.addStep(SetProperty(command=["echo", "value"], property="propname")) ss = SourceStamp() req = FakeBuildRequest("Testing", ss, None) b = f.newBuild([req]) b.build_status = FakeBuildStatus() b.slavebuilder = FakeSlaveBuilder() # This shouldn't raise an exception b.setupBuild(None)
def testGoodStep(self): f = BuildFactory() f.addStep(SetProperty(command=["echo", "value"], property="propname")) ss = SourceStamp() req = FakeBuildRequest("Testing", ss, None) b = f.newBuild([req]) b.build_status = FakeBuildStatus() b.slavebuilder = FakeSlaveBuilder() # This shouldn't raise an exception b.setupBuild(None)