def masterConfig(self, step): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(step) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def test_latent_workers_start_in_parallel(self): """ If there are two latent workers configured, and two build requests for them, both workers will start substantiating conccurently. """ controllers = [ LatentController('local1'), LatentController('local2'), ] config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local1", "local2"], factory=BuildFactory()), ], 'workers': [controller.worker for controller in controllers], 'protocols': { 'null': {} }, 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Request two builds. for i in range(2): bsid, brids = self.successResultOf( master.data.updates.addBuildset( waited_for=False, builderids=[builder_id], sourcestamps=[ { 'codebase': '', 'repository': '', 'branch': None, 'revision': None, 'project': '' }, ], )) # Check that both workers were requested to start. self.assertEqual(controllers[0].started, True) self.assertEqual(controllers[1].started, True)
def build_triggerer(c, distro, arch, rosdistro, machines, ordered_repos): f = BuildFactory() for repos in ordered_repos: f.addStep( Trigger(schedulerNames=[ t.replace('_', '-') + '-' + rosdistro + '-' + distro + '-' + arch + '-debtrigger' for t in repos ], waitForFinish=True, alwaysRun=True)) # Add to builders c['builders'].append( BuilderConfig(name='build_triggerer' + '_' + rosdistro + '_' + distro + '_' + arch, slavenames=machines, factory=f)) return 'build_triggerer' + '_' + rosdistro + '_' + distro + '_' + arch
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, util c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = BuildFactory() # do a bunch of transfer to exercise the protocol f.addStep(steps.ShellCommand(command=["echo", util.URLForBuild])) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched1", builderNames=["testy1"]), schedulers.ForceScheduler(name="sched2", builderNames=["testy2"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name=name, workernames=["local1"], factory=f) for name in ['testy1', 'testy2'] ] return c
def windows_builder(buildtype, arch): if buildtype == "rtdist": factory = rtdist_factory elif buildtype == "runtime": factory = runtime_factory else: factory = sdk_factory if arch == "amd64": platform = "win_amd64" else: platform = "win32" return BuilderConfig(name='-'.join((buildtype, "windows", arch)), slavenames=config.windows_slaves, factory=factory, properties={"buildtype": buildtype, "arch": arch, "platform": platform})
def test_failed_substantiations_get_requeued(self): """ If a latent worker fails to substantiate, the build request becomes unclaimed. """ controller = LatentController(self, 'local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), {req['buildrequestid'] for req in unclaimed_build_requests}) controller.auto_stop(True)
def test_stalled_substantiation_then_timeout_get_requeued(self): """ If a latent worker substantiate, but not connect, and then be unsubstantiated, the build request becomes unclaimed. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig( name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf( master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # We never start the worker, rather timeout it. master.reactor.advance(controller.worker.missing_timeout) # Flush the errors logged by the failure. self.flushLoggedErrors(defer.TimeoutError) # When the substantiation fails, the buildrequest becomes unclaimed. self.assertEqual( set(brids), set([req['buildrequestid'] for req in unclaimed_build_requests]), ) controller.auto_stop(True)
def getBaseConfig(self): return { 'builders': [ BuilderConfig(name="testy", workernames=["local1", "local2"], factory=BuildFactory( [steps.ShellCommand(command='echo hello')])), ], 'workers': [Worker('local' + str(i), 'pass') for i in range(3)], 'schedulers': [ForceScheduler(name="force", builderNames=["testy"])], 'protocols': { 'null': {} }, 'multiMaster': True, }
def create_config_for_step(self, step): config_dict = { 'builders': [ BuilderConfig(name="builder", workernames=["worker1"], factory=BuildFactory([step]) ), ], 'workers': [self.createLocalWorker('worker1')], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } yield self.setup_master(config_dict) builder_id = yield self.master.data.updates.findBuilderId('builder') return builder_id
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]), schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='sleep 100', name='sleep')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def masterConfig(build_set_summary): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, reporters c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] formatter = MessageFormatter(template='This is a message.') formatter_worker = MessageFormatterMissingWorker(template='No worker.') if build_set_summary: generators_mail = [ BuildSetStatusGenerator(mode='all'), WorkerMissingGenerator(workers='all'), ] generators_pushover = [ BuildSetStatusGenerator(mode='all', message_formatter=formatter), WorkerMissingGenerator(workers=['local1'], message_formatter=formatter_worker), ] else: generators_mail = [ BuildStatusGenerator(mode='all'), WorkerMissingGenerator(workers='all'), ] generators_pushover = [ BuildStatusGenerator(mode='all', message_formatter=formatter), WorkerMissingGenerator(workers=['local1'], message_formatter=formatter_worker), ] c['services'] = [ reporters.MailNotifier("*****@*****.**", generators=generators_mail), reporters.PushoverNotifier('1234', 'abcd', generators=generators_pushover) ] return c
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers, reporters c['schedulers'] = [ schedulers.AnyBranchScheduler( name="sched", builderNames=["testy"])] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] c['services'] = [reporters.MailNotifier("*****@*****.**", mode="all")] return c
def masterConfig(is_list_mkdir=True, is_list_rmdir=True): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers c['schedulers'] = [ schedulers.AnyBranchScheduler(name="sched", builderNames=["testy"]) ] f = BuildFactory() f.addStep( TestCompositeMixinStep(is_list_mkdir=is_list_mkdir, is_list_rmdir=is_list_rmdir)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def as_config(self): factory = BuildFactory(self.steps) properties = self._render_properties() workernames = [w.name for w in self.workers] return BuilderConfig(name=self.name, workernames=workernames, factory=factory, properties=properties, description=self.__doc__, tags=self.tags, env=self.env, locks=self.locks, builddir=str(self.builddir), workerbuilddir=str(self.workerbuilddir), nextWorker=self.next_worker, nextBuild=self.next_build, collapseRequests=self.collapse_requests, canStartBuild=self.can_start_build)
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.Try_Userpass(name="try", builderNames=["testy"], port=8031, userpass=[("alice", "pw1")]) ] f = BuildFactory() f.addStep(steps.ShellCommand(command='echo hello')) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def addProject(self): """ Add project to buildbot_configuration: """ self._all_builder_names = [] # Create builders after we resolve all group_builder_names. for group_name, members in self._raw['groups'].items(): for member_name in members: builder_name = self._getEnvironmentBuilderName(member_name) # Don't add the same builder twice. if builder_name in self._all_builder_names: continue # Create new builder for environment. self._all_builder_names.append(builder_name) steps = self._getSteps(member_name) slaves = self._parent.getSlaves(member_name) builder = BuilderConfig( name=builder_name, slavenames=slaves, category=self._name, nextSlave=self._nextSlave, factory=RunStepsFactory( project=self, steps=steps, environment=self._parent.getStepEnvironment( member_name), ), ) self._parent.addBuilder(builder) # self._parent.addNotifications( # builder=builder_name, # configuration={ # 'email_success' # }, # )) self._addTryBuilders() self._addGateKeepers()
def test_build_stop_with_retry_during_substantiation(self): """ If master is shutting down during latent worker substantiating, the build becomes retry. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder = master.botmaster.builders['testy'] builder_id = self.successResultOf(builder.getBuilderId()) # Trigger a buildrequest _, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # Stop the build build = builder.getBuild(0) build.stopBuild('no reason', results=RETRY) # Indicate that the worker can't start an instance. controller.start_instance(False) dbdict = yield master.db.builds.getBuildByNumber(builder_id, 1) self.assertEqual(RETRY, dbdict['results']) self.assertEqual( set(brids), {req['buildrequestid'] for req in unclaimed_build_requests} ) controller.auto_stop(True) self.flushLoggedErrors(LatentWorkerFailedToSubstantiate)
def make_builders(self, name, factory, build_category=None, build_for=None, build_requires=(), next_slave=None): """Produce the builders for the given factory. :param name: base name for the builders. :param factory: :class:`BuildFactory` instance :param build_requires: list of capability requirements that the buildslave must match to run a builder from the factory. :param build_for: a dict whose keys are capability names and values are corresponding :class:`VersionFilter` instances. :param build_category: forwarded to :class:`BuilderConfig` instantiation :param next_slave: forwarded to :class:`BuilderConfig` instantiation as ``nextSlave``. """ slavenames = self.filter_slaves_by_requires(build_requires) if not slavenames: # buildbot does not allow builders with empty list of slaves return () base_conf = dict(name=name, category=build_category, factory=factory, nextSlave=next_slave, slavenames=list(slavenames)) # forward requirement in the build properties if build_requires: base_conf['properties'] = dict( build_requires=[str(req) for req in build_requires]) preconfs = [base_conf] for cap_name, cap_vf in factory.build_for.items(): preconfs = self.dispatch_builders_by_capability( preconfs, cap_name, cap_vf) return [BuilderConfig(**conf) for conf in preconfs]
def get_config_single_step(self, step): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = BuildFactory() f.addStep( steps.FileUpload(workersrc="dir/noexist_path", masterdest="master_dest")) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def macosx_builder(osxver): if osxver in ('10.6', '10.7', '10.8'): workernames = config.macosx_10_6_workers buildsteps = build_steps_10_6 elif osxver.startswith('11.'): workernames = config.macosx_11_0_workers buildsteps = build_steps_11_0 else: workernames = config.macosx_10_9_workers buildsteps = build_steps_10_9 factory = BuildFactory() for step in buildsteps: factory.addStep(step) return BuilderConfig(name='macosx' + ('-' + osxver if osxver else ''), workernames=workernames, factory=factory, properties={"osxtarget": osxver, "buildtype": "sdk"})
def create_master_config(self): self.num_reconfig += 1 from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.steps.shell import ShellCommand from buildbot.util.service import BuildbotService class MyShellCommand(ShellCommand): def getResultSummary(self): service = self.master.service_manager.namedServices[ 'myService'] return dict(step="num reconfig: %d" % (service.num_reconfig, )) class MyService(BuildbotService): name = "myService" def reconfigService(self, num_reconfig): self.num_reconfig = num_reconfig return defer.succeed(None) config_dict = { 'builders': [ BuilderConfig(name="builder", workernames=["worker1"], factory=BuildFactory( [MyShellCommand(command='echo hei')])), ], 'workers': [self.createLocalWorker('worker1')], 'protocols': { 'null': {} }, # Disable checks about missing scheduler. 'multiMaster': True, 'db_url': 'sqlite://', # we need to make sure reconfiguration uses the same URL 'services': [MyService(num_reconfig=self.num_reconfig)] } if self.num_reconfig == 3: config_dict['services'].append( MyService(name="myService2", num_reconfig=self.num_reconfig)) return config_dict
def generate_builder(target: TestTarget) -> BuilderConfig: factory = util.BuildFactory() factory.addStep(download_step) factory.addStep( GenerateStartMovieCommands( directory=target.directory, game_id=target.game_id, debugflags=target.debugflags, name=f"Generate commands: {target.builder_name}", command=[ "cat", os.path.join(target.directory, "test_scripts.txt") ], haltOnFailure=True, **default_step_kwargs, )) return BuilderConfig(name=target.builder_name, workernames=["director-worker"], factory=factory)
def masterConfig(usePTY): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import steps, schedulers c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] f = BuildFactory() f.addStep( steps.ShellCommand( command= 'if [ -t 1 ] ; then echo in a terminal; else echo "not a terminal"; fi', usePTY=usePTY)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def macosx_builder(buildtype, osxver): if buildtype == "sdk": name = '-'.join((buildtype, "macosx" + osxver)) else: name = '-'.join((buildtype, "macosx")) if buildtype == "rtdist": factory = rtdist_factory elif buildtype == "runtime": factory = runtime_factory else: factory = sdk_factory return BuilderConfig(name=name, slavenames=config.macosx_slaves, factory=factory, properties={ "osxtarget": osxver, "buildtype": buildtype })
def masterConfig(use_interpolation): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps, util c['services'] = [ FakeSecretReporter('http://example.com/hook', auth=('user', Interpolate('%(secret:httppasswd)s'))) ] c['schedulers'] = [ schedulers.ForceScheduler(name="force", builderNames=["testy"]) ] c['secretsProviders'] = [ FakeSecretStorage( secretdict={ "foo": "secretvalue", "something": "more", 'httppasswd': 'myhttppasswd' }) ] f = BuildFactory() if use_interpolation: if os.name == "posix": # on posix we can also check whether the password was passed to the command command = Interpolate( 'echo %(secret:foo)s | ' + 'sed "s/secretvalue/The password was there/"') else: command = Interpolate('echo %(secret:foo)s') else: command = ['echo', util.Secret('foo')] f.addStep(steps.ShellCommand(command=command)) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f) ] return c
def addSimpleBuilder(self, name, buildername, category, repourl, builderconfig, sos, sbranch, bparams): """Private. Add a single builder on the given OS type for the given repo and branch. """ factory = BuildFactory() factory.addStep( Git(repourl=repourl, mode='full', submodules=True, method='copy', branch=sbranch, getDescription={'tags': True})) if "tag" in builderconfig and not (builderconfig["tag"] is None): stag = builderconfig["tag"].encode('ascii', 'ignore') factory.addStep( ShellCommand(command=['git', 'checkout', stag], workdir="build", description="checkout tag")) # Delegate all knowlege of how to build to a script called buildshim in the project # directory. Call this script in nine standardize steps. It should ignore steps that it doesn't need. # Pass the step name as the first arg, and if params was given in the json for this builder, pass that as the # second arg. for step in [ "patch", "install_deps", "configure", "compile", "check", "package", "upload", "compile_extra", "uninstall_deps" ]: factory.addStep( ShellCommand(command=["./buildshim", step, bparams], description=step, haltOnFailure=True)) self['builders'].append( BuilderConfig(name=buildername, slavenames=self._os2slaves[sos], factory=factory, category=category)) return factory
def test_worker_accepts_builds_after_failure(self): """ If a latent worker fails to substantiate, the worker is still able to accept jobs. """ controller = LatentController('local') config_dict = { 'builders': [ BuilderConfig(name="testy", workernames=["local"], factory=BuildFactory(), ), ], 'workers': [controller.worker], 'protocols': {'null': {}}, # Disable checks about missing scheduler. 'multiMaster': True, } master = self.getMaster(config_dict) builder_id = self.successResultOf( master.data.updates.findBuilderId('testy')) # Trigger a buildrequest bsid, brids = self.createBuildrequest(master, [builder_id]) unclaimed_build_requests = [] self.successResultOf(master.mq.startConsuming( lambda key, request: unclaimed_build_requests.append(request), ('buildrequests', None, 'unclaimed'))) # The worker fails to substantiate. controller.start_instance( Failure(TestException("substantiation failed"))) # Flush the errors logged by the failure. self.flushLoggedErrors(TestException) # If the worker started again after the failure, then the retry logic will have # already kicked in to start a new build on this (the only) worker. We check that # a new instance was requested, which indicates that the worker # accepted the build. self.assertEqual(controller.started, True) controller.auto_stop(True)
def masterConfig(): c = {} from buildbot.config import BuilderConfig from buildbot.process.factory import BuildFactory from buildbot.plugins import schedulers, steps c['schedulers'] = [ schedulers.ForceScheduler( name="force", builderNames=["testy"])] c['secretsProviders'] = [FakeSecretStorage( secretdict={"foo": "bar", "something": "more"})] f = BuildFactory() f.addStep(steps.ShellCommand(command=Interpolate('echo %(secrets:foo)s'))) c['builders'] = [ BuilderConfig(name="testy", workernames=["local1"], factory=f)] return c
def __init__(self, name, workers): self.name = utils.name(name) self.workers = workers self.build = BuildFactory() BuildmasterConfig['builders'].append(BuilderConfig( name=self.name, factory=self.build, slavenames=self.workers)) BuildmasterConfig['schedulers'].append(ForceScheduler( name=scheduler_name(self, 'force'), builderNames=[self.name], branch=FixedParameter(name="reason", default=""), reason=FixedParameter(name="reason", default="Forced build"), revision=FixedParameter(name="revision", default=""), repository=FixedParameter(name="repository", default=""), project=FixedParameter(name="project", default=""), properties=[], username=FixedParameter(name="username", default="WebUI")))
def __init__(self, name, branch, slavenames=None, factory=None, category=None): BuilderConfig.__init__(self, name=name, slavenames=slavenames, factory=factory, category=category) self.branch = branch
def __init__(self, name, **kwargs): BuilderConfig.__init__(self, name, builddir = "work/builders/" + name, slavenames = get_slavenames(), **kwargs)