def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = dict([(b.name, b) for b in list(self) if isinstance(b, Builder)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(bc.name, bc) for bc in new_config.builders]) new_set = set(new_by_name.iterkeys()) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None yield defer.maybeDeferred( lambda: builder.disownServiceParent()) for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master builder.setServiceParent(self) self.builderNames = self.builders.keys() metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) # remove unclaimed builds if the builder has been removed from configuration if len(self.master.config.projects) > 1: queued_builds = yield self.master.db.buildrequests.getBuildRequestInQueue( sorted=True) # TODO: if we are running in multimaster mode with multiple instance of katana we need # to check for the project key as well removed_builders = [ b for b in queued_builds if b["buildername"] not in new_set ] self.removeQueuedBuilds(removed_builders) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = {b.name: b for b in list(self) if isinstance(b, Builder)} old_set = set(old_by_name) new_by_name = {bc.name: bc for bc in new_config.builders} new_set = set(new_by_name) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None # pylint: disable=cell-var-from-loop yield defer.maybeDeferred(lambda: builder.disownServiceParent()) for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master yield builder.setServiceParent(self) self.builderNames = list(self.builders) yield self.master.data.updates.updateBuilderList( self.master.masterid, [util.bytes2unicode(n) for n in self.builderNames]) metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = dict([(b.name, b) for b in list(self) if isinstance(b, Builder)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(bc.name, bc) for bc in new_config.builders]) new_set = set(new_by_name.iterkeys()) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None wfd = defer.waitForDeferred( defer.maybeDeferred(lambda: builder.disownServiceParent())) yield wfd wfd.getResult() for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master builder.setServiceParent(self) self.builderNames = self.builders.keys() metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) timer.stop()
def makeBuilder(self, klass): dummy_factory = klass([Dummy()]) builder = { 'name': 'b1', 'slavenames': ['s1'], 'builddir': 'b1', 'slavebuilddir': 'b1', 'factory': dummy_factory, } builder_status = mock.Mock() builder = Builder(builder, builder_status) builder.running = True builder.db = self.dbc builder.master_name = 'test_master' builder.master_incarnation = '12345' builder.botmaster = mock.Mock() builder.botmaster.shouldMergeRequests.return_value = True return builder
def makeBuildStep(basedir, step_class=BuildStep, **kwargs): bss = setupBuildStepStatus(basedir) ss = SourceStamp() setup = { 'name': "builder1", "slavename": "bot1", 'builddir': "builddir", 'factory': None } b0 = Builder(setup, bss.getBuild().getBuilder()) br = BuildRequest("reason", ss) b = Build([br]) b.setBuilder(b0) s = step_class(**kwargs) s.setBuild(b) s.setStepStatus(bss) b.setupStatus(bss.getBuild()) s.slaveVersion = fake_slaveVersion return s
def makeBuildStep(basedir, step_class=BuildStep, **kwargs): bss = setupBuildStepStatus(basedir) ss = SourceStamp() setup = { 'name': "builder1", "slavename": "bot1", 'builddir': "builddir", 'slavebuilddir': "slavebuilddir", 'factory': None } b0 = Builder(setup, bss.getBuild().getBuilder()) b0.botmaster = FakeBotMaster() br = BuildRequest("reason", ss, 'test_builder') b = Build([br]) b.setBuilder(b0) s = step_class(**kwargs) s.setBuild(b) s.setStepStatus(bss) b.build_status = bss.getBuild() b.setupProperties() s.slaveVersion = fake_slaveVersion s.step_status.setText(s.describe(False)) return s
def loadConfig_Builders(self, newBuilderData): somethingChanged = False newList = {} newBuilderNames = [] allBuilders = self.botmaster.builders.copy() for data in newBuilderData: name = data['name'] newList[name] = data newBuilderNames.append(name) # identify all that were removed for oldname in self.botmaster.getBuildernames(): if oldname not in newList: log.msg("removing old builder %s" % oldname) del allBuilders[oldname] somethingChanged = True # announce the change self.status.builderRemoved(oldname) # everything in newList is either unchanged, changed, or new for name, data in newList.items(): old = self.botmaster.builders.get(name) basedir = data['builddir'] # used on both master and slave #name, slave, builddir, factory = data if not old: # new # category added after 0.6.2 category = data.get('category', None) log.msg("adding new builder %s for category %s" % (name, category)) statusbag = self.status.builderAdded(name, basedir, category) builder = Builder(data, statusbag) allBuilders[name] = builder somethingChanged = True elif old.compareToSetup(data): # changed: try to minimize the disruption and only modify the # pieces that really changed diffs = old.compareToSetup(data) log.msg("updating builder %s: %s" % (name, "\n".join(diffs))) statusbag = old.builder_status statusbag.saveYourself() # seems like a good idea # TODO: if the basedir was changed, we probably need to make # a new statusbag new_builder = Builder(data, statusbag) new_builder.consumeTheSoulOfYourPredecessor(old) # that migrates any retained slavebuilders too # point out that the builder was updated. On the Waterfall, # this will appear just after any currently-running builds. statusbag.addPointEvent(["config", "updated"]) allBuilders[name] = new_builder somethingChanged = True else: # unchanged: leave it alone log.msg("builder %s is unchanged" % name) pass if somethingChanged: sortedAllBuilders = [allBuilders[name] for name in newBuilderNames] d = self.botmaster.setBuilders(sortedAllBuilders) return d return None
def test_interruptConnectionLostShouldRetry(self): self.master = fakemaster.FakeMaster() self.master.maybeStartBuildsForSlave = lambda slave: True self.master.db = fakedb.FakeDBConnector(self) rows = [fakedb.SourceStampSet(id=1), fakedb.SourceStamp(id=1, sourcestampsetid=1, codebase='c', branch="az", repository="xz", revision="ww"), fakedb.Buildset(id=1, reason='because', sourcestampsetid=1), fakedb.BuildRequest(id=1, buildsetid=1, buildername="bldr", submitted_at=130000)] yield self.master.db.insertTestData(rows) self.master.status = master.Status(self.master) self.master.config.buildbotURL = "baseurl/" self.scheduler_a = a = FakeTriggerable(name='a') self.scheduler_b = b = FakeTriggerable(name='b') def allSchedulers(): return [a, b] self.master.allSchedulers = allSchedulers self.factory = factory.BuildFactory() self.step = trigger.Trigger(schedulerNames=['a'], waitForFinish=True) self.step.addCompleteLog = lambda x,y: True self.factory.addStep(self.step) config_args = dict(name="bldr", slavename="slv", builddir="bdir", slavebuilddir="sbdir", project='default', factory=self.factory) builder_config = config.BuilderConfig(**config_args) self.bldr = Builder(builder_config.name, _addServices=False) self.bldr.master = self.master self.bldr.botmaster = self.master.botmaster mastercfg = config.MasterConfig() mastercfg.builders = [ builder_config ] self.bldr.startService() yield self.bldr.reconfigService(mastercfg) def newBuild(buildrequests): self.build = Build(buildrequests) self.build.setStepFactories([fakebuild.FakeStepFactory(self.step)]) return self.build self.bldr.config.factory.newBuild = newBuild self.bldr.notifyRequestsRemoved = lambda x: True sb = Mock(spec=['isAvailable'], name='test-slave-1') sb.name = 'test-slave-1' sb.isAvailable.return_value = 1 sb.slave = Mock() sb.slave.updateStatusBuildFinished = lambda result, build: result sb.slave.properties = properties.Properties() sb.prepare = lambda x, y: True sb.ping = lambda timeout: True sb.buildStarted = lambda: True sb.buildFinished = lambda _: False sb.setSlaveIdle = lambda: False sb.remote = Mock() self.bldr.slaves.append(sb) self.assertEqual(self.master.db.buildrequests.claims, {}) from buildbot.process import buildrequestdistributor self.brd = buildrequestdistributor.BuildRequestDistributor(self.master.botmaster) self.brd.startService() yield self.brd._maybeStartBuildsOnBuilder(self.bldr) self.assertEqual(self.master.db.buildrequests.claims[1].brid, 1) self.build.build_status.saveYourself = lambda: True self.build.currentStep.start() self.build.lostRemote() self.assertEqual(self.master.db.buildrequests.claims, {}) if self.brd.running: self.brd.stopService()