def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([(b.name, b) for b in list(self)
if isinstance(b, Builder)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(bc.name, bc) for bc in new_config.builders])
new_set = set(new_by_name.iterkeys())
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
yield defer.maybeDeferred(
lambda: builder.disownServiceParent())
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
builder.setServiceParent(self)
self.builderNames = self.builders.keys()
metrics.MetricCountEvent.log("num_builders",
len(self.builders),
absolute=True)
# remove unclaimed builds if the builder has been removed from configuration
if len(self.master.config.projects) > 1:
queued_builds = yield self.master.db.buildrequests.getBuildRequestInQueue(
sorted=True)
# TODO: if we are running in multimaster mode with multiple instance of katana we need
# to check for the project key as well
removed_builders = [
b for b in queued_builds if b["buildername"] not in new_set
]
self.removeQueuedBuilds(removed_builders)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = {b.name: b
for b in list(self)
if isinstance(b, Builder)}
old_set = set(old_by_name)
new_by_name = {bc.name: bc
for bc in new_config.builders}
new_set = set(new_by_name)
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
# pylint: disable=cell-var-from-loop
yield defer.maybeDeferred(lambda:
builder.disownServiceParent())
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
yield builder.setServiceParent(self)
self.builderNames = list(self.builders)
yield self.master.data.updates.updateBuilderList(
self.master.masterid,
[util.bytes2unicode(n) for n in self.builderNames])
metrics.MetricCountEvent.log("num_builders",
len(self.builders), absolute=True)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([(b.name, b) for b in list(self)
if isinstance(b, Builder)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(bc.name, bc) for bc in new_config.builders])
new_set = set(new_by_name.iterkeys())
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda: builder.disownServiceParent()))
yield wfd
wfd.getResult()
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
builder.setServiceParent(self)
self.builderNames = self.builders.keys()
metrics.MetricCountEvent.log("num_builders",
len(self.builders),
absolute=True)
timer.stop()
def makeBuilder(self, klass):
dummy_factory = klass([Dummy()])
builder = {
'name': 'b1',
'slavenames': ['s1'],
'builddir': 'b1',
'slavebuilddir': 'b1',
'factory': dummy_factory,
}
builder_status = mock.Mock()
builder = Builder(builder, builder_status)
builder.running = True
builder.db = self.dbc
builder.master_name = 'test_master'
builder.master_incarnation = '12345'
builder.botmaster = mock.Mock()
builder.botmaster.shouldMergeRequests.return_value = True
return builder
def makeBuildStep(basedir, step_class=BuildStep, **kwargs):
bss = setupBuildStepStatus(basedir)
ss = SourceStamp()
setup = {
'name': "builder1",
"slavename": "bot1",
'builddir': "builddir",
'factory': None
}
b0 = Builder(setup, bss.getBuild().getBuilder())
br = BuildRequest("reason", ss)
b = Build([br])
b.setBuilder(b0)
s = step_class(**kwargs)
s.setBuild(b)
s.setStepStatus(bss)
b.setupStatus(bss.getBuild())
s.slaveVersion = fake_slaveVersion
return s
def makeBuildStep(basedir, step_class=BuildStep, **kwargs):
bss = setupBuildStepStatus(basedir)
ss = SourceStamp()
setup = {
'name': "builder1",
"slavename": "bot1",
'builddir': "builddir",
'slavebuilddir': "slavebuilddir",
'factory': None
}
b0 = Builder(setup, bss.getBuild().getBuilder())
b0.botmaster = FakeBotMaster()
br = BuildRequest("reason", ss, 'test_builder')
b = Build([br])
b.setBuilder(b0)
s = step_class(**kwargs)
s.setBuild(b)
s.setStepStatus(bss)
b.build_status = bss.getBuild()
b.setupProperties()
s.slaveVersion = fake_slaveVersion
s.step_status.setText(s.describe(False))
return s
def loadConfig_Builders(self, newBuilderData):
somethingChanged = False
newList = {}
newBuilderNames = []
allBuilders = self.botmaster.builders.copy()
for data in newBuilderData:
name = data['name']
newList[name] = data
newBuilderNames.append(name)
# identify all that were removed
for oldname in self.botmaster.getBuildernames():
if oldname not in newList:
log.msg("removing old builder %s" % oldname)
del allBuilders[oldname]
somethingChanged = True
# announce the change
self.status.builderRemoved(oldname)
# everything in newList is either unchanged, changed, or new
for name, data in newList.items():
old = self.botmaster.builders.get(name)
basedir = data['builddir'] # used on both master and slave
#name, slave, builddir, factory = data
if not old: # new
# category added after 0.6.2
category = data.get('category', None)
log.msg("adding new builder %s for category %s" %
(name, category))
statusbag = self.status.builderAdded(name, basedir, category)
builder = Builder(data, statusbag)
allBuilders[name] = builder
somethingChanged = True
elif old.compareToSetup(data):
# changed: try to minimize the disruption and only modify the
# pieces that really changed
diffs = old.compareToSetup(data)
log.msg("updating builder %s: %s" % (name, "\n".join(diffs)))
statusbag = old.builder_status
statusbag.saveYourself() # seems like a good idea
# TODO: if the basedir was changed, we probably need to make
# a new statusbag
new_builder = Builder(data, statusbag)
new_builder.consumeTheSoulOfYourPredecessor(old)
# that migrates any retained slavebuilders too
# point out that the builder was updated. On the Waterfall,
# this will appear just after any currently-running builds.
statusbag.addPointEvent(["config", "updated"])
allBuilders[name] = new_builder
somethingChanged = True
else:
# unchanged: leave it alone
log.msg("builder %s is unchanged" % name)
pass
if somethingChanged:
sortedAllBuilders = [allBuilders[name] for name in newBuilderNames]
d = self.botmaster.setBuilders(sortedAllBuilders)
return d
return None
def test_interruptConnectionLostShouldRetry(self):
self.master = fakemaster.FakeMaster()
self.master.maybeStartBuildsForSlave = lambda slave: True
self.master.db = fakedb.FakeDBConnector(self)
rows = [fakedb.SourceStampSet(id=1),
fakedb.SourceStamp(id=1, sourcestampsetid=1, codebase='c', branch="az", repository="xz", revision="ww"),
fakedb.Buildset(id=1, reason='because', sourcestampsetid=1),
fakedb.BuildRequest(id=1, buildsetid=1, buildername="bldr", submitted_at=130000)]
yield self.master.db.insertTestData(rows)
self.master.status = master.Status(self.master)
self.master.config.buildbotURL = "baseurl/"
self.scheduler_a = a = FakeTriggerable(name='a')
self.scheduler_b = b = FakeTriggerable(name='b')
def allSchedulers():
return [a, b]
self.master.allSchedulers = allSchedulers
self.factory = factory.BuildFactory()
self.step = trigger.Trigger(schedulerNames=['a'], waitForFinish=True)
self.step.addCompleteLog = lambda x,y: True
self.factory.addStep(self.step)
config_args = dict(name="bldr", slavename="slv", builddir="bdir",
slavebuilddir="sbdir", project='default', factory=self.factory)
builder_config = config.BuilderConfig(**config_args)
self.bldr = Builder(builder_config.name, _addServices=False)
self.bldr.master = self.master
self.bldr.botmaster = self.master.botmaster
mastercfg = config.MasterConfig()
mastercfg.builders = [ builder_config ]
self.bldr.startService()
yield self.bldr.reconfigService(mastercfg)
def newBuild(buildrequests):
self.build = Build(buildrequests)
self.build.setStepFactories([fakebuild.FakeStepFactory(self.step)])
return self.build
self.bldr.config.factory.newBuild = newBuild
self.bldr.notifyRequestsRemoved = lambda x: True
sb = Mock(spec=['isAvailable'], name='test-slave-1')
sb.name = 'test-slave-1'
sb.isAvailable.return_value = 1
sb.slave = Mock()
sb.slave.updateStatusBuildFinished = lambda result, build: result
sb.slave.properties = properties.Properties()
sb.prepare = lambda x, y: True
sb.ping = lambda timeout: True
sb.buildStarted = lambda: True
sb.buildFinished = lambda _: False
sb.setSlaveIdle = lambda: False
sb.remote = Mock()
self.bldr.slaves.append(sb)
self.assertEqual(self.master.db.buildrequests.claims, {})
from buildbot.process import buildrequestdistributor
self.brd = buildrequestdistributor.BuildRequestDistributor(self.master.botmaster)
self.brd.startService()
yield self.brd._maybeStartBuildsOnBuilder(self.bldr)
self.assertEqual(self.master.db.buildrequests.claims[1].brid, 1)
self.build.build_status.saveYourself = lambda: True
self.build.currentStep.start()
self.build.lostRemote()
self.assertEqual(self.master.db.buildrequests.claims, {})
if self.brd.running:
self.brd.stopService()
