def __init__(self, master): service.MultiService.__init__(self) self.master = master self.builders = {} self.builderNames = [] # builders maps Builder names to instances of bb.p.builder.Builder, # which is the master-side object that defines and controls a build. # They are added by calling botmaster.addBuilder() from the startup # code. # self.slaves contains a ready BuildSlave instance for each # potential buildslave, i.e. all the ones listed in the config file. # If the slave is connected, self.slaves[slavename].slave will # contain a RemoteReference to their Bot instance. If it is not # connected, that attribute will hold None. self.slaves = {} # maps slavename to BuildSlave self.watchers = {} # self.locks holds the real Lock instances self.locks = {} # self.mergeRequests is the callable override for merging build # requests self.mergeRequests = None # self.prioritizeBuilders is the callable override for builder order # traversal self.prioritizeBuilders = None self.loop = DelegateLoop(self._get_processors) self.loop.setServiceParent(self) self.shuttingDown = False self.lastSlavePortnum = None
class BotMaster(service.MultiService): """This is the master-side service which manages remote buildbot slaves. It provides them with BuildSlaves, and distributes build requests to them.""" debug = 0 reactor = reactor def __init__(self, master): service.MultiService.__init__(self) self.master = master self.builders = {} self.builderNames = [] # builders maps Builder names to instances of bb.p.builder.Builder, # which is the master-side object that defines and controls a build. # They are added by calling botmaster.addBuilder() from the startup # code. # self.slaves contains a ready BuildSlave instance for each # potential buildslave, i.e. all the ones listed in the config file. # If the slave is connected, self.slaves[slavename].slave will # contain a RemoteReference to their Bot instance. If it is not # connected, that attribute will hold None. self.slaves = {} # maps slavename to BuildSlave self.watchers = {} # self.locks holds the real Lock instances self.locks = {} # self.mergeRequests is the callable override for merging build # requests self.mergeRequests = None # self.prioritizeBuilders is the callable override for builder order # traversal self.prioritizeBuilders = None self.loop = DelegateLoop(self._get_processors) self.loop.setServiceParent(self) self.shuttingDown = False self.lastSlavePortnum = None def setMasterName(self, name, incarnation): self.master_name = name self.master_incarnation = incarnation def cleanShutdown(self): if self.shuttingDown: return log.msg("Initiating clean shutdown") self.shuttingDown = True # Wait for all builds to finish l = [] for builder in self.builders.values(): for build in builder.builder_status.getCurrentBuilds(): l.append(build.waitUntilFinished()) if len(l) == 0: log.msg("No running jobs, starting shutdown immediately") self.loop.trigger() d = self.loop.when_quiet() else: log.msg("Waiting for %i build(s) to finish" % len(l)) d = defer.DeferredList(l) d.addCallback(lambda ign: self.loop.when_quiet()) # Flush the eventual queue d.addCallback(eventual.flushEventualQueue) # Finally, shut the whole process down def shutdown(ign): # Double check that we're still supposed to be shutting down # The shutdown may have been cancelled! if self.shuttingDown: # Check that there really aren't any running builds for builder in self.builders.values(): n = len(builder.builder_status.getCurrentBuilds()) if n > 0: log.msg( "Not shutting down, builder %s has %i builds running" % (builder, n)) log.msg("Trying shutdown sequence again") self.shuttingDown = False self.cleanShutdown() return log.msg("Stopping reactor") self.reactor.stop() d.addCallback(shutdown) return d def cancelCleanShutdown(self): if not self.shuttingDown: return log.msg("Cancelling clean shutdown") self.shuttingDown = False def _sortfunc(self, b1, b2): t1 = b1.getOldestRequestTime() t2 = b2.getOldestRequestTime() # If t1 or t2 is None, then there are no build requests, # so sort it at the end if t1 is None: return 1 if t2 is None: return -1 return cmp(t1, t2) def _sort_builders(self, parent, builders): return sorted(builders, self._sortfunc) def _get_processors(self): if self.shuttingDown: return [] builders = self.builders.values() sorter = self.prioritizeBuilders or self._sort_builders try: builders = sorter(self.parent, builders) except: log.msg("Exception prioritizing builders") log.err(Failure()) # leave them in the original order return [b.run for b in builders] def loadConfig_Slaves(self, new_slaves): new_portnum = (self.lastSlavePortnum is not None and self.lastSlavePortnum != self.master.slavePortnum) if new_portnum: # it turns out this is pretty hard.. raise ValueError( "changing slavePortnum in reconfig is not supported") self.lastSlavePortnum = self.master.slavePortnum old_slaves = [ c for c in list(self) if interfaces.IBuildSlave.providedBy(c) ] # identify added/removed slaves. For each slave we construct a tuple # of (name, password, class), and we consider the slave to be already # present if the tuples match. (we include the class to make sure # that BuildSlave(name,pw) is different than # SubclassOfBuildSlave(name,pw) ). If the password or class has # changed, we will remove the old version of the slave and replace it # with a new one. If anything else has changed, we just update the # old BuildSlave instance in place. If the name has changed, of # course, it looks exactly the same as deleting one slave and adding # an unrelated one. old_t = {} for s in old_slaves: old_t[(s.slavename, s.password, s.__class__)] = s new_t = {} for s in new_slaves: new_t[(s.slavename, s.password, s.__class__)] = s removed = [old_t[t] for t in old_t if t not in new_t] added = [new_t[t] for t in new_t if t not in old_t] remaining_t = [t for t in new_t if t in old_t] # removeSlave will hang up on the old bot dl = [] for s in removed: dl.append(self.removeSlave(s)) d = defer.DeferredList(dl, fireOnOneErrback=True) def add_new(res): for s in added: self.addSlave(s) d.addCallback(add_new) def update_remaining(_): for t in remaining_t: old_t[t].update(new_t[t]) d.addCallback(update_remaining) return d def addSlave(self, s): s.setServiceParent(self) s.setBotmaster(self) self.slaves[s.slavename] = s s.pb_registration = self.master.pbmanager.register( self.master.slavePortnum, s.slavename, s.password, self.getPerspective) def removeSlave(self, s): d = s.disownServiceParent() d.addCallback(lambda _: s.pb_registration.unregister()) d.addCallback(lambda _: self.slaves[s.slavename].disconnect()) def delslave(_): del self.slaves[s.slavename] d.addCallback(delslave) return d def slaveLost(self, bot): for name, b in self.builders.items(): if bot.slavename in b.slavenames: b.detached(bot) def getBuildersForSlave(self, slavename): return [b for b in self.builders.values() if slavename in b.slavenames] def getBuildernames(self): return self.builderNames def getBuilders(self): allBuilders = [self.builders[name] for name in self.builderNames] return allBuilders def setBuilders(self, builders): # TODO: remove self.builders and just use the Service hierarchy to # keep track of active builders. We could keep self.builderNames to # retain ordering, if it seems important. self.builders = {} self.builderNames = [] d = defer.DeferredList([ b.disownServiceParent() for b in list(self) if isinstance(b, Builder) ], fireOnOneErrback=True) def _add(ign): log.msg("setBuilders._add: %s %s" % (list(self), builders)) for b in builders: for slavename in b.slavenames: # this is actually validated earlier assert slavename in self.slaves self.builders[b.name] = b self.builderNames.append(b.name) b.setBotmaster(self) b.setServiceParent(self) d.addCallback(_add) d.addCallback(lambda ign: self._updateAllSlaves()) return d def _updateAllSlaves(self): """Notify all buildslaves about changes in their Builders.""" dl = [] for s in self.slaves.values(): d = s.updateSlave() d.addErrback(log.err) dl.append(d) return defer.DeferredList(dl) def shouldMergeRequests(self, builder, req1, req2): """Determine whether two BuildRequests should be merged for the given builder. """ if self.mergeRequests is not None: if callable(self.mergeRequests): return self.mergeRequests(builder, req1, req2) elif self.mergeRequests == False: # To save typing, this allows c['mergeRequests'] = False return False return req1.canBeMergedWith(req2) def getPerspective(self, mind, slavename): sl = self.slaves[slavename] if not sl: return None # record when this connection attempt occurred sl.recordConnectTime() if sl.isConnected(): # duplicate slave - send it to arbitration arb = DuplicateSlaveArbitrator(sl) return arb.getPerspective(mind, slavename) else: log.msg("slave '%s' attaching from %s" % (slavename, mind.broker.transport.getPeer())) return sl def stopService(self): for b in self.builders.values(): b.builder_status.addPointEvent(["master", "shutdown"]) b.builder_status.saveYourself() return service.MultiService.stopService(self) def getLockByID(self, lockid): """Convert a Lock identifier into an actual Lock instance. @param lockid: a locks.MasterLock or locks.SlaveLock instance @return: a locks.RealMasterLock or locks.RealSlaveLock instance """ assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock)) if not lockid in self.locks: self.locks[lockid] = lockid.lockClass(lockid) # if the master.cfg file has changed maxCount= on the lock, the next # time a build is started, they'll get a new RealLock instance. Note # that this requires that MasterLock and SlaveLock (marker) instances # be hashable and that they should compare properly. return self.locks[lockid] def triggerNewBuildCheck(self): # TODO: old name -- should go self.loop.trigger()
class BotMaster(service.MultiService): """This is the master-side service which manages remote buildbot slaves. It provides them with BuildSlaves, and distributes build requests to them.""" debug = 0 reactor = reactor def __init__(self, master): service.MultiService.__init__(self) self.master = master self.builders = {} self.builderNames = [] # builders maps Builder names to instances of bb.p.builder.Builder, # which is the master-side object that defines and controls a build. # They are added by calling botmaster.addBuilder() from the startup # code. # self.slaves contains a ready BuildSlave instance for each # potential buildslave, i.e. all the ones listed in the config file. # If the slave is connected, self.slaves[slavename].slave will # contain a RemoteReference to their Bot instance. If it is not # connected, that attribute will hold None. self.slaves = {} # maps slavename to BuildSlave self.watchers = {} # self.locks holds the real Lock instances self.locks = {} # self.mergeRequests is the callable override for merging build # requests self.mergeRequests = None # self.prioritizeBuilders is the callable override for builder order # traversal self.prioritizeBuilders = None self.loop = DelegateLoop(self._get_processors) self.loop.setServiceParent(self) self.shuttingDown = False self.lastSlavePortnum = None def setMasterName(self, name, incarnation): self.master_name = name self.master_incarnation = incarnation def cleanShutdown(self): if self.shuttingDown: return log.msg("Initiating clean shutdown") self.shuttingDown = True # Wait for all builds to finish l = [] for builder in self.builders.values(): for build in builder.builder_status.getCurrentBuilds(): l.append(build.waitUntilFinished()) if len(l) == 0: log.msg("No running jobs, starting shutdown immediately") self.loop.trigger() d = self.loop.when_quiet() else: log.msg("Waiting for %i build(s) to finish" % len(l)) d = defer.DeferredList(l) d.addCallback(lambda ign: self.loop.when_quiet()) # Flush the eventual queue d.addCallback(eventual.flushEventualQueue) # Finally, shut the whole process down def shutdown(ign): # Double check that we're still supposed to be shutting down # The shutdown may have been cancelled! if self.shuttingDown: # Check that there really aren't any running builds for builder in self.builders.values(): n = len(builder.builder_status.getCurrentBuilds()) if n > 0: log.msg("Not shutting down, builder %s has %i builds running" % (builder, n)) log.msg("Trying shutdown sequence again") self.shuttingDown = False self.cleanShutdown() return log.msg("Stopping reactor") self.reactor.stop() d.addCallback(shutdown) return d def cancelCleanShutdown(self): if not self.shuttingDown: return log.msg("Cancelling clean shutdown") self.shuttingDown = False def _sortfunc(self, b1, b2): t1 = b1.getOldestRequestTime() t2 = b2.getOldestRequestTime() # If t1 or t2 is None, then there are no build requests, # so sort it at the end if t1 is None: return 1 if t2 is None: return -1 return cmp(t1, t2) def _sort_builders(self, parent, builders): return sorted(builders, self._sortfunc) def _get_processors(self): if self.shuttingDown: return [] builders = self.builders.values() sorter = self.prioritizeBuilders or self._sort_builders try: builders = sorter(self.parent, builders) except: log.msg("Exception prioritizing builders") log.err(Failure()) # leave them in the original order return [b.run for b in builders] def loadConfig_Slaves(self, new_slaves): new_portnum = (self.lastSlavePortnum is not None and self.lastSlavePortnum != self.master.slavePortnum) if new_portnum: # it turns out this is pretty hard.. raise ValueError("changing slavePortnum in reconfig is not supported") self.lastSlavePortnum = self.master.slavePortnum old_slaves = [c for c in list(self) if interfaces.IBuildSlave.providedBy(c)] # identify added/removed slaves. For each slave we construct a tuple # of (name, password, class), and we consider the slave to be already # present if the tuples match. (we include the class to make sure # that BuildSlave(name,pw) is different than # SubclassOfBuildSlave(name,pw) ). If the password or class has # changed, we will remove the old version of the slave and replace it # with a new one. If anything else has changed, we just update the # old BuildSlave instance in place. If the name has changed, of # course, it looks exactly the same as deleting one slave and adding # an unrelated one. old_t = {} for s in old_slaves: old_t[(s.slavename, s.password, s.__class__)] = s new_t = {} for s in new_slaves: new_t[(s.slavename, s.password, s.__class__)] = s removed = [old_t[t] for t in old_t if t not in new_t] added = [new_t[t] for t in new_t if t not in old_t] remaining_t = [t for t in new_t if t in old_t] # removeSlave will hang up on the old bot dl = [] for s in removed: dl.append(self.removeSlave(s)) d = defer.DeferredList(dl, fireOnOneErrback=True) def add_new(res): for s in added: self.addSlave(s) d.addCallback(add_new) def update_remaining(_): for t in remaining_t: old_t[t].update(new_t[t]) d.addCallback(update_remaining) return d def addSlave(self, s): s.setServiceParent(self) s.setBotmaster(self) self.slaves[s.slavename] = s s.pb_registration = self.master.pbmanager.register( self.master.slavePortnum, s.slavename, s.password, self.getPerspective) def removeSlave(self, s): d = s.disownServiceParent() d.addCallback(lambda _ : s.pb_registration.unregister()) d.addCallback(lambda _ : self.slaves[s.slavename].disconnect()) def delslave(_): del self.slaves[s.slavename] d.addCallback(delslave) return d def slaveLost(self, bot): for name, b in self.builders.items(): if bot.slavename in b.slavenames: b.detached(bot) def getBuildersForSlave(self, slavename): return [b for b in self.builders.values() if slavename in b.slavenames] def getBuildernames(self): return self.builderNames def getBuilders(self): allBuilders = [self.builders[name] for name in self.builderNames] return allBuilders def setBuilders(self, builders): # TODO: remove self.builders and just use the Service hierarchy to # keep track of active builders. We could keep self.builderNames to # retain ordering, if it seems important. self.builders = {} self.builderNames = [] d = defer.DeferredList([b.disownServiceParent() for b in list(self) if isinstance(b, Builder)], fireOnOneErrback=True) def _add(ign): log.msg("setBuilders._add: %s %s" % (list(self), builders)) for b in builders: for slavename in b.slavenames: # this is actually validated earlier assert slavename in self.slaves self.builders[b.name] = b self.builderNames.append(b.name) b.setBotmaster(self) b.setServiceParent(self) d.addCallback(_add) d.addCallback(lambda ign: self._updateAllSlaves()) return d def _updateAllSlaves(self): """Notify all buildslaves about changes in their Builders.""" dl = [] for s in self.slaves.values(): d = s.updateSlave() d.addErrback(log.err) dl.append(d) return defer.DeferredList(dl) def shouldMergeRequests(self, builder, req1, req2): """Determine whether two BuildRequests should be merged for the given builder. """ if self.mergeRequests is not None: if callable(self.mergeRequests): return self.mergeRequests(builder, req1, req2) elif self.mergeRequests == False: # To save typing, this allows c['mergeRequests'] = False return False return req1.canBeMergedWith(req2) def getPerspective(self, mind, slavename): sl = self.slaves[slavename] if not sl: return None # record when this connection attempt occurred sl.recordConnectTime() if sl.isConnected(): # duplicate slave - send it to arbitration arb = DuplicateSlaveArbitrator(sl) return arb.getPerspective(mind, slavename) else: log.msg("slave '%s' attaching from %s" % (slavename, mind.broker.transport.getPeer())) return sl def stopService(self): for b in self.builders.values(): b.builder_status.addPointEvent(["master", "shutdown"]) b.builder_status.saveYourself() return service.MultiService.stopService(self) def getLockByID(self, lockid): """Convert a Lock identifier into an actual Lock instance. @param lockid: a locks.MasterLock or locks.SlaveLock instance @return: a locks.RealMasterLock or locks.RealSlaveLock instance """ assert isinstance(lockid, (locks.MasterLock, locks.SlaveLock)) if not lockid in self.locks: self.locks[lockid] = lockid.lockClass(lockid) # if the master.cfg file has changed maxCount= on the lock, the next # time a build is started, they'll get a new RealLock instance. Note # that this requires that MasterLock and SlaveLock (marker) instances # be hashable and that they should compare properly. return self.locks[lockid] def triggerNewBuildCheck(self): # TODO: old name -- should go self.loop.trigger()