def reconfigServiceWithBuildbotConfig(self, new_config): timer = metrics.Timer( "ChangeManager.reconfigServiceWithBuildbotConfig") timer.start() removed, added = util.diffSets(set(self), new_config.change_sources) if removed or added: log.msg("adding %d new changesources, removing %d" % (len(added), len(removed))) for src in removed: yield defer.maybeDeferred(src.disownServiceParent) for src in added: yield src.setServiceParent(self) num_sources = len(list(self)) assert num_sources == len(new_config.change_sources) metrics.MetricCountEvent.log("num_sources", num_sources, absolute=True) # reconfig any newly-added change sources, as well as existing yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig( self, new_config) timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config): timer = metrics.Timer( "ChangeManager.reconfigServiceWithBuildbotConfig") timer.start() removed, added = util.diffSets( set(self), new_config.change_sources) if removed or added: log.msg("adding %d new changesources, removing %d" % (len(added), len(removed))) for src in removed: yield src.deactivate() yield defer.maybeDeferred( src.disownServiceParent) for src in added: yield src.setServiceParent(self) num_sources = len(list(self)) assert num_sources == len(new_config.change_sources) metrics.MetricCountEvent.log("num_sources", num_sources, absolute=True) # reconfig any newly-added change sources, as well as existing yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self, new_config) timer.stop()
def reconfigServiceSlaves(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceSlaves") timer.start() # arrange slaves by name old_by_name = dict([ (s.slavename, s) for s in list(self) if interfaces.IBuildSlave.providedBy(s) ]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([ (s.slavename, s) for s in new_config.slaves ]) new_set = set(new_by_name.iterkeys()) # calculate new slaves, by name, and removed slaves removed_names, added_names = util.diffSets(old_set, new_set) # find any slaves for which the fully qualified class name has # changed, and treat those as an add and remove for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new slaves, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: slave = old_by_name[n] del self.slaves[n] slave.master = None slave.botmaster = None wfd = defer.waitForDeferred( defer.maybeDeferred(lambda : slave.disownServiceParent())) yield wfd wfd.getResult() for n in added_names: slave = new_by_name[n] slave.setServiceParent(self) slave.botmaster = self slave.master = self.master self.slaves[n] = slave metrics.MetricCountEvent.log("num_slaves", len(self.slaves), absolute=True) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = dict([(b.name, b) for b in list(self) if isinstance(b, Builder)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(bc.name, bc) for bc in new_config.builders]) new_set = set(new_by_name.iterkeys()) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None yield defer.maybeDeferred( lambda: builder.disownServiceParent()) for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master builder.setServiceParent(self) self.builderNames = self.builders.keys() metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) # remove unclaimed builds if the builder has been removed from configuration if len(self.master.config.projects) > 1: queued_builds = yield self.master.db.buildrequests.getBuildRequestInQueue( sorted=True) # TODO: if we are running in multimaster mode with multiple instance of katana we need # to check for the project key as well removed_builders = [ b for b in queued_builds if b["buildername"] not in new_set ] self.removeQueuedBuilds(removed_builders) timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name.iterkeys()) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = dict([(s.name, s) for s in new_config_attr]) elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError("config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name.iterkeys()) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] yield child.disownServiceParent() for n in added_names: child = new_by_name[n] yield child.setServiceParent(self) # get a list of child services to reconfigure reconfigurable_services = [svc for svc in self] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError( "%r: child %r should have a defined name attribute", self, svc) config_sibling = new_by_name.get(svc.name) yield svc.reconfigServiceWithSibling(config_sibling)
def reconfigServiceSlaves(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceSlaves") timer.start() # arrange slaves by name old_by_name = dict([(s.slavename, s) for s in list(self) if interfaces.IBuildSlave.providedBy(s)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(s.slavename, s) for s in new_config.slaves]) new_set = set(new_by_name.iterkeys()) # calculate new slaves, by name, and removed slaves removed_names, added_names = util.diffSets(old_set, new_set) # find any slaves for which the fully qualified class name has # changed, and treat those as an add and remove for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new slaves, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: slave = old_by_name[n] del self.slaves[n] slave.master = None slave.botmaster = None wfd = defer.waitForDeferred( defer.maybeDeferred(lambda: slave.disownServiceParent())) yield wfd wfd.getResult() for n in added_names: slave = new_by_name[n] slave.setServiceParent(self) slave.botmaster = self slave.master = self.master self.slaves[n] = slave metrics.MetricCountEvent.log("num_slaves", len(self.slaves), absolute=True) timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name.iterkeys()) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = dict([(s.name, s) for s in new_config_attr]) elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError("config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name.iterkeys()) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] yield child.disownServiceParent() for n in added_names: child = new_by_name[n] yield child.setServiceParent(self) # get a list of child services to reconfigure reconfigurable_services = [svc for svc in self] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError("%r: child %r should have a defined name attribute", self, svc) config_sibling = new_by_name.get(svc.name) yield svc.reconfigServiceWithSibling(config_sibling)
def reconfigServiceSlaves(self, new_config): timer = metrics.Timer("BuildSlaveManager.reconfigServiceSlaves") timer.start() # first we deconfigure everything to let the slaves register again yield self.master.data.updates.deconfigureAllBuidslavesForMaster( self.master.masterid) # arrange slaves by name old_by_name = dict([(s.slavename, s) for s in list(self) if interfaces.IBuildSlave.providedBy(s)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(s.slavename, s) for s in new_config.slaves]) new_set = set(new_by_name.iterkeys()) # calculate new slaves, by name, and removed slaves removed_names, added_names = util.diffSets(old_set, new_set) # find any slaves for which the fully qualified class name has # changed, and treat those as an add and remove for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new slaves, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: slave = old_by_name[n] del self.slaves[n] slave.master = None yield slave.disownServiceParent() for n in added_names: slave = new_by_name[n] yield slave.setServiceParent(self) self.slaves[n] = slave metrics.MetricCountEvent.log("num_slaves", len(self.slaves), absolute=True) timer.stop()
def reconfigServiceSlaves(self, new_config): timer = metrics.Timer("BuildSlaveManager.reconfigServiceSlaves") timer.start() # first we deconfigure everything to let the slaves register again yield self.master.data.updates.deconfigureAllBuidslavesForMaster(self.master.masterid) # arrange slaves by name old_by_name = dict([(s.slavename, s) for s in list(self) if IBuildSlave.providedBy(s)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(s.slavename, s) for s in new_config.slaves]) new_set = set(new_by_name.iterkeys()) # calculate new slaves, by name, and removed slaves removed_names, added_names = util.diffSets(old_set, new_set) # find any slaves for which the fully qualified class name has # changed, and treat those as an add and remove for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new slaves, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: slave = old_by_name[n] del self.slaves[n] slave.master = None yield slave.disownServiceParent() for n in added_names: slave = new_by_name[n] yield slave.setServiceParent(self) self.slaves[n] = slave metrics.MetricCountEvent.log("num_slaves", len(self.slaves), absolute=True) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = dict([(b.name, b) for b in list(self) if isinstance(b, Builder)]) old_set = set(old_by_name) new_by_name = dict([(bc.name, bc) for bc in new_config.builders]) new_set = set(new_by_name) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None # pylint: disable=cell-var-from-loop yield defer.maybeDeferred(lambda: builder.disownServiceParent()) for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master yield builder.setServiceParent(self) self.builderNames = list(self.builders) yield self.master.data.updates.updateBuilderList( self.master.masterid, [util.ascii2unicode(n) for n in self.builderNames]) metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = {b.name: b for b in list(self) if isinstance(b, Builder)} old_set = set(old_by_name) new_by_name = {bc.name: bc for bc in new_config.builders} new_set = set(new_by_name) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None # pylint: disable=cell-var-from-loop yield defer.maybeDeferred(lambda: builder.disownServiceParent()) for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master yield builder.setServiceParent(self) self.builderNames = list(self.builders) yield self.master.data.updates.updateBuilderList( self.master.masterid, [util.bytes2unicode(n) for n in self.builderNames]) metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = dict([ (b.name, b) for b in list(self) if isinstance(b, Builder) ]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([ (bc.name, bc) for bc in new_config.builders ]) new_set = set(new_by_name.iterkeys()) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None wfd = defer.waitForDeferred( defer.maybeDeferred(lambda : builder.disownServiceParent())) yield wfd wfd.getResult() for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master builder.setServiceParent(self) self.builderNames = self.builders.keys() metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) timer.stop()
def reconfigServiceBuilders(self, new_config): timer = metrics.Timer("BotMaster.reconfigServiceBuilders") timer.start() # arrange builders by name old_by_name = dict([(b.name, b) for b in list(self) if isinstance(b, Builder)]) old_set = set(old_by_name.iterkeys()) new_by_name = dict([(bc.name, bc) for bc in new_config.builders]) new_set = set(new_by_name.iterkeys()) # calculate new builders, by name, and removed builders removed_names, added_names = util.diffSets(old_set, new_set) if removed_names or added_names: log.msg("adding %d new builders, removing %d" % (len(added_names), len(removed_names))) for n in removed_names: builder = old_by_name[n] del self.builders[n] builder.master = None builder.botmaster = None wfd = defer.waitForDeferred( defer.maybeDeferred(lambda: builder.disownServiceParent())) yield wfd wfd.getResult() for n in added_names: builder = Builder(n) self.builders[n] = builder builder.botmaster = self builder.master = self.master builder.setServiceParent(self) self.builderNames = self.builders.keys() metrics.MetricCountEvent.log("num_builders", len(self.builders), absolute=True) timer.stop()
def reconfigService(self, new_config): timer = metrics.Timer("ChangeManager.reconfigService") timer.start() removed, added = util.diffSets( set(self), new_config.change_sources) if removed or added: log.msg("adding %d new changesources, removing %d" % (len(added), len(removed))) for src in removed: wfd = defer.waitForDeferred( defer.maybeDeferred( src.disownServiceParent)) yield wfd wfd.getResult() src.master = None for src in added: src.master = self.master src.setServiceParent(self) num_sources = len(list(self)) assert num_sources == len(new_config.change_sources) metrics.MetricCountEvent.log("num_sources", num_sources, absolute=True) # reconfig any newly-added change sources, as well as existing wfd = defer.waitForDeferred( config.ReconfigurableServiceMixin.reconfigService(self, new_config)) yield wfd wfd.getResult() timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config): timer = metrics.Timer("SchedulerManager.reconfigServiceWithBuildbotConfig") timer.start() old_by_name = dict((sch.name, sch) for sch in self) old_set = set(old_by_name.iterkeys()) new_by_name = new_config.schedulers new_set = set(new_by_name.iterkeys()) removed_names, added_names = util.diffSets(old_set, new_set) # find any schedulers that don't know how to reconfig, and, if they # have changed, add them to both removed and added, so that we # run the new version. While we're at it, find any schedulers whose # fully qualified class name has changed, and consider those a removal # and re-add as well. for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'): if old != new: removed_names.add(n) added_names.add(n) # removals first for sch_name in removed_names: log.msg("removing scheduler '%s'" % (sch_name,)) sch = old_by_name[sch_name] yield defer.maybeDeferred(lambda: sch.disownServiceParent()) sch.master = None # .. then additions for sch_name in added_names: log.msg("adding scheduler '%s'" % (sch_name,)) sch = new_by_name[sch_name] # get the scheduler's objectid class_name = '%s.%s' % (sch.__class__.__module__, sch.__class__.__name__) objectid = yield self.master.db.state.getObjectId( sch.name, class_name) # set up the scheduler sch.objectid = objectid sch.master = self.master # *then* attacah and start it yield sch.setServiceParent(self) metrics.MetricCountEvent.log("num_schedulers", len(list(self)), absolute=True) # reconfig any newly-added schedulers, as well as existing yield util_service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self, new_config) timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = {s.name: s for s in new_config_attr} elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError("config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove # While we're at it find any service that don't know how to reconfig, # and, if they have changed, add them to both removed and added, so that we # run the new version for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'): if old != new: removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] # disownServiceParent calls stopService after removing the relationship # as child might use self.master.data to stop itself, its better to stop it first # (this is related to the fact that self.master is found by recursively looking at self.parent # for a master) yield child.stopService() # it has already called, so do not call it again child.stopService = lambda: None yield child.disownServiceParent() # HACK: we still keep a reference to the master for some cleanup tasks which are not waited by # to stopService (like the complex worker disconnection mechanism) # http://trac.buildbot.net/ticket/3583 child.parent = self.master for n in added_names: child = new_by_name[n] # setup service's objectid if hasattr(child, 'objectid'): class_name = '%s.%s' % (child.__class__.__module__, child.__class__.__name__) objectid = yield self.master.db.state.getObjectId( child.name, class_name) child.objectid = objectid yield defer.maybeDeferred(child.setServiceParent, self) # As the services that were just added got # reconfigServiceWithSibling called by # setServiceParent->startService, # we avoid calling it again by selecting # in reconfigurable_services, services # that were not added just now reconfigurable_services = [ svc for svc in self if svc.name not in added_names ] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError( "{}: child {} should have a defined name attribute".format( self, svc)) config_sibling = new_by_name.get(svc.name) try: yield svc.reconfigServiceWithSibling(config_sibling) except NotImplementedError: # legacy support. Its too painful to transition old code to new Service life cycle # so we implement switch of child when the service raises NotImplementedError # Note this means that self will stop, and sibling will take ownership # means that we have a small time where the service is unavailable. yield svc.disownServiceParent() config_sibling.objectid = svc.objectid yield config_sibling.setServiceParent(self)
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name.iterkeys()) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = dict([(s.name, s) for s in new_config_attr]) elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError("config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name.iterkeys()) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove # While we're at it find any service that don't know how to reconfig, # and, if they have changed, add them to both removed and added, so that we # run the new version for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'): if old != new: removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] yield child.disownServiceParent() for n in added_names: child = new_by_name[n] # setup service's objectid class_name = '%s.%s' % (child.__class__.__module__, child.__class__.__name__) objectid = yield self.master.db.state.getObjectId( child.name, class_name) child.objectid = objectid yield defer.maybeDeferred(child.setServiceParent, self) # As the services that were just added got # reconfigServiceWithSibling called by # setServiceParent->startService, # we avoid calling it again by selecting # in reconfigurable_services, services # that were not added just now reconfigurable_services = [svc for svc in self if svc.name not in added_names] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError("%r: child %r should have a defined name attribute", self, svc) config_sibling = new_by_name.get(svc.name) yield svc.reconfigServiceWithSibling(config_sibling)
def test_removed(self): removed, added = util.diffSets(set([1, 2]), set([1])) self.assertEqual((removed, added), (set([2]), set([])))
def test_no_overlap(self): removed, added = util.diffSets(set([1, 2]), set([3, 4])) self.assertEqual((removed, added), (set([1, 2]), set([3, 4])))
def test_no_lists(self): removed, added = util.diffSets([1, 2], [2, 3]) self.assertEqual((removed, added), (set([1]), set([3])))
def test_empty(self): removed, added = util.diffSets(set([]), set([])) self.assertEqual((removed, added), (set([]), set([])))
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = {s.name: s for s in new_config_attr} elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError( f"config.{self.config_attr} should be a list or dictionary") new_set = set(new_by_name) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any children for which the old instance is not # able to do a reconfig with the new sibling # and add them to both removed and added, so that we # run the new version for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # check if we are able to reconfig service if not old.canReconfigWithSibling(new): removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg(f"adding {len(added_names)} new {self.config_attr}, " f"removing {len(removed_names)}") for n in removed_names: child = old_by_name[n] # disownServiceParent calls stopService after removing the relationship # as child might use self.master.data to stop itself, its better to stop it first # (this is related to the fact that self.master is found by recursively looking at # self.parent for a master) yield child.stopService() # it has already called, so do not call it again child.stopService = lambda: None yield child.disownServiceParent() for n in added_names: child = new_by_name[n] # setup service's objectid if hasattr(child, 'objectid'): class_name = f'{child.__class__.__module__}.{child.__class__.__name__}' objectid = yield self.master.db.state.getObjectId( child.name, class_name) child.objectid = objectid yield child.setServiceParent(self) # As the services that were just added got # reconfigServiceWithSibling called by # setServiceParent->startService, # we avoid calling it again by selecting # in reconfigurable_services, services # that were not added just now reconfigurable_services = [ svc for svc in self if svc.name not in added_names ] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError( f"{self}: child {svc} should have a defined name attribute" ) config_sibling = new_by_name.get(svc.name) try: yield svc.reconfigServiceWithSibling(config_sibling) except NotImplementedError: # legacy support. Its too painful to transition old code to new Service life cycle # so we implement switch of child when the service raises NotImplementedError # Note this means that self will stop, and sibling will take ownership # means that we have a small time where the service is unavailable. yield svc.disownServiceParent() config_sibling.objectid = svc.objectid yield config_sibling.setServiceParent(self) except Exception as e: # pragma: no cover log.err( e, f'Got exception while reconfiguring {self} child service {svc.name}:\n' 'current config dict:\n{svc.getConfigDict()}\n' 'new config dict:\n{config_sibling.getConfigDict()}') raise
def reconfigService(self, new_config): timer = metrics.Timer("SchedulerManager.reconfigService") timer.start() old_by_name = dict((sch.name, sch) for sch in self) old_set = set(old_by_name.iterkeys()) new_by_name = new_config.schedulers new_set = set(new_by_name.iterkeys()) removed_names, added_names = util.diffSets(old_set, new_set) # find any schedulers that don't know how to reconfig, and, if they # have changed, add them to both removed and added, so that we # run the new version. While we're at it, find any schedulers whose # fully qualified class name has changed, and consider those a removal # and re-add as well. for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigService'): if old != new: removed_names.add(n) added_names.add(n) # removals first for sch_name in removed_names: log.msg("removing scheduler '%s'" % (sch_name, )) sch = old_by_name[sch_name] yield defer.maybeDeferred(lambda: sch.disownServiceParent()) sch.master = None # .. then additions for sch_name in added_names: log.msg("adding scheduler '%s'" % (sch_name, )) sch = new_by_name[sch_name] # get the scheduler's objectid class_name = '%s.%s' % (sch.__class__.__module__, sch.__class__.__name__) objectid = yield self.master.db.state.getObjectId( sch.name, class_name) # set up the scheduler sch.objectid = objectid sch.master = self.master # *then* attacah and start it yield sch.setServiceParent(self) metrics.MetricCountEvent.log("num_schedulers", len(list(self)), absolute=True) # reconfig any newly-added schedulers, as well as existing yield config.ReconfigurableServiceMixin.reconfigService( self, new_config) timer.stop()
def reconfigService(self, new_config): timer = metrics.Timer("SchedulerManager.reconfigService") timer.start() old_by_name = dict((sch.name, sch) for sch in self) old_set = set(old_by_name.iterkeys()) new_by_name = new_config.schedulers new_set = set(new_by_name.iterkeys()) removed_names, added_names = util.diffSets(old_set, new_set) # find any schedulers that don't know how to reconfig, and, if they # have changed, add them to both removed and added, so that we # run the new version. While we're at it, find any schedulers whose # fully qualified class name has changed, and consider those a removal # and re-add as well. for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigService'): if old != new: removed_names.add(n) added_names.add(n) # removals first for sch_name in removed_names: log.msg("removing scheduler '%s'" % (sch_name,)) sch = old_by_name[sch_name] wfd = defer.waitForDeferred( defer.maybeDeferred(lambda : sch.disownServiceParent())) yield wfd wfd.getResult() sch.master = None # .. then additions # account for some renamed classes in buildbot - classes that have # changed their module import path, but should still access the same # state new_class_names = { # new : old 'buildbot.schedulers.dependent.Dependent' : 'buildbot.schedulers.basic.Dependent', 'buildbot.schedulers.basic.SingleBranchScheduler' : 'buildbot.schedulers.basic.Scheduler', } for sch_name in added_names: log.msg("adding scheduler '%s'" % (sch_name,)) sch = new_by_name[sch_name] # get the translated class name class_name = '%s.%s' % (sch.__class__.__module__, sch.__class__.__name__) class_name = new_class_names.get(class_name, class_name) # get the schedulerid wfd = defer.waitForDeferred( self.master.db.schedulers.getSchedulerId( sch.name, class_name)) yield wfd schedulerid = wfd.getResult() # set up the scheduler sch.schedulerid = schedulerid sch.master = self.master # *then* attacah and start it sch.setServiceParent(self) metrics.MetricCountEvent.log("num_schedulers", len(list(self)), absolute=True) # reconfig any newly-added schedulers, as well as existing wfd = defer.waitForDeferred( config.ReconfigurableServiceMixin.reconfigService(self, new_config)) yield wfd wfd.getResult() timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = dict([(s.name, s) for s in new_config_attr]) elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError( "config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove # While we're at it find any service that don't know how to reconfig, # and, if they have changed, add them to both removed and added, so that we # run the new version for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'): if old != new: removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] yield child.disownServiceParent() for n in added_names: child = new_by_name[n] # setup service's objectid class_name = '%s.%s' % (child.__class__.__module__, child.__class__.__name__) objectid = yield self.master.db.state.getObjectId( child.name, class_name) child.objectid = objectid yield defer.maybeDeferred(child.setServiceParent, self) # As the services that were just added got # reconfigServiceWithSibling called by # setServiceParent->startService, # we avoid calling it again by selecting # in reconfigurable_services, services # that were not added just now reconfigurable_services = [svc for svc in self if svc.name not in added_names] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError( "%r: child %r should have a defined name attribute", self, svc) config_sibling = new_by_name.get(svc.name) yield svc.reconfigServiceWithSibling(config_sibling)
def reconfigServiceWithBuildbotConfig(self, new_config): # arrange childs by name old_by_name = self.namedServices old_set = set(old_by_name) new_config_attr = getattr(new_config, self.config_attr) if isinstance(new_config_attr, list): new_by_name = {s.name: s for s in new_config_attr} elif isinstance(new_config_attr, dict): new_by_name = new_config_attr else: raise TypeError( "config.%s should be a list or dictionary" % (self.config_attr)) new_set = set(new_by_name) # calculate new childs, by name, and removed childs removed_names, added_names = util.diffSets(old_set, new_set) # find any childs for which the fully qualified class name has # changed, and treat those as an add and remove # While we're at it find any service that don't know how to reconfig, # and, if they have changed, add them to both removed and added, so that we # run the new version for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'): if old != new: removed_names.add(n) added_names.add(n) if removed_names or added_names: log.msg("adding %d new %s, removing %d" % (len(added_names), self.config_attr, len(removed_names))) for n in removed_names: child = old_by_name[n] # disownServiceParent calls stopService after removing the relationship # as child might use self.master.data to stop itself, its better to stop it first # (this is related to the fact that self.master is found by recursively looking at self.parent # for a master) yield child.stopService() # it has already called, so do not call it again child.stopService = lambda: None yield child.disownServiceParent() # HACK: we still keep a reference to the master for some cleanup tasks which are not waited by # to stopService (like the complex worker disconnection mechanism) # http://trac.buildbot.net/ticket/3583 child.parent = self.master for n in added_names: child = new_by_name[n] # setup service's objectid if hasattr(child, 'objectid'): class_name = '%s.%s' % (child.__class__.__module__, child.__class__.__name__) objectid = yield self.master.db.state.getObjectId( child.name, class_name) child.objectid = objectid yield defer.maybeDeferred(child.setServiceParent, self) # As the services that were just added got # reconfigServiceWithSibling called by # setServiceParent->startService, # we avoid calling it again by selecting # in reconfigurable_services, services # that were not added just now reconfigurable_services = [svc for svc in self if svc.name not in added_names] # sort by priority reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority) for svc in reconfigurable_services: if not svc.name: raise ValueError( "{}: child {} should have a defined name attribute".format(self, svc)) config_sibling = new_by_name.get(svc.name) try: yield svc.reconfigServiceWithSibling(config_sibling) except NotImplementedError: # legacy support. Its too painful to transition old code to new Service life cycle # so we implement switch of child when the service raises NotImplementedError # Note this means that self will stop, and sibling will take ownership # means that we have a small time where the service is unavailable. yield svc.disownServiceParent() config_sibling.objectid = svc.objectid yield config_sibling.setServiceParent(self)
def reconfigService(self, new_config): timer = metrics.Timer("SchedulerManager.reconfigService") timer.start() old_by_name = dict((sch.name, sch) for sch in self) old_set = set(old_by_name.iterkeys()) new_by_name = new_config.schedulers new_set = set(new_by_name.iterkeys()) removed_names, added_names = util.diffSets(old_set, new_set) # find any schedulers that don't know how to reconfig, and, if they # have changed, add them to both removed and added, so that we # run the new version. While we're at it, find any schedulers whose # fully qualified class name has changed, and consider those a removal # and re-add as well. for n in old_set & new_set: old = old_by_name[n] new = new_by_name[n] # detect changed class name if reflect.qual(old.__class__) != reflect.qual(new.__class__): removed_names.add(n) added_names.add(n) # compare using ComparableMixin if they don't support reconfig elif not hasattr(old, 'reconfigService'): if old != new: removed_names.add(n) added_names.add(n) # removals first for sch_name in removed_names: log.msg("removing scheduler '%s'" % (sch_name, )) sch = old_by_name[sch_name] wfd = defer.waitForDeferred( defer.maybeDeferred(lambda: sch.disownServiceParent())) yield wfd wfd.getResult() sch.master = None # .. then additions # account for some renamed classes in buildbot - classes that have # changed their module import path, but should still access the same # state new_class_names = { # new : old 'buildbot.schedulers.dependent.Dependent': 'buildbot.schedulers.basic.Dependent', 'buildbot.schedulers.basic.SingleBranchScheduler': 'buildbot.schedulers.basic.Scheduler', } for sch_name in added_names: log.msg("adding scheduler '%s'" % (sch_name, )) sch = new_by_name[sch_name] # get the translated class name class_name = '%s.%s' % (sch.__class__.__module__, sch.__class__.__name__) class_name = new_class_names.get(class_name, class_name) # get the schedulerid wfd = defer.waitForDeferred( self.master.db.schedulers.getSchedulerId(sch.name, class_name)) yield wfd schedulerid = wfd.getResult() # set up the scheduler sch.schedulerid = schedulerid sch.master = self.master # *then* attacah and start it sch.setServiceParent(self) metrics.MetricCountEvent.log("num_schedulers", len(list(self)), absolute=True) # reconfig any newly-added schedulers, as well as existing wfd = defer.waitForDeferred( config.ReconfigurableServiceMixin.reconfigService( self, new_config)) yield wfd wfd.getResult() timer.stop()