def reconfigServiceWithBuildbotConfig(self, new_config):
timer = metrics.Timer(
"ChangeManager.reconfigServiceWithBuildbotConfig")
timer.start()
removed, added = util.diffSets(set(self), new_config.change_sources)
if removed or added:
log.msg("adding %d new changesources, removing %d" %
(len(added), len(removed)))
for src in removed:
yield defer.maybeDeferred(src.disownServiceParent)
for src in added:
yield src.setServiceParent(self)
num_sources = len(list(self))
assert num_sources == len(new_config.change_sources)
metrics.MetricCountEvent.log("num_sources", num_sources, absolute=True)
# reconfig any newly-added change sources, as well as existing
yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
self, new_config)
timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config):
timer = metrics.Timer(
"ChangeManager.reconfigServiceWithBuildbotConfig")
timer.start()
removed, added = util.diffSets(
set(self),
new_config.change_sources)
if removed or added:
log.msg("adding %d new changesources, removing %d" %
(len(added), len(removed)))
for src in removed:
yield src.deactivate()
yield defer.maybeDeferred(
src.disownServiceParent)
for src in added:
yield src.setServiceParent(self)
num_sources = len(list(self))
assert num_sources == len(new_config.change_sources)
metrics.MetricCountEvent.log("num_sources", num_sources, absolute=True)
# reconfig any newly-added change sources, as well as existing
yield service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
new_config)
timer.stop()
def reconfigServiceSlaves(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceSlaves")
timer.start()
# arrange slaves by name
old_by_name = dict([ (s.slavename, s)
for s in list(self)
if interfaces.IBuildSlave.providedBy(s) ])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([ (s.slavename, s)
for s in new_config.slaves ])
new_set = set(new_by_name.iterkeys())
# calculate new slaves, by name, and removed slaves
removed_names, added_names = util.diffSets(old_set, new_set)
# find any slaves for which the fully qualified class name has
# changed, and treat those as an add and remove
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new slaves, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
slave = old_by_name[n]
del self.slaves[n]
slave.master = None
slave.botmaster = None
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda :
slave.disownServiceParent()))
yield wfd
wfd.getResult()
for n in added_names:
slave = new_by_name[n]
slave.setServiceParent(self)
slave.botmaster = self
slave.master = self.master
self.slaves[n] = slave
metrics.MetricCountEvent.log("num_slaves",
len(self.slaves), absolute=True)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([(b.name, b) for b in list(self)
if isinstance(b, Builder)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(bc.name, bc) for bc in new_config.builders])
new_set = set(new_by_name.iterkeys())
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
yield defer.maybeDeferred(
lambda: builder.disownServiceParent())
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
builder.setServiceParent(self)
self.builderNames = self.builders.keys()
metrics.MetricCountEvent.log("num_builders",
len(self.builders),
absolute=True)
# remove unclaimed builds if the builder has been removed from configuration
if len(self.master.config.projects) > 1:
queued_builds = yield self.master.db.buildrequests.getBuildRequestInQueue(
sorted=True)
# TODO: if we are running in multimaster mode with multiple instance of katana we need
# to check for the project key as well
removed_builders = [
b for b in queued_builds if b["buildername"] not in new_set
]
self.removeQueuedBuilds(removed_builders)
timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name.iterkeys())
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = dict([(s.name, s) for s in new_config_attr])
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError("config.%s should be a list or dictionary" %
(self.config_attr))
new_set = set(new_by_name.iterkeys())
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
yield child.disownServiceParent()
for n in added_names:
child = new_by_name[n]
yield child.setServiceParent(self)
# get a list of child services to reconfigure
reconfigurable_services = [svc for svc in self]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError(
"%r: child %r should have a defined name attribute", self,
svc)
config_sibling = new_by_name.get(svc.name)
yield svc.reconfigServiceWithSibling(config_sibling)
def reconfigServiceSlaves(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceSlaves")
timer.start()
# arrange slaves by name
old_by_name = dict([(s.slavename, s) for s in list(self)
if interfaces.IBuildSlave.providedBy(s)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(s.slavename, s) for s in new_config.slaves])
new_set = set(new_by_name.iterkeys())
# calculate new slaves, by name, and removed slaves
removed_names, added_names = util.diffSets(old_set, new_set)
# find any slaves for which the fully qualified class name has
# changed, and treat those as an add and remove
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new slaves, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
slave = old_by_name[n]
del self.slaves[n]
slave.master = None
slave.botmaster = None
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda: slave.disownServiceParent()))
yield wfd
wfd.getResult()
for n in added_names:
slave = new_by_name[n]
slave.setServiceParent(self)
slave.botmaster = self
slave.master = self.master
self.slaves[n] = slave
metrics.MetricCountEvent.log("num_slaves",
len(self.slaves),
absolute=True)
timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name.iterkeys())
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = dict([(s.name, s)
for s in new_config_attr])
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError("config.%s should be a list or dictionary" % (self.config_attr))
new_set = set(new_by_name.iterkeys())
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
yield child.disownServiceParent()
for n in added_names:
child = new_by_name[n]
yield child.setServiceParent(self)
# get a list of child services to reconfigure
reconfigurable_services = [svc for svc in self]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError("%r: child %r should have a defined name attribute", self, svc)
config_sibling = new_by_name.get(svc.name)
yield svc.reconfigServiceWithSibling(config_sibling)
def reconfigServiceSlaves(self, new_config):
timer = metrics.Timer("BuildSlaveManager.reconfigServiceSlaves")
timer.start()
# first we deconfigure everything to let the slaves register again
yield self.master.data.updates.deconfigureAllBuidslavesForMaster(
self.master.masterid)
# arrange slaves by name
old_by_name = dict([(s.slavename, s) for s in list(self)
if interfaces.IBuildSlave.providedBy(s)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(s.slavename, s) for s in new_config.slaves])
new_set = set(new_by_name.iterkeys())
# calculate new slaves, by name, and removed slaves
removed_names, added_names = util.diffSets(old_set, new_set)
# find any slaves for which the fully qualified class name has
# changed, and treat those as an add and remove
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new slaves, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
slave = old_by_name[n]
del self.slaves[n]
slave.master = None
yield slave.disownServiceParent()
for n in added_names:
slave = new_by_name[n]
yield slave.setServiceParent(self)
self.slaves[n] = slave
metrics.MetricCountEvent.log("num_slaves",
len(self.slaves),
absolute=True)
timer.stop()
def reconfigServiceSlaves(self, new_config):
timer = metrics.Timer("BuildSlaveManager.reconfigServiceSlaves")
timer.start()
# first we deconfigure everything to let the slaves register again
yield self.master.data.updates.deconfigureAllBuidslavesForMaster(self.master.masterid)
# arrange slaves by name
old_by_name = dict([(s.slavename, s)
for s in list(self)
if IBuildSlave.providedBy(s)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(s.slavename, s)
for s in new_config.slaves])
new_set = set(new_by_name.iterkeys())
# calculate new slaves, by name, and removed slaves
removed_names, added_names = util.diffSets(old_set, new_set)
# find any slaves for which the fully qualified class name has
# changed, and treat those as an add and remove
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new slaves, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
slave = old_by_name[n]
del self.slaves[n]
slave.master = None
yield slave.disownServiceParent()
for n in added_names:
slave = new_by_name[n]
yield slave.setServiceParent(self)
self.slaves[n] = slave
metrics.MetricCountEvent.log("num_slaves",
len(self.slaves), absolute=True)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([(b.name, b)
for b in list(self)
if isinstance(b, Builder)])
old_set = set(old_by_name)
new_by_name = dict([(bc.name, bc)
for bc in new_config.builders])
new_set = set(new_by_name)
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
# pylint: disable=cell-var-from-loop
yield defer.maybeDeferred(lambda:
builder.disownServiceParent())
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
yield builder.setServiceParent(self)
self.builderNames = list(self.builders)
yield self.master.data.updates.updateBuilderList(
self.master.masterid,
[util.ascii2unicode(n) for n in self.builderNames])
metrics.MetricCountEvent.log("num_builders",
len(self.builders), absolute=True)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = {b.name: b
for b in list(self)
if isinstance(b, Builder)}
old_set = set(old_by_name)
new_by_name = {bc.name: bc
for bc in new_config.builders}
new_set = set(new_by_name)
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
# pylint: disable=cell-var-from-loop
yield defer.maybeDeferred(lambda:
builder.disownServiceParent())
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
yield builder.setServiceParent(self)
self.builderNames = list(self.builders)
yield self.master.data.updates.updateBuilderList(
self.master.masterid,
[util.bytes2unicode(n) for n in self.builderNames])
metrics.MetricCountEvent.log("num_builders",
len(self.builders), absolute=True)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([ (b.name, b)
for b in list(self)
if isinstance(b, Builder) ])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([ (bc.name, bc)
for bc in new_config.builders ])
new_set = set(new_by_name.iterkeys())
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda :
builder.disownServiceParent()))
yield wfd
wfd.getResult()
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
builder.setServiceParent(self)
self.builderNames = self.builders.keys()
metrics.MetricCountEvent.log("num_builders",
len(self.builders), absolute=True)
timer.stop()
def reconfigServiceBuilders(self, new_config):
timer = metrics.Timer("BotMaster.reconfigServiceBuilders")
timer.start()
# arrange builders by name
old_by_name = dict([(b.name, b) for b in list(self)
if isinstance(b, Builder)])
old_set = set(old_by_name.iterkeys())
new_by_name = dict([(bc.name, bc) for bc in new_config.builders])
new_set = set(new_by_name.iterkeys())
# calculate new builders, by name, and removed builders
removed_names, added_names = util.diffSets(old_set, new_set)
if removed_names or added_names:
log.msg("adding %d new builders, removing %d" %
(len(added_names), len(removed_names)))
for n in removed_names:
builder = old_by_name[n]
del self.builders[n]
builder.master = None
builder.botmaster = None
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda: builder.disownServiceParent()))
yield wfd
wfd.getResult()
for n in added_names:
builder = Builder(n)
self.builders[n] = builder
builder.botmaster = self
builder.master = self.master
builder.setServiceParent(self)
self.builderNames = self.builders.keys()
metrics.MetricCountEvent.log("num_builders",
len(self.builders),
absolute=True)
timer.stop()
def reconfigService(self, new_config):
timer = metrics.Timer("ChangeManager.reconfigService")
timer.start()
removed, added = util.diffSets(
set(self),
new_config.change_sources)
if removed or added:
log.msg("adding %d new changesources, removing %d" %
(len(added), len(removed)))
for src in removed:
wfd = defer.waitForDeferred(
defer.maybeDeferred(
src.disownServiceParent))
yield wfd
wfd.getResult()
src.master = None
for src in added:
src.master = self.master
src.setServiceParent(self)
num_sources = len(list(self))
assert num_sources == len(new_config.change_sources)
metrics.MetricCountEvent.log("num_sources", num_sources, absolute=True)
# reconfig any newly-added change sources, as well as existing
wfd = defer.waitForDeferred(
config.ReconfigurableServiceMixin.reconfigService(self,
new_config))
yield wfd
wfd.getResult()
timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config):
timer = metrics.Timer("SchedulerManager.reconfigServiceWithBuildbotConfig")
timer.start()
old_by_name = dict((sch.name, sch) for sch in self)
old_set = set(old_by_name.iterkeys())
new_by_name = new_config.schedulers
new_set = set(new_by_name.iterkeys())
removed_names, added_names = util.diffSets(old_set, new_set)
# find any schedulers that don't know how to reconfig, and, if they
# have changed, add them to both removed and added, so that we
# run the new version. While we're at it, find any schedulers whose
# fully qualified class name has changed, and consider those a removal
# and re-add as well.
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
if old != new:
removed_names.add(n)
added_names.add(n)
# removals first
for sch_name in removed_names:
log.msg("removing scheduler '%s'" % (sch_name,))
sch = old_by_name[sch_name]
yield defer.maybeDeferred(lambda:
sch.disownServiceParent())
sch.master = None
# .. then additions
for sch_name in added_names:
log.msg("adding scheduler '%s'" % (sch_name,))
sch = new_by_name[sch_name]
# get the scheduler's objectid
class_name = '%s.%s' % (sch.__class__.__module__,
sch.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
sch.name, class_name)
# set up the scheduler
sch.objectid = objectid
sch.master = self.master
# *then* attacah and start it
yield sch.setServiceParent(self)
metrics.MetricCountEvent.log("num_schedulers", len(list(self)),
absolute=True)
# reconfig any newly-added schedulers, as well as existing
yield util_service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
new_config)
timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name)
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = {s.name: s for s in new_config_attr}
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError("config.%s should be a list or dictionary" %
(self.config_attr))
new_set = set(new_by_name)
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
# While we're at it find any service that don't know how to reconfig,
# and, if they have changed, add them to both removed and added, so that we
# run the new version
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
if old != new:
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
# disownServiceParent calls stopService after removing the relationship
# as child might use self.master.data to stop itself, its better to stop it first
# (this is related to the fact that self.master is found by recursively looking at self.parent
# for a master)
yield child.stopService()
# it has already called, so do not call it again
child.stopService = lambda: None
yield child.disownServiceParent()
# HACK: we still keep a reference to the master for some cleanup tasks which are not waited by
# to stopService (like the complex worker disconnection mechanism)
# http://trac.buildbot.net/ticket/3583
child.parent = self.master
for n in added_names:
child = new_by_name[n]
# setup service's objectid
if hasattr(child, 'objectid'):
class_name = '%s.%s' % (child.__class__.__module__,
child.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
child.name, class_name)
child.objectid = objectid
yield defer.maybeDeferred(child.setServiceParent, self)
# As the services that were just added got
# reconfigServiceWithSibling called by
# setServiceParent->startService,
# we avoid calling it again by selecting
# in reconfigurable_services, services
# that were not added just now
reconfigurable_services = [
svc for svc in self if svc.name not in added_names
]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError(
"{}: child {} should have a defined name attribute".format(
self, svc))
config_sibling = new_by_name.get(svc.name)
try:
yield svc.reconfigServiceWithSibling(config_sibling)
except NotImplementedError:
# legacy support. Its too painful to transition old code to new Service life cycle
# so we implement switch of child when the service raises NotImplementedError
# Note this means that self will stop, and sibling will take ownership
# means that we have a small time where the service is unavailable.
yield svc.disownServiceParent()
config_sibling.objectid = svc.objectid
yield config_sibling.setServiceParent(self)
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name.iterkeys())
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = dict([(s.name, s)
for s in new_config_attr])
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError("config.%s should be a list or dictionary" % (self.config_attr))
new_set = set(new_by_name.iterkeys())
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
# While we're at it find any service that don't know how to reconfig,
# and, if they have changed, add them to both removed and added, so that we
# run the new version
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
if old != new:
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
yield child.disownServiceParent()
for n in added_names:
child = new_by_name[n]
# setup service's objectid
class_name = '%s.%s' % (child.__class__.__module__,
child.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
child.name, class_name)
child.objectid = objectid
yield defer.maybeDeferred(child.setServiceParent, self)
# As the services that were just added got
# reconfigServiceWithSibling called by
# setServiceParent->startService,
# we avoid calling it again by selecting
# in reconfigurable_services, services
# that were not added just now
reconfigurable_services = [svc for svc in self
if svc.name not in added_names]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError("%r: child %r should have a defined name attribute", self, svc)
config_sibling = new_by_name.get(svc.name)
yield svc.reconfigServiceWithSibling(config_sibling)
def test_removed(self):
removed, added = util.diffSets(set([1, 2]), set([1]))
self.assertEqual((removed, added), (set([2]), set([])))
def test_no_overlap(self):
removed, added = util.diffSets(set([1, 2]), set([3, 4]))
self.assertEqual((removed, added), (set([1, 2]), set([3, 4])))
def test_no_lists(self):
removed, added = util.diffSets([1, 2], [2, 3])
self.assertEqual((removed, added), (set([1]), set([3])))
def test_empty(self):
removed, added = util.diffSets(set([]), set([]))
self.assertEqual((removed, added), (set([]), set([])))
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name)
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = {s.name: s for s in new_config_attr}
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError(
f"config.{self.config_attr} should be a list or dictionary")
new_set = set(new_by_name)
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any children for which the old instance is not
# able to do a reconfig with the new sibling
# and add them to both removed and added, so that we
# run the new version
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# check if we are able to reconfig service
if not old.canReconfigWithSibling(new):
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg(f"adding {len(added_names)} new {self.config_attr}, "
f"removing {len(removed_names)}")
for n in removed_names:
child = old_by_name[n]
# disownServiceParent calls stopService after removing the relationship
# as child might use self.master.data to stop itself, its better to stop it first
# (this is related to the fact that self.master is found by recursively looking at
# self.parent for a master)
yield child.stopService()
# it has already called, so do not call it again
child.stopService = lambda: None
yield child.disownServiceParent()
for n in added_names:
child = new_by_name[n]
# setup service's objectid
if hasattr(child, 'objectid'):
class_name = f'{child.__class__.__module__}.{child.__class__.__name__}'
objectid = yield self.master.db.state.getObjectId(
child.name, class_name)
child.objectid = objectid
yield child.setServiceParent(self)
# As the services that were just added got
# reconfigServiceWithSibling called by
# setServiceParent->startService,
# we avoid calling it again by selecting
# in reconfigurable_services, services
# that were not added just now
reconfigurable_services = [
svc for svc in self if svc.name not in added_names
]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError(
f"{self}: child {svc} should have a defined name attribute"
)
config_sibling = new_by_name.get(svc.name)
try:
yield svc.reconfigServiceWithSibling(config_sibling)
except NotImplementedError:
# legacy support. Its too painful to transition old code to new Service life cycle
# so we implement switch of child when the service raises NotImplementedError
# Note this means that self will stop, and sibling will take ownership
# means that we have a small time where the service is unavailable.
yield svc.disownServiceParent()
config_sibling.objectid = svc.objectid
yield config_sibling.setServiceParent(self)
except Exception as e: # pragma: no cover
log.err(
e,
f'Got exception while reconfiguring {self} child service {svc.name}:\n'
'current config dict:\n{svc.getConfigDict()}\n'
'new config dict:\n{config_sibling.getConfigDict()}')
raise
def reconfigService(self, new_config):
timer = metrics.Timer("SchedulerManager.reconfigService")
timer.start()
old_by_name = dict((sch.name, sch) for sch in self)
old_set = set(old_by_name.iterkeys())
new_by_name = new_config.schedulers
new_set = set(new_by_name.iterkeys())
removed_names, added_names = util.diffSets(old_set, new_set)
# find any schedulers that don't know how to reconfig, and, if they
# have changed, add them to both removed and added, so that we
# run the new version. While we're at it, find any schedulers whose
# fully qualified class name has changed, and consider those a removal
# and re-add as well.
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigService'):
if old != new:
removed_names.add(n)
added_names.add(n)
# removals first
for sch_name in removed_names:
log.msg("removing scheduler '%s'" % (sch_name, ))
sch = old_by_name[sch_name]
yield defer.maybeDeferred(lambda: sch.disownServiceParent())
sch.master = None
# .. then additions
for sch_name in added_names:
log.msg("adding scheduler '%s'" % (sch_name, ))
sch = new_by_name[sch_name]
# get the scheduler's objectid
class_name = '%s.%s' % (sch.__class__.__module__,
sch.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
sch.name, class_name)
# set up the scheduler
sch.objectid = objectid
sch.master = self.master
# *then* attacah and start it
yield sch.setServiceParent(self)
metrics.MetricCountEvent.log("num_schedulers",
len(list(self)),
absolute=True)
# reconfig any newly-added schedulers, as well as existing
yield config.ReconfigurableServiceMixin.reconfigService(
self, new_config)
timer.stop()
def reconfigService(self, new_config):
timer = metrics.Timer("SchedulerManager.reconfigService")
timer.start()
old_by_name = dict((sch.name, sch) for sch in self)
old_set = set(old_by_name.iterkeys())
new_by_name = new_config.schedulers
new_set = set(new_by_name.iterkeys())
removed_names, added_names = util.diffSets(old_set, new_set)
# find any schedulers that don't know how to reconfig, and, if they
# have changed, add them to both removed and added, so that we
# run the new version. While we're at it, find any schedulers whose
# fully qualified class name has changed, and consider those a removal
# and re-add as well.
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigService'):
if old != new:
removed_names.add(n)
added_names.add(n)
# removals first
for sch_name in removed_names:
log.msg("removing scheduler '%s'" % (sch_name,))
sch = old_by_name[sch_name]
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda :
sch.disownServiceParent()))
yield wfd
wfd.getResult()
sch.master = None
# .. then additions
# account for some renamed classes in buildbot - classes that have
# changed their module import path, but should still access the same
# state
new_class_names = {
# new : old
'buildbot.schedulers.dependent.Dependent' :
'buildbot.schedulers.basic.Dependent',
'buildbot.schedulers.basic.SingleBranchScheduler' :
'buildbot.schedulers.basic.Scheduler',
}
for sch_name in added_names:
log.msg("adding scheduler '%s'" % (sch_name,))
sch = new_by_name[sch_name]
# get the translated class name
class_name = '%s.%s' % (sch.__class__.__module__,
sch.__class__.__name__)
class_name = new_class_names.get(class_name, class_name)
# get the schedulerid
wfd = defer.waitForDeferred(
self.master.db.schedulers.getSchedulerId(
sch.name, class_name))
yield wfd
schedulerid = wfd.getResult()
# set up the scheduler
sch.schedulerid = schedulerid
sch.master = self.master
# *then* attacah and start it
sch.setServiceParent(self)
metrics.MetricCountEvent.log("num_schedulers", len(list(self)),
absolute=True)
# reconfig any newly-added schedulers, as well as existing
wfd = defer.waitForDeferred(
config.ReconfigurableServiceMixin.reconfigService(self,
new_config))
yield wfd
wfd.getResult()
timer.stop()
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name)
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = dict([(s.name, s)
for s in new_config_attr])
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError(
"config.%s should be a list or dictionary" % (self.config_attr))
new_set = set(new_by_name)
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
# While we're at it find any service that don't know how to reconfig,
# and, if they have changed, add them to both removed and added, so that we
# run the new version
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
if old != new:
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
yield child.disownServiceParent()
for n in added_names:
child = new_by_name[n]
# setup service's objectid
class_name = '%s.%s' % (child.__class__.__module__,
child.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
child.name, class_name)
child.objectid = objectid
yield defer.maybeDeferred(child.setServiceParent, self)
# As the services that were just added got
# reconfigServiceWithSibling called by
# setServiceParent->startService,
# we avoid calling it again by selecting
# in reconfigurable_services, services
# that were not added just now
reconfigurable_services = [svc for svc in self
if svc.name not in added_names]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError(
"%r: child %r should have a defined name attribute", self, svc)
config_sibling = new_by_name.get(svc.name)
yield svc.reconfigServiceWithSibling(config_sibling)
def test_no_overlap(self):
removed, added = util.diffSets(set([1, 2]), set([3, 4]))
self.assertEqual((removed, added), (set([1, 2]), set([3, 4])))
def test_empty(self):
removed, added = util.diffSets(set([]), set([]))
self.assertEqual((removed, added), (set([]), set([])))
def test_no_lists(self):
removed, added = util.diffSets([1, 2], [2, 3])
self.assertEqual((removed, added), (set([1]), set([3])))
def reconfigServiceWithBuildbotConfig(self, new_config):
# arrange childs by name
old_by_name = self.namedServices
old_set = set(old_by_name)
new_config_attr = getattr(new_config, self.config_attr)
if isinstance(new_config_attr, list):
new_by_name = {s.name: s
for s in new_config_attr}
elif isinstance(new_config_attr, dict):
new_by_name = new_config_attr
else:
raise TypeError(
"config.%s should be a list or dictionary" % (self.config_attr))
new_set = set(new_by_name)
# calculate new childs, by name, and removed childs
removed_names, added_names = util.diffSets(old_set, new_set)
# find any childs for which the fully qualified class name has
# changed, and treat those as an add and remove
# While we're at it find any service that don't know how to reconfig,
# and, if they have changed, add them to both removed and added, so that we
# run the new version
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigServiceWithBuildbotConfig'):
if old != new:
removed_names.add(n)
added_names.add(n)
if removed_names or added_names:
log.msg("adding %d new %s, removing %d" %
(len(added_names), self.config_attr, len(removed_names)))
for n in removed_names:
child = old_by_name[n]
# disownServiceParent calls stopService after removing the relationship
# as child might use self.master.data to stop itself, its better to stop it first
# (this is related to the fact that self.master is found by recursively looking at self.parent
# for a master)
yield child.stopService()
# it has already called, so do not call it again
child.stopService = lambda: None
yield child.disownServiceParent()
# HACK: we still keep a reference to the master for some cleanup tasks which are not waited by
# to stopService (like the complex worker disconnection mechanism)
# http://trac.buildbot.net/ticket/3583
child.parent = self.master
for n in added_names:
child = new_by_name[n]
# setup service's objectid
if hasattr(child, 'objectid'):
class_name = '%s.%s' % (child.__class__.__module__,
child.__class__.__name__)
objectid = yield self.master.db.state.getObjectId(
child.name, class_name)
child.objectid = objectid
yield defer.maybeDeferred(child.setServiceParent, self)
# As the services that were just added got
# reconfigServiceWithSibling called by
# setServiceParent->startService,
# we avoid calling it again by selecting
# in reconfigurable_services, services
# that were not added just now
reconfigurable_services = [svc for svc in self
if svc.name not in added_names]
# sort by priority
reconfigurable_services.sort(key=lambda svc: -svc.reconfig_priority)
for svc in reconfigurable_services:
if not svc.name:
raise ValueError(
"{}: child {} should have a defined name attribute".format(self, svc))
config_sibling = new_by_name.get(svc.name)
try:
yield svc.reconfigServiceWithSibling(config_sibling)
except NotImplementedError:
# legacy support. Its too painful to transition old code to new Service life cycle
# so we implement switch of child when the service raises NotImplementedError
# Note this means that self will stop, and sibling will take ownership
# means that we have a small time where the service is unavailable.
yield svc.disownServiceParent()
config_sibling.objectid = svc.objectid
yield config_sibling.setServiceParent(self)
def test_removed(self):
removed, added = util.diffSets(set([1, 2]), set([1]))
self.assertEqual((removed, added), (set([2]), set([])))
def reconfigService(self, new_config):
timer = metrics.Timer("SchedulerManager.reconfigService")
timer.start()
old_by_name = dict((sch.name, sch) for sch in self)
old_set = set(old_by_name.iterkeys())
new_by_name = new_config.schedulers
new_set = set(new_by_name.iterkeys())
removed_names, added_names = util.diffSets(old_set, new_set)
# find any schedulers that don't know how to reconfig, and, if they
# have changed, add them to both removed and added, so that we
# run the new version. While we're at it, find any schedulers whose
# fully qualified class name has changed, and consider those a removal
# and re-add as well.
for n in old_set & new_set:
old = old_by_name[n]
new = new_by_name[n]
# detect changed class name
if reflect.qual(old.__class__) != reflect.qual(new.__class__):
removed_names.add(n)
added_names.add(n)
# compare using ComparableMixin if they don't support reconfig
elif not hasattr(old, 'reconfigService'):
if old != new:
removed_names.add(n)
added_names.add(n)
# removals first
for sch_name in removed_names:
log.msg("removing scheduler '%s'" % (sch_name, ))
sch = old_by_name[sch_name]
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda: sch.disownServiceParent()))
yield wfd
wfd.getResult()
sch.master = None
# .. then additions
# account for some renamed classes in buildbot - classes that have
# changed their module import path, but should still access the same
# state
new_class_names = {
# new : old
'buildbot.schedulers.dependent.Dependent':
'buildbot.schedulers.basic.Dependent',
'buildbot.schedulers.basic.SingleBranchScheduler':
'buildbot.schedulers.basic.Scheduler',
}
for sch_name in added_names:
log.msg("adding scheduler '%s'" % (sch_name, ))
sch = new_by_name[sch_name]
# get the translated class name
class_name = '%s.%s' % (sch.__class__.__module__,
sch.__class__.__name__)
class_name = new_class_names.get(class_name, class_name)
# get the schedulerid
wfd = defer.waitForDeferred(
self.master.db.schedulers.getSchedulerId(sch.name, class_name))
yield wfd
schedulerid = wfd.getResult()
# set up the scheduler
sch.schedulerid = schedulerid
sch.master = self.master
# *then* attacah and start it
sch.setServiceParent(self)
metrics.MetricCountEvent.log("num_schedulers",
len(list(self)),
absolute=True)
# reconfig any newly-added schedulers, as well as existing
wfd = defer.waitForDeferred(
config.ReconfigurableServiceMixin.reconfigService(
self, new_config))
yield wfd
wfd.getResult()
timer.stop()