def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.change_svc = ChangeManager(self)
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster(self)
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager(self)
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = connector.DBConnector(self, self.basedir)
self.db.setServiceParent(self)
self.debug = debug.DebugServices(self)
self.debug.setServiceParent(self)
self.status = Status(self)
self.status.setServiceParent(self)
def setUp(self):
self.master = master = fakemaster.make_master(testcase=self,
wantData=True)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(master)
self.reactor = mock.Mock()
self.botmaster.startService()
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.buildslaves = bslavemanager.BuildslaveManager(self)
self.buildslaves.setServiceParent(self)
self.change_svc = ChangeManager(self)
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster(self)
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager(self)
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self, self.basedir)
self.db.setServiceParent(self)
self.mq = mqconnector.MQConnector(self)
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector(self)
self.data.setServiceParent(self)
self.www = wwwservice.WWWService(self)
self.www.setServiceParent(self)
self.debug = debug.DebugServices(self)
self.debug.setServiceParent(self)
self.status = Status(self)
self.status.setServiceParent(self)
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# force housekeeping once a day
yield self.data.updates.expireMasters(
(self.masterHouskeepingTimer % (24 * 60)) == 0)
self.masterHouskeepingTimer += 1
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
self.masterHeartbeatService.setServiceParent(self)
def setUp(self):
self.master = fakemaster.make_master(testcase=self,
wantMq=True,
wantData=True)
self.master.mq = self.master.mq
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantMq=True, wantData=True)
self.master.mq = self.master.mq
self.master.botmaster.disownServiceParent()
self.botmaster = BotMaster()
yield self.botmaster.setServiceParent(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def setUp(self):
self.master = master = fakemaster.make_master(
testcase=self, wantData=True)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(master)
self.reactor = mock.Mock()
self.botmaster.startService()
def setUp(self):
self.master = fakemaster.make_master(testcase=self, wantMq=True,
wantData=True)
self.master.mq = self.master.mq
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.buildslaves = bslavemanager.BuildslaveManager(self)
self.buildslaves.setServiceParent(self)
self.change_svc = ChangeManager(self)
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster(self)
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager(self)
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self, self.basedir)
self.db.setServiceParent(self)
self.mq = mqconnector.MQConnector(self)
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector(self)
self.data.setServiceParent(self)
self.www = wwwservice.WWWService(self)
self.www.setServiceParent(self)
self.debug = debug.DebugServices(self)
self.debug.setServiceParent(self)
self.status = Status(self)
self.status.setServiceParent(self)
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# force housekeeping once a day
yield self.data.updates.expireMasters((self.masterHouskeepingTimer % (24 * 60)) == 0)
self.masterHouskeepingTimer += 1
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
self.masterHeartbeatService.setServiceParent(self)
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantMq=True, wantData=True)
self.master.mq = self.master.mq
self.master.botmaster.disownServiceParent()
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder('frank')
brd.maybeStartBuildsOn.assert_called_once_with(['frank'])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name='frank')
b1.name = 'frank'
b2 = mock.Mock(name='larry')
b2.name = 'larry'
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave('centos')
self.botmaster.getBuildersForSlave.assert_called_once_with('centos')
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ['frank', 'larry']
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder('frank')
brd.maybeStartBuildsOn.assert_called_once_with(['frank'])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name='frank')
b1.name = 'frank'
b2 = mock.Mock(name='larry')
b2.name = 'larry'
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave('centos')
self.botmaster.getBuildersForSlave.assert_called_once_with('centos')
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ['frank', 'larry']
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.workers = workermanager.WorkerManager(self)
self.workers.setServiceParent(self)
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager()
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self.basedir)
self.db.setServiceParent(self)
self.wamp = wampconnector.WampConnector()
self.wamp.setServiceParent(self)
self.mq = mqconnector.MQConnector()
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector()
self.data.setServiceParent(self)
self.www = wwwservice.WWWService()
self.www.setServiceParent(self)
self.debug = debug.DebugServices()
self.debug.setServiceParent(self)
self.status = Status()
self.status.setServiceParent(self)
self.secrets_manager = SecretManager()
self.secrets_manager.setServiceParent(self)
self.secrets_manager.reconfig_priority = 2000
self.service_manager = service.BuildbotServiceManager()
self.service_manager.setServiceParent(self)
self.service_manager.reconfig_priority = 1000
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
yield self.data.updates.expireMasters()
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
def setUp(self):
self.master = fakemaster.make_master(True, "TestBotMaster")
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
self.reactor = mock.Mock()
self.botmaster.startService()
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
WorkerAPICompatMixin):
# frequency with which to reclaim running builds; this should be set to
# something fairly long, to avoid undue database load
RECLAIM_BUILD_INTERVAL = 10 * 60
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
def __init__(self,
basedir,
configFileName=None,
umask=None,
reactor=None,
config_loader=None):
service.AsyncMultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
if config_loader is not None and configFileName is not None:
raise config.ConfigErrors([
"Can't specify both `config_loader` and `configFilename`.",
])
elif config_loader is None:
if configFileName is None:
configFileName = 'master.cfg'
config_loader = config.FileLoader(self.basedir, configFileName)
self.config_loader = config_loader
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" %
(self.hostname, os.path.abspath(self.basedir or '.')))
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.workers = workermanager.WorkerManager(self)
self.workers.setServiceParent(self)
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager()
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self.basedir)
self.db.setServiceParent(self)
self.wamp = wampconnector.WampConnector()
self.wamp.setServiceParent(self)
self.mq = mqconnector.MQConnector()
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector()
self.data.setServiceParent(self)
self.www = wwwservice.WWWService()
self.www.setServiceParent(self)
self.debug = debug.DebugServices()
self.debug.setServiceParent(self)
self.status = Status()
self.status.setServiceParent(self)
self.service_manager = service.BuildbotServiceManager()
self.service_manager.setServiceParent(self)
self.service_manager.reconfig_priority = 1000
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# force housekeeping once a day
yield self.data.updates.expireMasters(
(self.masterHouskeepingTimer % (24 * 60)) == 0)
self.masterHouskeepingTimer += 1
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
# we do setServiceParent only when the master is configured
# master should advertise itself only at that time
# setup and reconfig handling
_already_started = False
@defer.inlineCallbacks
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
self.reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
self._master_initialized = True
@defer.inlineCallbacks
def stopService(self):
if self.masterid is not None:
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
if self.running:
yield service.AsyncMultiService.stopService(self)
log.msg("BuildMaster is stopped")
self._master_initialized = False
def reconfig(self):
# this method wraps doConfig, ensuring it is only ever called once at
# a time, and alerting the user if the reconfig takes too long
if self.reconfig_active:
log.msg("reconfig already active; will reconfig again after")
self.reconfig_requested = True
return
self.reconfig_active = self.reactor.seconds()
metrics.MetricCountEvent.log("loaded_config", 1)
# notify every 10 seconds that the reconfig is still going on, although
# reconfigs should not take that long!
self.reconfig_notifier = task.LoopingCall(
lambda: log.msg("reconfig is ongoing for %d s" %
(self.reactor.seconds() - self.reconfig_active)))
self.reconfig_notifier.start(10, now=False)
timer = metrics.Timer("BuildMaster.reconfig")
timer.start()
d = self.doReconfig()
@d.addBoth
def cleanup(res):
timer.stop()
self.reconfig_notifier.stop()
self.reconfig_notifier = None
self.reconfig_active = False
if self.reconfig_requested:
self.reconfig_requested = False
self.reconfig()
return res
@d.addCallback
def advertiseItself(res):
if self.masterHeartbeatService.parent is None:
self.masterHeartbeatService.setServiceParent(self)
d.addErrback(log.err, 'while reconfiguring')
return d # for tests
@defer.inlineCallbacks
def doReconfig(self):
log.msg("beginning configuration update")
changes_made = False
failed = False
try:
# Run the master.cfg in thread, so that it cas use blocking code
new_config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
changes_made = True
self.config = new_config
yield self.reconfigServiceWithBuildbotConfig(new_config)
except config.ConfigErrors as e:
for msg in e.errors:
log.msg(msg)
failed = True
except Exception:
log.err(failure.Failure(), 'during reconfig:')
failed = True
if failed:
if changes_made:
log.msg("WARNING: reconfig partially applied; master "
"may malfunction")
else:
log.msg("reconfig aborted without making any changes")
else:
log.msg("configuration update complete")
def reconfigServiceWithBuildbotConfig(self, new_config):
if self.configured_db_url is None:
self.configured_db_url = new_config.db['db_url']
elif (self.configured_db_url != new_config.db['db_url']):
config.error(
"Cannot change c['db']['db_url'] after the master has started",
)
if self.config.mq['type'] != new_config.mq['type']:
raise config.ConfigErrors([
"Cannot change c['mq']['type'] after the master has started",
])
return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
self, new_config)
# informational methods
def allSchedulers(self):
return list(self.scheduler_manager)
def getStatus(self):
"""
@rtype: L{buildbot.status.builder.Status}
"""
return self.status
# triggering methods
@defer.inlineCallbacks
def addChange(self, who=None, files=None, comments=None, **kwargs):
# deprecated in 0.9.0; will be removed in 1.0.0
log.msg("WARNING: change source is using deprecated "
"self.master.addChange method; this method will disappear in "
"Buildbot-1.0.0")
# handle positional arguments
kwargs['who'] = who
kwargs['files'] = files
kwargs['comments'] = comments
def handle_deprec(oldname, newname):
if oldname not in kwargs:
return
old = kwargs.pop(oldname)
if old is not None:
if kwargs.get(newname) is None:
log.msg("WARNING: change source is using deprecated "
"addChange parameter '%s'" % oldname)
return old
raise TypeError("Cannot provide '%s' and '%s' to addChange" %
(oldname, newname))
return kwargs.get(newname)
kwargs['author'] = handle_deprec("who", "author")
kwargs['when_timestamp'] = handle_deprec("when", "when_timestamp")
# timestamp must be an epoch timestamp now
if isinstance(kwargs.get('when_timestamp'), datetime.datetime):
kwargs['when_timestamp'] = datetime2epoch(kwargs['when_timestamp'])
# unicodify stuff
for k in ('comments', 'author', 'revision', 'branch', 'category',
'revlink', 'repository', 'codebase', 'project'):
if k in kwargs:
kwargs[k] = ascii2unicode(kwargs[k])
if kwargs.get('files'):
kwargs['files'] = [ascii2unicode(f) for f in kwargs['files']]
if kwargs.get('properties'):
kwargs['properties'] = dict(
(ascii2unicode(k), v)
for k, v in iteritems(kwargs['properties']))
# pass the converted call on to the data API
changeid = yield self.data.updates.addChange(**kwargs)
# and turn that changeid into a change object, since that's what
# callers expected (and why this method was deprecated)
chdict = yield self.db.changes.getChange(changeid)
change = yield changes.Change.fromChdict(self, chdict)
defer.returnValue(change)
@defer.inlineCallbacks
def addBuildset(self, scheduler, **kwargs):
log.msg("WARNING: master.addBuildset is deprecated; "
"use the data API update method")
bsid, brids = yield self.data.updates.addBuildset(scheduler=scheduler,
**kwargs)
defer.returnValue((bsid, brids))
# state maintenance (private)
def getObjectId(self):
"""
Return the object id for this master, for associating state with the
master.
@returns: ID, via Deferred
"""
# try to get the cached value
if self._object_id is not None:
return defer.succeed(self._object_id)
# failing that, get it from the DB; multiple calls to this function
# at the same time will not hurt
d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster")
@d.addCallback
def keep(id):
self._object_id = id
return id
return d
def _getState(self, name, default=None):
"private wrapper around C{self.db.state.getState}"
d = self.getObjectId()
@d.addCallback
def get(objectid):
return self.db.state.getState(objectid, name, default)
return d
def _setState(self, name, value):
"private wrapper around C{self.db.state.setState}"
d = self.getObjectId()
@d.addCallback
def set(objectid):
return self.db.state.setState(objectid, name, value)
return d
class BuildMaster(config.ReconfigurableServiceMixin, service.MultiService):
# frequency with which to reclaim running builds; this should be set to
# something fairly long, to avoid undue database load
RECLAIM_BUILD_INTERVAL = 10*60
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
# if this quantity of unclaimed build requests are present in the table,
# then something is probably wrong! The master will log a WARNING on every
# database poll operation.
WARNING_UNCLAIMED_COUNT = 10000
def __init__(self, basedir, configFileName="master.cfg"):
service.MultiService.__init__(self)
self.setName("buildmaster")
self.basedir = basedir
assert os.path.isdir(self.basedir)
self.configFileName = configFileName
# set up child services
self.create_child_services()
# loop for polling the db
self.db_loop = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# subscription points
self._change_subs = \
subscription.SubscriptionPoint("changes")
self._new_buildrequest_subs = \
subscription.SubscriptionPoint("buildrequest_additions")
self._new_buildset_subs = \
subscription.SubscriptionPoint("buildset_additions")
self._complete_buildset_subs = \
subscription.SubscriptionPoint("buildset_completion")
# local cache for this master's object ID
self._object_id = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.change_svc = ChangeManager(self)
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster(self)
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager(self)
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = connector.DBConnector(self, self.basedir)
self.db.setServiceParent(self)
self.debug = debug.DebugServices(self)
self.debug.setServiceParent(self)
self.status = Status(self)
self.status.setServiceParent(self)
# setup and reconfig handling
_already_started = False
@defer.deferredGenerator
def startService(self, _reactor=reactor):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
_reactor.callWhenRunning(d.callback, None)
wfd = defer.waitForDeferred(d)
yield wfd
wfd.getResult()
try:
# load the configuration file, treating errors as fatal
try:
self.config = config.MasterConfig.loadConfig(self.basedir,
self.configFileName)
except config.ConfigErrors, e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
_reactor.stop()
return
except:
log.err(failure.Failure(), 'while starting BuildMaster')
_reactor.stop()
return
# set up services that need access to the config before everything else
# gets told to reconfig
try:
wfd = defer.waitForDeferred(
self.db.setup())
yield wfd
wfd.getResult()
except connector.DatabaseNotReadyError:
# (message was already logged)
_reactor.stop()
return
if hasattr(signal, "SIGHUP"):
def sighup(*args):
_reactor.callLater(0, self.reconfig)
signal.signal(signal.SIGHUP, sighup)
# call the parent method
wfd = defer.waitForDeferred(
defer.maybeDeferred(lambda :
service.MultiService.startService(self)))
yield wfd
wfd.getResult()
# give all services a chance to load the new configuration, rather than
# the base configuration
wfd = defer.waitForDeferred(
self.reconfigService(self.config))
yield wfd
wfd.getResult()
except:
def setUp(self):
self.master = BotMaster(Mock())
self.master.reactor = Mock()
self.master.startService()
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantData=True)
self.botmaster = BotMaster()
yield self.botmaster.setServiceParent(self.master)
self.botmaster.startService()
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantData=True)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self.master)
self.botmaster.startService()
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
self.reactor = mock.Mock()
self.botmaster.startService()
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.workers = workermanager.WorkerManager(self)
self.workers.setServiceParent(self)
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager()
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self.basedir)
self.db.setServiceParent(self)
self.wamp = wampconnector.WampConnector()
self.wamp.setServiceParent(self)
self.mq = mqconnector.MQConnector()
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector()
self.data.setServiceParent(self)
self.www = wwwservice.WWWService()
self.www.setServiceParent(self)
self.debug = debug.DebugServices()
self.debug.setServiceParent(self)
self.status = Status()
self.status.setServiceParent(self)
self.secrets_manager = SecretManager()
self.secrets_manager.setServiceParent(self)
self.secrets_manager.reconfig_priority = 2000
self.service_manager = service.BuildbotServiceManager()
self.service_manager.setServiceParent(self)
self.service_manager.reconfig_priority = 1000
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
yield self.data.updates.expireMasters()
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
self.masterHeartbeatService.clock = self.reactor
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
WorkerAPICompatMixin):
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
def __init__(self,
basedir,
configFileName=None,
umask=None,
reactor=None,
config_loader=None):
service.AsyncMultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
if config_loader is not None and configFileName is not None:
raise config.ConfigErrors([
"Can't specify both `config_loader` and `configFilename`.",
])
elif config_loader is None:
if configFileName is None:
configFileName = 'master.cfg'
config_loader = config.FileLoader(self.basedir, configFileName)
self.config_loader = config_loader
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
self.initLock = defer.DeferredLock()
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" %
(self.hostname, os.path.abspath(self.basedir or '.')))
if isinstance(self.name, bytes):
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.workers = workermanager.WorkerManager(self)
self.workers.setServiceParent(self)
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager()
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self.basedir)
self.db.setServiceParent(self)
self.wamp = wampconnector.WampConnector()
self.wamp.setServiceParent(self)
self.mq = mqconnector.MQConnector()
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector()
self.data.setServiceParent(self)
self.www = wwwservice.WWWService()
self.www.setServiceParent(self)
self.debug = debug.DebugServices()
self.debug.setServiceParent(self)
self.status = Status()
self.status.setServiceParent(self)
self.secrets_manager = SecretManager()
self.secrets_manager.setServiceParent(self)
self.secrets_manager.reconfig_priority = 2000
self.service_manager = service.BuildbotServiceManager()
self.service_manager.setServiceParent(self)
self.service_manager.reconfig_priority = 1000
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
yield self.data.updates.expireMasters()
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
self.masterHeartbeatService.clock = self.reactor
# we do setServiceParent only when the master is configured
# master should advertise itself only at that time
# setup and reconfig handling
_already_started = False
@defer.inlineCallbacks
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
yield self.initLock.acquire()
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
eventually(self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# We make sure the housekeeping is done before configuring in order to cleanup
# any remaining claimed schedulers or change sources from zombie
# masters
yield self.data.updates.expireMasters(forceHouseKeeping=True)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# Mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# Start the heartbeat timer
yield self.masterHeartbeatService.setServiceParent(self)
# send the statistics to buildbot.net, without waiting
self.sendBuildbotNetUsageData()
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
yield self.initLock.release()
self._master_initialized = True
def sendBuildbotNetUsageData(self):
if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
raise RuntimeError(
"Should not enable buildbotNetUsageData in trial tests!")
sendBuildbotNetUsageData(self)
@defer.inlineCallbacks
def stopService(self):
try:
yield self.initLock.acquire()
if self.masterid is not None:
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
if self.running:
yield self.botmaster.cleanShutdown(quickMode=True,
stopReactor=False)
yield service.AsyncMultiService.stopService(self)
log.msg("BuildMaster is stopped")
self._master_initialized = False
finally:
yield self.initLock.release()
def reconfig(self):
# this method wraps doConfig, ensuring it is only ever called once at
# a time, and alerting the user if the reconfig takes too long
if self.reconfig_active:
log.msg("reconfig already active; will reconfig again after")
self.reconfig_requested = True
return
self.reconfig_active = self.reactor.seconds()
metrics.MetricCountEvent.log("loaded_config", 1)
# notify every 10 seconds that the reconfig is still going on, although
# reconfigs should not take that long!
self.reconfig_notifier = task.LoopingCall(
lambda: log.msg("reconfig is ongoing for %d s" %
(self.reactor.seconds() - self.reconfig_active)))
self.reconfig_notifier.start(10, now=False)
timer = metrics.Timer("BuildMaster.reconfig")
timer.start()
d = self.doReconfig()
@d.addBoth
def cleanup(res):
timer.stop()
self.reconfig_notifier.stop()
self.reconfig_notifier = None
self.reconfig_active = False
if self.reconfig_requested:
self.reconfig_requested = False
self.reconfig()
return res
d.addErrback(log.err, 'while reconfiguring')
return d # for tests
@defer.inlineCallbacks
def doReconfig(self):
log.msg("beginning configuration update")
changes_made = False
failed = False
try:
yield self.initLock.acquire()
# Run the master.cfg in thread, so that it can use blocking code
new_config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
changes_made = True
self.config = new_config
yield self.reconfigServiceWithBuildbotConfig(new_config)
except config.ConfigErrors as e:
for msg in e.errors:
log.msg(msg)
failed = True
except Exception:
log.err(failure.Failure(), 'during reconfig:')
failed = True
finally:
yield self.initLock.release()
if failed:
if changes_made:
log.msg("WARNING: reconfig partially applied; master "
"may malfunction")
else:
log.msg("reconfig aborted without making any changes")
else:
log.msg("configuration update complete")
def reconfigServiceWithBuildbotConfig(self, new_config):
if self.configured_db_url is None:
self.configured_db_url = new_config.db['db_url']
elif (self.configured_db_url != new_config.db['db_url']):
config.error(
"Cannot change c['db']['db_url'] after the master has started",
)
if self.config.mq['type'] != new_config.mq['type']:
raise config.ConfigErrors([
"Cannot change c['mq']['type'] after the master has started",
])
return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(
self, new_config)
# informational methods
def allSchedulers(self):
return list(self.scheduler_manager)
def getStatus(self):
"""
@rtype: L{buildbot.status.builder.Status}
"""
return self.status
# state maintenance (private)
def getObjectId(self):
"""
Return the object id for this master, for associating state with the
master.
@returns: ID, via Deferred
"""
# try to get the cached value
if self._object_id is not None:
return defer.succeed(self._object_id)
# failing that, get it from the DB; multiple calls to this function
# at the same time will not hurt
d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster")
@d.addCallback
def keep(id):
self._object_id = id
return id
return d
def _getState(self, name, default=None):
"private wrapper around C{self.db.state.getState}"
d = self.getObjectId()
@d.addCallback
def get(objectid):
return self.db.state.getState(objectid, name, default)
return d
def _setState(self, name, value):
"private wrapper around C{self.db.state.setState}"
d = self.getObjectId()
@d.addCallback
def set(objectid):
return self.db.state.setState(objectid, name, value)
return d
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.master = fakemaster.make_master(testcase=self,
wantMq=True,
wantData=True)
self.master.mq = self.master.mq
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def tearDown(self):
return self.botmaster.stopService()
def test_reconfigService(self):
# check that reconfigServiceSlaves and reconfigServiceBuilders are
# both called; they will be tested invidually below
self.patch(self.botmaster, 'reconfigServiceBuilders',
mock.Mock(side_effect=lambda c: defer.succeed(None)))
self.patch(self.botmaster, 'reconfigServiceSlaves',
mock.Mock(side_effect=lambda c: defer.succeed(None)))
self.patch(self.botmaster, 'maybeStartBuildsForAllBuilders',
mock.Mock())
new_config = mock.Mock()
d = self.botmaster.reconfigService(new_config)
@d.addCallback
def check(_):
self.botmaster.reconfigServiceBuilders.assert_called_with(
new_config)
self.botmaster.reconfigServiceSlaves.assert_called_with(new_config)
self.assertTrue(
self.botmaster.maybeStartBuildsForAllBuilders.called)
return d
@defer.inlineCallbacks
def test_reconfigServiceSlaves_add_remove(self):
sl = FakeBuildSlave('sl1')
self.new_config.slaves = [sl]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
self.assertIdentical(sl.parent, self.botmaster)
self.assertEqual(self.botmaster.slaves, {'sl1': sl})
self.new_config.slaves = []
yield self.botmaster.reconfigServiceSlaves(self.new_config)
self.assertIdentical(sl.parent, None)
self.assertIdentical(sl.master, None)
@defer.inlineCallbacks
def test_reconfigServiceSlaves_reconfig(self):
sl = FakeBuildSlave('sl1')
self.botmaster.slaves = dict(sl1=sl)
sl.setServiceParent(self.botmaster)
sl.master = self.master
sl.botmaster = self.botmaster
sl_new = FakeBuildSlave('sl1')
self.new_config.slaves = [sl_new]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
# sl was not replaced..
self.assertIdentical(self.botmaster.slaves['sl1'], sl)
@defer.inlineCallbacks
def test_reconfigServiceSlaves_class_changes(self):
sl = FakeBuildSlave('sl1')
self.botmaster.slaves = dict(sl1=sl)
sl.setServiceParent(self.botmaster)
sl.master = self.master
sl.botmaster = self.botmaster
sl_new = FakeBuildSlave2('sl1')
self.new_config.slaves = [sl_new]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
# sl *was* replaced (different class)
self.assertIdentical(self.botmaster.slaves['sl1'], sl_new)
@defer.inlineCallbacks
def test_reconfigServiceBuilders_add_remove(self):
bc = config.BuilderConfig(name='bldr',
factory=factory.BuildFactory(),
slavename='f')
self.new_config.builders = [bc]
yield self.botmaster.reconfigServiceBuilders(self.new_config)
bldr = self.botmaster.builders['bldr']
self.assertIdentical(bldr.parent, self.botmaster)
self.assertIdentical(bldr.master, self.master)
self.assertEqual(self.botmaster.builderNames, ['bldr'])
self.new_config.builders = []
yield self.botmaster.reconfigServiceBuilders(self.new_config)
self.assertIdentical(bldr.parent, None)
self.assertIdentical(bldr.master, None)
self.assertEqual(self.botmaster.builders, {})
self.assertEqual(self.botmaster.builderNames, [])
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder('frank')
brd.maybeStartBuildsOn.assert_called_once_with(['frank'])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name='frank')
b1.name = 'frank'
b2 = mock.Mock(name='larry')
b2.name = 'larry'
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave('centos')
self.botmaster.getBuildersForSlave.assert_called_once_with('centos')
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ['frank', 'larry']
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
class TestCleanShutdown(unittest.TestCase):
def setUp(self):
self.master = BotMaster(Mock())
self.master.reactor = Mock()
self.master.startService()
def test_shutdown_idle(self):
"""Test that the master shuts down when it's idle"""
d = self.master.cleanShutdown()
def _check(ign):
self.assertEquals(self.master.reactor.stop.called, True)
d.addCallback(_check)
return d
def test_shutdown_busy(self):
"""Test that the master shuts down after builds finish"""
# Fake some builds
builder = Mock()
build = Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
d_finished = defer.Deferred()
build.waitUntilFinished.return_value = d_finished
self.master.builders = Mock()
self.master.builders.values.return_value = [builder]
d_shutdown = self.master.cleanShutdown()
# Trigger the loop to get things going
self.master.loop.trigger()
# First we wait for it to quiet down again
d = self.master.loop.when_quiet()
# Next we check that we haven't stopped yet, since there's a running
# build
def _check1(ign):
self.assertEquals(self.master.reactor.stop.called, False)
d.addCallback(_check1)
# Now we cause the build to finish, then kick the loop again,
# empty out the list of running builds, and wait for the shutdown
# process to finish
def _finish_build(ign):
d_finished.callback(None)
self.master.loop.trigger()
self.master.builders.values.return_value = []
return d_shutdown
d.addCallback(_finish_build)
# And now we should be done
def _check2(ign):
self.assertEquals(self.master.reactor.stop.called, True)
d.addCallback(_check2)
return d
def test_shutdown_cancel(self):
"""Test that we can cancel a shutdown"""
# Fake some builds
builder = Mock()
build = Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
d_finished = defer.Deferred()
build.waitUntilFinished.return_value = d_finished
self.master.builders = Mock()
self.master.builders.values.return_value = [builder]
d_shutdown = self.master.cleanShutdown()
# Trigger the loop to get things going
self.master.loop.trigger()
# First we wait for it to quiet down again
d = self.master.loop.when_quiet()
# Next we check that we haven't stopped yet, since there's a running
# build.
# We cancel the shutdown here too
def _check1(ign):
self.assertEquals(self.master.reactor.stop.called, False)
self.master.cancelCleanShutdown()
d.addCallback(_check1)
# Now we cause the build to finish, then kick the loop again,
# empty out the list of running builds, and wait for the shutdown
# process to finish
def _finish_build(ign):
d_finished.callback(None)
self.master.loop.trigger()
self.master.builders.values.return_value = []
return d_shutdown
d.addCallback(_finish_build)
# We should still be running!
def _check2(ign):
self.assertEquals(self.master.reactor.stop.called, False)
d.addCallback(_check2)
return d
def test_shutdown_no_new_builds(self):
"""Test that no new builds get handed out when we're shutting down"""
# Fake some builds
builder = Mock()
build = Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
d_finished = defer.Deferred()
build.waitUntilFinished.return_value = d_finished
self.master.builders = Mock()
self.master.builders.values.return_value = [builder]
self.assertEquals(self.master._get_processors(), [builder.run])
d_shutdown = self.master.cleanShutdown()
assert d_shutdown
# Trigger the loop to get things going
self.master.loop.trigger()
# First we wait for it to quiet down again
d = self.master.loop.when_quiet()
# Next we check that we haven't stopped yet, since there's a running
# build.
# Also check that we're not trying to hand out new builds!
def _check1(ign):
self.assertEquals(self.master.reactor.stop.called, False)
self.assertEquals(self.master._get_processors(), [])
d.addCallback(_check1)
return d
def setUp(self):
self.master = BotMaster(Mock())
self.master.reactor = Mock()
self.master.startService()
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
WorkerAPICompatMixin):
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
service.AsyncMultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
if config_loader is not None and configFileName is not None:
raise config.ConfigErrors([
"Can't specify both `config_loader` and `configFilename`.",
])
elif config_loader is None:
if configFileName is None:
configFileName = 'master.cfg'
config_loader = config.FileLoader(self.basedir, configFileName)
self.config_loader = config_loader
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
self.initLock = defer.DeferredLock()
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" % (self.hostname,
os.path.abspath(self.basedir or '.')))
if isinstance(self.name, bytes):
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.workers = workermanager.WorkerManager(self)
self.workers.setServiceParent(self)
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager()
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self.basedir)
self.db.setServiceParent(self)
self.wamp = wampconnector.WampConnector()
self.wamp.setServiceParent(self)
self.mq = mqconnector.MQConnector()
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector()
self.data.setServiceParent(self)
self.www = wwwservice.WWWService()
self.www.setServiceParent(self)
self.debug = debug.DebugServices()
self.debug.setServiceParent(self)
self.status = Status()
self.status.setServiceParent(self)
self.secrets_manager = SecretManager()
self.secrets_manager.setServiceParent(self)
self.secrets_manager.reconfig_priority = 2000
self.service_manager = service.BuildbotServiceManager()
self.service_manager.setServiceParent(self)
self.service_manager.reconfig_priority = 1000
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
yield self.data.updates.expireMasters()
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
# we do setServiceParent only when the master is configured
# master should advertise itself only at that time
# setup and reconfig handling
_already_started = False
@defer.inlineCallbacks
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
yield self.initLock.acquire()
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
eventually(self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(
name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# We make sure the housekeeping is done before configuring in order to cleanup
# any remaining claimed schedulers or change sources from zombie
# masters
yield self.data.updates.expireMasters(forceHouseKeeping=True)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# Mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# Start the heartbeat timer
yield self.masterHeartbeatService.setServiceParent(self)
# send the statistics to buildbot.net, without waiting
self.sendBuildbotNetUsageData()
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
yield self.initLock.release()
self._master_initialized = True
def sendBuildbotNetUsageData(self):
if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None:
raise RuntimeError(
"Should not enable buildbotNetUsageData in trial tests!")
sendBuildbotNetUsageData(self)
@defer.inlineCallbacks
def stopService(self):
try:
yield self.initLock.acquire()
if self.masterid is not None:
yield self.data.updates.masterStopped(
name=self.name, masterid=self.masterid)
if self.running:
yield self.botmaster.cleanShutdown(
quickMode=True, stopReactor=False)
yield service.AsyncMultiService.stopService(self)
log.msg("BuildMaster is stopped")
self._master_initialized = False
finally:
yield self.initLock.release()
def reconfig(self):
# this method wraps doConfig, ensuring it is only ever called once at
# a time, and alerting the user if the reconfig takes too long
if self.reconfig_active:
log.msg("reconfig already active; will reconfig again after")
self.reconfig_requested = True
return
self.reconfig_active = self.reactor.seconds()
metrics.MetricCountEvent.log("loaded_config", 1)
# notify every 10 seconds that the reconfig is still going on, although
# reconfigs should not take that long!
self.reconfig_notifier = task.LoopingCall(lambda:
log.msg("reconfig is ongoing for %d s" %
(self.reactor.seconds() - self.reconfig_active)))
self.reconfig_notifier.start(10, now=False)
timer = metrics.Timer("BuildMaster.reconfig")
timer.start()
d = self.doReconfig()
@d.addBoth
def cleanup(res):
timer.stop()
self.reconfig_notifier.stop()
self.reconfig_notifier = None
self.reconfig_active = False
if self.reconfig_requested:
self.reconfig_requested = False
self.reconfig()
return res
d.addErrback(log.err, 'while reconfiguring')
return d # for tests
@defer.inlineCallbacks
def doReconfig(self):
log.msg("beginning configuration update")
changes_made = False
failed = False
try:
yield self.initLock.acquire()
# Run the master.cfg in thread, so that it can use blocking code
new_config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
changes_made = True
self.config = new_config
yield self.reconfigServiceWithBuildbotConfig(new_config)
except config.ConfigErrors as e:
for msg in e.errors:
log.msg(msg)
failed = True
except Exception:
log.err(failure.Failure(), 'during reconfig:')
failed = True
finally:
yield self.initLock.release()
if failed:
if changes_made:
log.msg("WARNING: reconfig partially applied; master "
"may malfunction")
else:
log.msg("reconfig aborted without making any changes")
else:
log.msg("configuration update complete")
def reconfigServiceWithBuildbotConfig(self, new_config):
if self.configured_db_url is None:
self.configured_db_url = new_config.db['db_url']
elif (self.configured_db_url != new_config.db['db_url']):
config.error(
"Cannot change c['db']['db_url'] after the master has started",
)
if self.config.mq['type'] != new_config.mq['type']:
raise config.ConfigErrors([
"Cannot change c['mq']['type'] after the master has started",
])
return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
new_config)
# informational methods
def allSchedulers(self):
return list(self.scheduler_manager)
def getStatus(self):
"""
@rtype: L{buildbot.status.builder.Status}
"""
return self.status
# state maintenance (private)
def getObjectId(self):
"""
Return the object id for this master, for associating state with the
master.
@returns: ID, via Deferred
"""
# try to get the cached value
if self._object_id is not None:
return defer.succeed(self._object_id)
# failing that, get it from the DB; multiple calls to this function
# at the same time will not hurt
d = self.db.state.getObjectId(self.name,
"buildbot.master.BuildMaster")
@d.addCallback
def keep(id):
self._object_id = id
return id
return d
def _getState(self, name, default=None):
"private wrapper around C{self.db.state.getState}"
d = self.getObjectId()
@d.addCallback
def get(objectid):
return self.db.state.getState(objectid, name, default)
return d
def _setState(self, name, value):
"private wrapper around C{self.db.state.setState}"
d = self.getObjectId()
@d.addCallback
def set(objectid):
return self.db.state.setState(objectid, name, value)
return d
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
class BuildMaster(config.ReconfigurableServiceMixin, service.AsyncMultiService):
# frequency with which to reclaim running builds; this should be set to
# something fairly long, to avoid undue database load
RECLAIM_BUILD_INTERVAL = 10 * 60
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
def __init__(self, basedir, configFileName="master.cfg", umask=None):
service.AsyncMultiService.__init__(self)
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" % (self.hostname,
os.path.abspath(self.basedir or '.')))
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.buildslaves = bslavemanager.BuildslaveManager(self)
self.buildslaves.setServiceParent(self)
self.change_svc = ChangeManager(self)
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster(self)
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager(self)
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self, self.basedir)
self.db.setServiceParent(self)
self.mq = mqconnector.MQConnector(self)
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector(self)
self.data.setServiceParent(self)
self.www = wwwservice.WWWService(self)
self.www.setServiceParent(self)
self.debug = debug.DebugServices(self)
self.debug.setServiceParent(self)
self.status = Status(self)
self.status.setServiceParent(self)
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
yield self.data.updates.expireMasters()
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
# setup and reconfig handling
_already_started = False
@defer.inlineCallbacks
def startService(self, _reactor=reactor):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
_reactor.callWhenRunning(d.callback, None)
yield d
try:
# load the configuration file, treating errors as fatal
try:
self.config = config.MasterConfig.loadConfig(self.basedir,
self.configFileName)
except config.ConfigErrors, e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
_reactor.stop()
return
except:
log.err(failure.Failure(), 'while starting BuildMaster')
_reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
_reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
_reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(
name=self.name)
yield self.doMasterHouseKeeping(self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigService(self.config)
# mark the master as active now that mq is running
yield self.data.updates.masterActive(
name=self.name,
masterid=self.masterid)
except:
def setUp(self):
self.master = fakemaster.make_master()
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
class TestCleanShutdown(TestReactorMixin, unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantData=True)
self.botmaster = BotMaster()
yield self.botmaster.setServiceParent(self.master)
self.botmaster.startService()
def assertReactorStopped(self, _=None):
self.assertTrue(self.reactor.stop_called)
def assertReactorNotStopped(self, _=None):
self.assertFalse(self.reactor.stop_called)
def makeFakeBuild(self, waitedFor=False):
self.fake_builder = builder = mock.Mock()
build_status = mock.Mock()
builder.builder_status.getCurrentBuilds.return_value = [build_status]
self.build_deferred = defer.Deferred()
request = mock.Mock()
request.waitedFor = waitedFor
build = mock.Mock()
build.stopBuild = self.stopFakeBuild
build.waitUntilFinished.return_value = self.build_deferred
build.requests = [request]
builder.building = [build]
self.botmaster.builders = mock.Mock()
self.botmaster.builders.values.return_value = [builder]
def stopFakeBuild(self, reason, results):
self.reason = reason
self.results = results
self.finishFakeBuild()
def finishFakeBuild(self):
self.fake_builder.building = []
self.build_deferred.callback(None)
# tests
def test_shutdown_idle(self):
"""Test that the master shuts down when it's idle"""
self.botmaster.cleanShutdown()
self.assertReactorStopped()
def test_shutdown_busy(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild()
self.botmaster.cleanShutdown()
# check that we haven't stopped yet, since there's a running build
self.assertReactorNotStopped()
# try to shut it down again, just to check that this does not fail
self.botmaster.cleanShutdown()
# Now we cause the build to finish
self.finishFakeBuild()
# And now we should be stopped
self.assertReactorStopped()
def test_shutdown_busy_quick(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(quickMode=True)
# And now we should be stopped
self.assertReactorStopped()
self.assertEqual(self.results, RETRY)
def test_shutdown_busy_quick_cancelled(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild(waitedFor=True)
self.botmaster.cleanShutdown(quickMode=True)
# And now we should be stopped
self.assertReactorStopped()
self.assertEqual(self.results, CANCELLED)
def test_shutdown_cancel_not_shutting_down(self):
"""Test that calling cancelCleanShutdown when none is in progress
works"""
# this just shouldn't fail..
self.botmaster.cancelCleanShutdown()
def test_shutdown_cancel(self):
"""Test that we can cancel a shutdown"""
self.makeFakeBuild()
self.botmaster.cleanShutdown()
# Next we check that we haven't stopped yet, since there's a running
# build.
self.assertReactorNotStopped()
# but the BuildRequestDistributor should not be running
self.assertFalse(self.botmaster.brd.running)
# Cancel the shutdown
self.botmaster.cancelCleanShutdown()
# Now we cause the build to finish
self.finishFakeBuild()
# We should still be running!
self.assertReactorNotStopped()
# and the BuildRequestDistributor should be, as well
self.assertTrue(self.botmaster.brd.running)
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.master = fakemaster.make_master(testcase=self, wantMq=True,
wantData=True)
self.master.mq = self.master.mq
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def tearDown(self):
return self.botmaster.stopService()
def test_reconfigService(self):
# check that reconfigServiceBuilders is called.
self.patch(self.botmaster, 'reconfigServiceBuilders',
mock.Mock(side_effect=lambda c: defer.succeed(None)))
self.patch(self.botmaster, 'maybeStartBuildsForAllBuilders',
mock.Mock())
new_config = mock.Mock()
d = self.botmaster.reconfigService(new_config)
@d.addCallback
def check(_):
self.botmaster.reconfigServiceBuilders.assert_called_with(
new_config)
self.assertTrue(
self.botmaster.maybeStartBuildsForAllBuilders.called)
return d
@defer.inlineCallbacks
def test_reconfigServiceBuilders_add_remove(self):
bc = config.BuilderConfig(name='bldr', factory=factory.BuildFactory(),
slavename='f')
self.new_config.builders = [bc]
yield self.botmaster.reconfigServiceBuilders(self.new_config)
bldr = self.botmaster.builders['bldr']
self.assertIdentical(bldr.parent, self.botmaster)
self.assertIdentical(bldr.master, self.master)
self.assertEqual(self.botmaster.builderNames, ['bldr'])
self.new_config.builders = []
yield self.botmaster.reconfigServiceBuilders(self.new_config)
self.assertIdentical(bldr.parent, None)
self.assertIdentical(bldr.master, None)
self.assertEqual(self.botmaster.builders, {})
self.assertEqual(self.botmaster.builderNames, [])
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder('frank')
brd.maybeStartBuildsOn.assert_called_once_with(['frank'])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name='frank')
b1.name = 'frank'
b2 = mock.Mock(name='larry')
b2.name = 'larry'
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave('centos')
self.botmaster.getBuildersForSlave.assert_called_once_with('centos')
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ['frank', 'larry']
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
class BuildMaster(config.ReconfigurableServiceMixin,
service.AsyncMultiService):
# frequency with which to reclaim running builds; this should be set to
# something fairly long, to avoid undue database load
RECLAIM_BUILD_INTERVAL = 10 * 60
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
def __init__(self, basedir, configFileName="master.cfg", umask=None):
service.AsyncMultiService.__init__(self)
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" %
(self.hostname, os.path.abspath(self.basedir or '.')))
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.buildslaves = bslavemanager.BuildslaveManager(self)
self.buildslaves.setServiceParent(self)
self.change_svc = ChangeManager(self)
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster(self)
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager(self)
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self, self.basedir)
self.db.setServiceParent(self)
self.mq = mqconnector.MQConnector(self)
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector(self)
self.data.setServiceParent(self)
self.www = wwwservice.WWWService(self)
self.www.setServiceParent(self)
self.debug = debug.DebugServices(self)
self.debug.setServiceParent(self)
self.status = Status(self)
self.status.setServiceParent(self)
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
yield self.data.updates.expireMasters()
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
# setup and reconfig handling
_already_started = False
@defer.inlineCallbacks
def startService(self, _reactor=reactor):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
_reactor.callWhenRunning(d.callback, None)
yield d
try:
# load the configuration file, treating errors as fatal
try:
self.config = config.MasterConfig.loadConfig(
self.basedir, self.configFileName)
except config.ConfigErrors, e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
_reactor.stop()
return
except:
log.err(failure.Failure(), 'while starting BuildMaster')
_reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
_reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
_reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(name=self.name)
yield self.doMasterHouseKeeping(self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigService(self.config)
# mark the master as active now that mq is running
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
except:
class TestCleanShutdown(TestReactorMixin, unittest.TestCase):
def setUp(self):
self.setUpTestReactor()
self.master = fakemaster.make_master(self, wantData=True)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self.master)
self.botmaster.startService()
def assertReactorStopped(self, _=None):
self.assertTrue(self.reactor.stop_called)
def assertReactorNotStopped(self, _=None):
self.assertFalse(self.reactor.stop_called)
def makeFakeBuild(self, waitedFor=False):
self.fake_builder = builder = mock.Mock()
build_status = mock.Mock()
builder.builder_status.getCurrentBuilds.return_value = [build_status]
self.build_deferred = defer.Deferred()
request = mock.Mock()
request.waitedFor = waitedFor
build = mock.Mock()
build.stopBuild = self.stopFakeBuild
build.waitUntilFinished.return_value = self.build_deferred
build.requests = [request]
builder.building = [build]
self.botmaster.builders = mock.Mock()
self.botmaster.builders.values.return_value = [builder]
def stopFakeBuild(self, reason, results):
self.reason = reason
self.results = results
self.finishFakeBuild()
def finishFakeBuild(self):
self.fake_builder.building = []
self.build_deferred.callback(None)
# tests
def test_shutdown_idle(self):
"""Test that the master shuts down when it's idle"""
self.botmaster.cleanShutdown(_reactor=self.reactor)
self.assertReactorStopped()
def test_shutdown_busy(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(_reactor=self.reactor)
# check that we haven't stopped yet, since there's a running build
self.assertReactorNotStopped()
# try to shut it down again, just to check that this does not fail
self.botmaster.cleanShutdown(_reactor=self.reactor)
# Now we cause the build to finish
self.finishFakeBuild()
# And now we should be stopped
self.assertReactorStopped()
def test_shutdown_busy_quick(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(quickMode=True, _reactor=self.reactor)
# And now we should be stopped
self.assertReactorStopped()
self.assertEqual(self.results, RETRY)
def test_shutdown_busy_quick_cancelled(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild(waitedFor=True)
self.botmaster.cleanShutdown(quickMode=True, _reactor=self.reactor)
# And now we should be stopped
self.assertReactorStopped()
self.assertEqual(self.results, CANCELLED)
def test_shutdown_cancel_not_shutting_down(self):
"""Test that calling cancelCleanShutdown when none is in progress
works"""
# this just shouldn't fail..
self.botmaster.cancelCleanShutdown()
def test_shutdown_cancel(self):
"""Test that we can cancel a shutdown"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(_reactor=self.reactor)
# Next we check that we haven't stopped yet, since there's a running
# build.
self.assertReactorNotStopped()
# but the BuildRequestDistributor should not be running
self.assertFalse(self.botmaster.brd.running)
# Cancel the shutdown
self.botmaster.cancelCleanShutdown()
# Now we cause the build to finish
self.finishFakeBuild()
# We should still be running!
self.assertReactorNotStopped()
# and the BuildRequestDistributor should be, as well
self.assertTrue(self.botmaster.brd.running)
class TestCleanShutdown(unittest.TestCase):
def setUp(self):
self.master = BotMaster(Mock())
self.master.reactor = Mock()
self.master.startService()
def test_shutdown_idle(self):
"""Test that the master shuts down when it's idle"""
d = self.master.cleanShutdown()
def _check(ign):
self.assertEquals(self.master.reactor.stop.called, True)
d.addCallback(_check)
return d
def test_shutdown_busy(self):
"""Test that the master shuts down after builds finish"""
# Fake some builds
builder = Mock()
build = Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
d_finished = defer.Deferred()
build.waitUntilFinished.return_value = d_finished
self.master.builders = Mock()
self.master.builders.values.return_value = [builder]
d_shutdown = self.master.cleanShutdown()
# Trigger the loop to get things going
self.master.loop.trigger()
# First we wait for it to quiet down again
d = self.master.loop.when_quiet()
# Next we check that we haven't stopped yet, since there's a running
# build
def _check1(ign):
self.assertEquals(self.master.reactor.stop.called, False)
d.addCallback(_check1)
# Now we cause the build to finish, then kick the loop again,
# empty out the list of running builds, and wait for the shutdown
# process to finish
def _finish_build(ign):
d_finished.callback(None)
self.master.loop.trigger()
self.master.builders.values.return_value = []
return d_shutdown
d.addCallback(_finish_build)
# And now we should be done
def _check2(ign):
self.assertEquals(self.master.reactor.stop.called, True)
d.addCallback(_check2)
return d
def test_shutdown_cancel(self):
"""Test that we can cancel a shutdown"""
# Fake some builds
builder = Mock()
build = Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
d_finished = defer.Deferred()
build.waitUntilFinished.return_value = d_finished
self.master.builders = Mock()
self.master.builders.values.return_value = [builder]
d_shutdown = self.master.cleanShutdown()
# Trigger the loop to get things going
self.master.loop.trigger()
# First we wait for it to quiet down again
d = self.master.loop.when_quiet()
# Next we check that we haven't stopped yet, since there's a running
# build.
# We cancel the shutdown here too
def _check1(ign):
self.assertEquals(self.master.reactor.stop.called, False)
self.master.cancelCleanShutdown()
d.addCallback(_check1)
# Now we cause the build to finish, then kick the loop again,
# empty out the list of running builds, and wait for the shutdown
# process to finish
def _finish_build(ign):
d_finished.callback(None)
self.master.loop.trigger()
self.master.builders.values.return_value = []
return d_shutdown
d.addCallback(_finish_build)
# We should still be running!
def _check2(ign):
self.assertEquals(self.master.reactor.stop.called, False)
d.addCallback(_check2)
return d
def test_shutdown_no_new_builds(self):
"""Test that no new builds get handed out when we're shutting down"""
# Fake some builds
builder = Mock()
build = Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
d_finished = defer.Deferred()
build.waitUntilFinished.return_value = d_finished
self.master.builders = Mock()
self.master.builders.values.return_value = [builder]
self.assertEquals(self.master._get_processors(), [builder.run])
d_shutdown = self.master.cleanShutdown()
assert d_shutdown
# Trigger the loop to get things going
self.master.loop.trigger()
# First we wait for it to quiet down again
d = self.master.loop.when_quiet()
# Next we check that we haven't stopped yet, since there's a running
# build.
# Also check that we're not trying to hand out new builds!
def _check1(ign):
self.assertEquals(self.master.reactor.stop.called, False)
self.assertEquals(self.master._get_processors(), [])
d.addCallback(_check1)
return d
class TestCleanShutdown(unittest.TestCase):
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
self.reactor = mock.Mock()
self.botmaster.startService()
def assertReactorStopped(self, _=None):
self.assertTrue(self.reactor.stop.called)
def assertReactorNotStopped(self, _=None):
self.assertFalse(self.reactor.stop.called)
def makeFakeBuild(self):
self.fake_builder = builder = mock.Mock()
build = mock.Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
self.build_deferred = defer.Deferred()
build.waitUntilFinished.return_value = self.build_deferred
self.botmaster.builders = mock.Mock()
self.botmaster.builders.values.return_value = [builder]
def finishFakeBuild(self):
self.fake_builder.builder_status.getCurrentBuilds.return_value = []
self.build_deferred.callback(None)
# tests
def test_shutdown_idle(self):
"""Test that the master shuts down when it's idle"""
self.botmaster.cleanShutdown(_reactor=self.reactor)
self.assertReactorStopped()
def test_shutdown_busy(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(_reactor=self.reactor)
# check that we haven't stopped yet, since there's a running build
self.assertReactorNotStopped()
# try to shut it down again, just to check that this does not fail
self.botmaster.cleanShutdown(_reactor=self.reactor)
# Now we cause the build to finish
self.finishFakeBuild()
# And now we should be stopped
self.assertReactorStopped()
def test_shutdown_cancel_not_shutting_down(self):
"""Test that calling cancelCleanShutdown when none is in progress
works"""
# this just shouldn't fail..
self.botmaster.cancelCleanShutdown()
def test_shutdown_cancel(self):
"""Test that we can cancel a shutdown"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(_reactor=self.reactor)
# Next we check that we haven't stopped yet, since there's a running
# build.
self.assertReactorNotStopped()
# but the BuildRequestDistributor should not be running
self.assertFalse(self.botmaster.brd.running)
# Cancel the shutdown
self.botmaster.cancelCleanShutdown()
# Now we cause the build to finish
self.finishFakeBuild()
# We should still be running!
self.assertReactorNotStopped()
# and the BuildRequestDistributor should be, as well
self.assertTrue(self.botmaster.brd.running)
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.master = fakemaster.make_master(testcase=self,
wantMq=True,
wantData=True)
self.master.mq = self.master.mq
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def tearDown(self):
return self.botmaster.stopService()
def test_reconfigServiceWithBuildbotConfig(self):
# check that reconfigServiceBuilders is called.
self.patch(self.botmaster, 'reconfigServiceBuilders',
mock.Mock(side_effect=lambda c: defer.succeed(None)))
self.patch(self.botmaster, 'maybeStartBuildsForAllBuilders',
mock.Mock())
new_config = mock.Mock()
d = self.botmaster.reconfigServiceWithBuildbotConfig(new_config)
@d.addCallback
def check(_):
self.botmaster.reconfigServiceBuilders.assert_called_with(
new_config)
self.assertTrue(
self.botmaster.maybeStartBuildsForAllBuilders.called)
return d
@defer.inlineCallbacks
def test_reconfigServiceBuilders_add_remove(self):
bc = config.BuilderConfig(name='bldr',
factory=factory.BuildFactory(),
slavename='f')
self.new_config.builders = [bc]
yield self.botmaster.reconfigServiceBuilders(self.new_config)
bldr = self.botmaster.builders['bldr']
self.assertIdentical(bldr.parent, self.botmaster)
self.assertIdentical(bldr.master, self.master)
self.assertEqual(self.botmaster.builderNames, ['bldr'])
self.new_config.builders = []
yield self.botmaster.reconfigServiceBuilders(self.new_config)
self.assertIdentical(bldr.parent, None)
self.assertIdentical(bldr.master, None)
self.assertEqual(self.botmaster.builders, {})
self.assertEqual(self.botmaster.builderNames, [])
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder('frank')
brd.maybeStartBuildsOn.assert_called_once_with(['frank'])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name='frank')
b1.name = 'frank'
b2 = mock.Mock(name='larry')
b2.name = 'larry'
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave('centos')
self.botmaster.getBuildersForSlave.assert_called_once_with('centos')
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ['frank', 'larry']
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.master = fakemaster.make_master()
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def tearDown(self):
return self.botmaster.stopService()
def test_reconfigService(self):
# check that reconfigServiceSlaves and reconfigServiceBuilders are
# both called; they will be tested invidually below
self.patch(self.botmaster, 'reconfigServiceBuilders',
mock.Mock(side_effect=lambda c : defer.succeed(None)))
self.patch(self.botmaster, 'reconfigServiceSlaves',
mock.Mock(side_effect=lambda c : defer.succeed(None)))
self.patch(self.botmaster, 'maybeStartBuildsForAllBuilders',
mock.Mock())
old_config, new_config = mock.Mock(), mock.Mock()
d = self.botmaster.reconfigService(new_config)
@d.addCallback
def check(_):
self.botmaster.reconfigServiceBuilders.assert_called_with(
new_config)
self.botmaster.reconfigServiceSlaves.assert_called_with(
new_config)
self.assertTrue(
self.botmaster.maybeStartBuildsForAllBuilders.called)
return d
@defer.inlineCallbacks
def test_reconfigServiceSlaves_add_remove(self):
sl = FakeBuildSlave('sl1')
self.new_config.slaves = [ sl ]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
self.assertIdentical(sl.parent, self.botmaster)
self.assertEqual(self.botmaster.slaves, { 'sl1' : sl })
self.new_config.slaves = [ ]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
self.assertIdentical(sl.parent, None)
self.assertIdentical(sl.master, None)
@defer.inlineCallbacks
def test_reconfigServiceSlaves_reconfig(self):
sl = FakeBuildSlave('sl1')
self.botmaster.slaves = dict(sl1=sl)
sl.setServiceParent(self.botmaster)
sl.master = self.master
sl.botmaster = self.botmaster
sl_new = FakeBuildSlave('sl1')
self.new_config.slaves = [ sl_new ]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
# sl was not replaced..
self.assertIdentical(self.botmaster.slaves['sl1'], sl)
@defer.inlineCallbacks
def test_reconfigServiceSlaves_class_changes(self):
sl = FakeBuildSlave('sl1')
self.botmaster.slaves = dict(sl1=sl)
sl.setServiceParent(self.botmaster)
sl.master = self.master
sl.botmaster = self.botmaster
sl_new = FakeBuildSlave2('sl1')
self.new_config.slaves = [ sl_new ]
yield self.botmaster.reconfigServiceSlaves(self.new_config)
# sl *was* replaced (different class)
self.assertIdentical(self.botmaster.slaves['sl1'], sl_new)
@defer.inlineCallbacks
def test_reconfigServiceBuilders_add_remove(self):
bc = config.BuilderConfig(name='bldr', factory=mock.Mock(),
slavename='f')
self.new_config.builders = [ bc ]
yield self.botmaster.reconfigServiceBuilders(self.new_config)
bldr = self.botmaster.builders['bldr']
self.assertIdentical(bldr.parent, self.botmaster)
self.assertIdentical(bldr.master, self.master)
self.assertEqual(self.botmaster.builderNames, [ 'bldr' ])
self.new_config.builders = [ ]
yield self.botmaster.reconfigServiceBuilders(self.new_config)
self.assertIdentical(bldr.parent, None)
self.assertIdentical(bldr.master, None)
self.assertEqual(self.botmaster.builders, {})
self.assertEqual(self.botmaster.builderNames, [])
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder('frank')
brd.maybeStartBuildsOn.assert_called_once_with(['frank'])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name='frank')
b1.name = 'frank'
b2 = mock.Mock(name='larry')
b2.name = 'larry'
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave('centos')
self.botmaster.getBuildersForSlave.assert_called_once_with('centos')
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ['frank', 'larry']
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(['frank', 'larry'])
class TestCleanShutdown(unittest.TestCase):
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
self.reactor = mock.Mock()
self.botmaster.startService()
def assertReactorStopped(self, _=None):
self.assertTrue(self.reactor.stop.called)
def assertReactorNotStopped(self, _=None):
self.assertFalse(self.reactor.stop.called)
def makeFakeBuild(self):
self.fake_builder = builder = mock.Mock()
build = mock.Mock()
builder.builder_status.getCurrentBuilds.return_value = [build]
self.build_deferred = defer.Deferred()
build.waitUntilFinished.return_value = self.build_deferred
self.botmaster.builders = mock.Mock()
self.botmaster.builders.values.return_value = [builder]
def finishFakeBuild(self):
self.fake_builder.builder_status.getCurrentBuilds.return_value = []
self.build_deferred.callback(None)
# tests
def test_shutdown_idle(self):
"""Test that the master shuts down when it's idle"""
self.botmaster.cleanShutdown(_reactor=self.reactor)
self.assertReactorStopped()
def test_shutdown_busy(self):
"""Test that the master shuts down after builds finish"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(_reactor=self.reactor)
# check that we haven't stopped yet, since there's a running build
self.assertReactorNotStopped()
# try to shut it down again, just to check that this does not fail
self.botmaster.cleanShutdown(_reactor=self.reactor)
# Now we cause the build to finish
self.finishFakeBuild()
# And now we should be stopped
self.assertReactorStopped()
def test_shutdown_cancel_not_shutting_down(self):
"""Test that calling cancelCleanShutdown when none is in progress
works"""
# this just shouldn't fail..
self.botmaster.cancelCleanShutdown()
def test_shutdown_cancel(self):
"""Test that we can cancel a shutdown"""
self.makeFakeBuild()
self.botmaster.cleanShutdown(_reactor=self.reactor)
# Next we check that we haven't stopped yet, since there's a running
# build.
self.assertReactorNotStopped()
# but the BuildRequestDistributor should not be running
self.assertFalse(self.botmaster.brd.running)
# Cancel the shutdown
self.botmaster.cancelCleanShutdown()
# Now we cause the build to finish
self.finishFakeBuild()
# We should still be running!
self.assertReactorNotStopped()
# and the BuildRequestDistributor should be, as well
self.assertTrue(self.botmaster.brd.running)
def setUp(self):
self.botmaster = BotMaster(mock.Mock())
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService,
WorkerAPICompatMixin):
# frequency with which to reclaim running builds; this should be set to
# something fairly long, to avoid undue database load
RECLAIM_BUILD_INTERVAL = 10 * 60
# multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered
# unclaimed; this should be at least 2 to avoid false positives
UNCLAIMED_BUILD_FACTOR = 6
def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None):
service.AsyncMultiService.__init__(self)
if reactor is None:
from twisted.internet import reactor
self.reactor = reactor
self.setName("buildmaster")
self.umask = umask
self.basedir = basedir
if basedir is not None: # None is used in tests
assert os.path.isdir(self.basedir)
if config_loader is not None and configFileName is not None:
raise config.ConfigErrors([
"Can't specify both `config_loader` and `configFilename`.",
])
elif config_loader is None:
if configFileName is None:
configFileName = 'master.cfg'
config_loader = config.FileLoader(self.basedir, configFileName)
self.config_loader = config_loader
self.configFileName = configFileName
# flag so we don't try to do fancy things before the master is ready
self._master_initialized = False
# set up child services
self.create_child_services()
# db configured values
self.configured_db_url = None
# configuration / reconfiguration handling
self.config = config.MasterConfig()
self.reconfig_active = False
self.reconfig_requested = False
self.reconfig_notifier = None
# this stores parameters used in the tac file, and is accessed by the
# WebStatus to duplicate those values.
self.log_rotation = LogRotation()
# local cache for this master's object ID
self._object_id = None
# Check environment is sensible
check_functional_environment(self.config)
# figure out local hostname
try:
self.hostname = os.uname()[1] # only on unix
except AttributeError:
self.hostname = socket.getfqdn()
# public attributes
self.name = ("%s:%s" % (self.hostname,
os.path.abspath(self.basedir or '.')))
self.name = self.name.decode('ascii', 'replace')
self.masterid = None
def create_child_services(self):
# note that these are order-dependent. If you get the order wrong,
# you'll know it, as the master will fail to start.
self.metrics = metrics.MetricLogObserver()
self.metrics.setServiceParent(self)
self.caches = cache.CacheManager()
self.caches.setServiceParent(self)
self.pbmanager = buildbot.pbmanager.PBManager()
self.pbmanager.setServiceParent(self)
self.workers = workermanager.WorkerManager(self)
self.workers.setServiceParent(self)
self.change_svc = ChangeManager()
self.change_svc.setServiceParent(self)
self.botmaster = BotMaster()
self.botmaster.setServiceParent(self)
self.scheduler_manager = SchedulerManager()
self.scheduler_manager.setServiceParent(self)
self.user_manager = UserManagerManager(self)
self.user_manager.setServiceParent(self)
self.db = dbconnector.DBConnector(self.basedir)
self.db.setServiceParent(self)
self.wamp = wampconnector.WampConnector()
self.wamp.setServiceParent(self)
self.mq = mqconnector.MQConnector()
self.mq.setServiceParent(self)
self.data = dataconnector.DataConnector()
self.data.setServiceParent(self)
self.www = wwwservice.WWWService()
self.www.setServiceParent(self)
self.debug = debug.DebugServices()
self.debug.setServiceParent(self)
self.status = Status()
self.status.setServiceParent(self)
self.service_manager = service.BuildbotServiceManager()
self.service_manager.setServiceParent(self)
self.service_manager.reconfig_priority = 1000
self.masterHouskeepingTimer = 0
@defer.inlineCallbacks
def heartbeat():
if self.masterid is not None:
yield self.data.updates.masterActive(name=self.name,
masterid=self.masterid)
# force housekeeping once a day
yield self.data.updates.expireMasters((self.masterHouskeepingTimer % (24 * 60)) == 0)
self.masterHouskeepingTimer += 1
self.masterHeartbeatService = internet.TimerService(60, heartbeat)
# we do setServiceParent only when the master is configured
# master should advertise itself only at that time
# setup and reconfig handling
_already_started = False
@defer.inlineCallbacks
def startService(self):
assert not self._already_started, "can only start the master once"
self._already_started = True
log.msg("Starting BuildMaster -- buildbot.version: %s" %
buildbot.version)
# Set umask
if self.umask is not None:
os.umask(self.umask)
# first, apply all monkeypatches
monkeypatches.patch_all()
# we want to wait until the reactor is running, so we can call
# reactor.stop() for fatal errors
d = defer.Deferred()
self.reactor.callWhenRunning(d.callback, None)
yield d
startup_succeed = False
try:
# load the configuration file, treating errors as fatal
try:
# run the master.cfg in thread, so that it can use blocking
# code
self.config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
except config.ConfigErrors as e:
log.msg("Configuration Errors:")
for msg in e.errors:
log.msg(" " + msg)
log.msg("Halting master.")
self.reactor.stop()
return
except Exception:
log.err(failure.Failure(), 'while starting BuildMaster')
self.reactor.stop()
return
# set up services that need access to the config before everything
# else gets told to reconfig
try:
yield self.db.setup()
except exceptions.DatabaseNotReadyError:
# (message was already logged)
self.reactor.stop()
return
self.mq.setup()
if hasattr(signal, "SIGHUP"):
def sighup(*args):
eventually(self.reconfig)
signal.signal(signal.SIGHUP, sighup)
if hasattr(signal, "SIGUSR1"):
def sigusr1(*args):
self.reactor.callLater(0, self.botmaster.cleanShutdown)
signal.signal(signal.SIGUSR1, sigusr1)
# get the masterid so other services can use it in
# startup/reconfig. This goes directly to the DB since the data
# API isn't initialized yet, and anyway, this method is aware of
# the DB API since it just called its setup function
self.masterid = yield self.db.masters.findMasterId(
name=self.name)
# mark this master as stopped, in case it crashed before
yield self.data.updates.masterStopped(name=self.name,
masterid=self.masterid)
# call the parent method
yield service.AsyncMultiService.startService(self)
# give all services a chance to load the new configuration, rather
# than the base configuration
yield self.reconfigServiceWithBuildbotConfig(self.config)
# mark the master as active now that mq is running
yield self.data.updates.masterActive(
name=self.name,
masterid=self.masterid)
startup_succeed = True
except Exception:
f = failure.Failure()
log.err(f, 'while starting BuildMaster')
self.reactor.stop()
finally:
if startup_succeed:
log.msg("BuildMaster is running")
else:
log.msg("BuildMaster startup failed")
self._master_initialized = True
@defer.inlineCallbacks
def stopService(self):
if self.masterid is not None:
yield self.data.updates.masterStopped(
name=self.name, masterid=self.masterid)
if self.running:
yield service.AsyncMultiService.stopService(self)
log.msg("BuildMaster is stopped")
self._master_initialized = False
def reconfig(self):
# this method wraps doConfig, ensuring it is only ever called once at
# a time, and alerting the user if the reconfig takes too long
if self.reconfig_active:
log.msg("reconfig already active; will reconfig again after")
self.reconfig_requested = True
return
self.reconfig_active = self.reactor.seconds()
metrics.MetricCountEvent.log("loaded_config", 1)
# notify every 10 seconds that the reconfig is still going on, although
# reconfigs should not take that long!
self.reconfig_notifier = task.LoopingCall(lambda:
log.msg("reconfig is ongoing for %d s" %
(self.reactor.seconds() - self.reconfig_active)))
self.reconfig_notifier.start(10, now=False)
timer = metrics.Timer("BuildMaster.reconfig")
timer.start()
d = self.doReconfig()
@d.addBoth
def cleanup(res):
timer.stop()
self.reconfig_notifier.stop()
self.reconfig_notifier = None
self.reconfig_active = False
if self.reconfig_requested:
self.reconfig_requested = False
self.reconfig()
return res
@d.addCallback
def advertiseItself(res):
if self.masterHeartbeatService.parent is None:
self.masterHeartbeatService.setServiceParent(self)
d.addErrback(log.err, 'while reconfiguring')
return d # for tests
@defer.inlineCallbacks
def doReconfig(self):
log.msg("beginning configuration update")
changes_made = False
failed = False
try:
# Run the master.cfg in thread, so that it cas use blocking code
new_config = yield threads.deferToThreadPool(
self.reactor, self.reactor.getThreadPool(),
self.config_loader.loadConfig)
changes_made = True
self.config = new_config
yield self.reconfigServiceWithBuildbotConfig(new_config)
except config.ConfigErrors as e:
for msg in e.errors:
log.msg(msg)
failed = True
except Exception:
log.err(failure.Failure(), 'during reconfig:')
failed = True
if failed:
if changes_made:
log.msg("WARNING: reconfig partially applied; master "
"may malfunction")
else:
log.msg("reconfig aborted without making any changes")
else:
log.msg("configuration update complete")
def reconfigServiceWithBuildbotConfig(self, new_config):
if self.configured_db_url is None:
self.configured_db_url = new_config.db['db_url']
elif (self.configured_db_url != new_config.db['db_url']):
config.error(
"Cannot change c['db']['db_url'] after the master has started",
)
if self.config.mq['type'] != new_config.mq['type']:
raise config.ConfigErrors([
"Cannot change c['mq']['type'] after the master has started",
])
return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self,
new_config)
# informational methods
def allSchedulers(self):
return list(self.scheduler_manager)
def getStatus(self):
"""
@rtype: L{buildbot.status.builder.Status}
"""
return self.status
# triggering methods
@defer.inlineCallbacks
def addChange(self, who=None, files=None, comments=None, **kwargs):
# deprecated in 0.9.0; will be removed in 1.0.0
log.msg("WARNING: change source is using deprecated "
"self.master.addChange method; this method will disappear in "
"Buildbot-1.0.0")
# handle positional arguments
kwargs['who'] = who
kwargs['files'] = files
kwargs['comments'] = comments
def handle_deprec(oldname, newname):
if oldname not in kwargs:
return
old = kwargs.pop(oldname)
if old is not None:
if kwargs.get(newname) is None:
log.msg("WARNING: change source is using deprecated "
"addChange parameter '%s'" % oldname)
return old
raise TypeError("Cannot provide '%s' and '%s' to addChange"
% (oldname, newname))
return kwargs.get(newname)
kwargs['author'] = handle_deprec("who", "author")
kwargs['when_timestamp'] = handle_deprec("when", "when_timestamp")
# timestamp must be an epoch timestamp now
if isinstance(kwargs.get('when_timestamp'), datetime.datetime):
kwargs['when_timestamp'] = datetime2epoch(kwargs['when_timestamp'])
# unicodify stuff
for k in ('comments', 'author', 'revision', 'branch', 'category',
'revlink', 'repository', 'codebase', 'project'):
if k in kwargs:
kwargs[k] = ascii2unicode(kwargs[k])
if kwargs.get('files'):
kwargs['files'] = [ascii2unicode(f)
for f in kwargs['files']]
if kwargs.get('properties'):
kwargs['properties'] = dict((ascii2unicode(k), v)
for k, v in iteritems(kwargs['properties']))
# pass the converted call on to the data API
changeid = yield self.data.updates.addChange(**kwargs)
# and turn that changeid into a change object, since that's what
# callers expected (and why this method was deprecated)
chdict = yield self.db.changes.getChange(changeid)
change = yield changes.Change.fromChdict(self, chdict)
defer.returnValue(change)
@defer.inlineCallbacks
def addBuildset(self, scheduler, **kwargs):
log.msg("WARNING: master.addBuildset is deprecated; "
"use the data API update method")
bsid, brids = yield self.data.updates.addBuildset(
scheduler=scheduler, **kwargs)
defer.returnValue((bsid, brids))
# state maintenance (private)
def getObjectId(self):
"""
Return the object id for this master, for associating state with the
master.
@returns: ID, via Deferred
"""
# try to get the cached value
if self._object_id is not None:
return defer.succeed(self._object_id)
# failing that, get it from the DB; multiple calls to this function
# at the same time will not hurt
d = self.db.state.getObjectId(self.name,
"buildbot.master.BuildMaster")
@d.addCallback
def keep(id):
self._object_id = id
return id
return d
def _getState(self, name, default=None):
"private wrapper around C{self.db.state.getState}"
d = self.getObjectId()
@d.addCallback
def get(objectid):
return self.db.state.getState(objectid, name, default)
return d
def _setState(self, name, value):
"private wrapper around C{self.db.state.setState}"
d = self.getObjectId()
@d.addCallback
def set(objectid):
return self.db.state.setState(objectid, name, value)
return d
class TestBotMaster(unittest.TestCase):
def setUp(self):
self.master = fakemaster.make_master()
self.botmaster = BotMaster(self.master)
self.new_config = mock.Mock()
self.botmaster.startService()
def tearDown(self):
return self.botmaster.stopService()
def test_reconfigService(self):
# check that reconfigServiceSlaves and reconfigServiceBuilders are
# both called; they will be tested invidually below
self.patch(self.botmaster, "reconfigServiceBuilders", mock.Mock(side_effect=lambda c: defer.succeed(None)))
self.patch(self.botmaster, "reconfigServiceSlaves", mock.Mock(side_effect=lambda c: defer.succeed(None)))
self.patch(self.botmaster, "maybeStartBuildsForAllBuilders", mock.Mock())
old_config, new_config = mock.Mock(), mock.Mock()
d = self.botmaster.reconfigService(new_config)
@d.addCallback
def check(_):
self.botmaster.reconfigServiceBuilders.assert_called_with(new_config)
self.botmaster.reconfigServiceSlaves.assert_called_with(new_config)
self.assertTrue(self.botmaster.maybeStartBuildsForAllBuilders.called)
return d
@defer.deferredGenerator
def test_reconfigServiceSlaves_add_remove(self):
sl = FakeBuildSlave("sl1")
self.new_config.slaves = [sl]
wfd = defer.waitForDeferred(self.botmaster.reconfigServiceSlaves(self.new_config))
yield wfd
wfd.getResult()
self.assertIdentical(sl.parent, self.botmaster)
self.assertEqual(self.botmaster.slaves, {"sl1": sl})
self.new_config.slaves = []
wfd = defer.waitForDeferred(self.botmaster.reconfigServiceSlaves(self.new_config))
yield wfd
wfd.getResult()
self.assertIdentical(sl.parent, None)
self.assertIdentical(sl.master, None)
@defer.deferredGenerator
def test_reconfigServiceSlaves_reconfig(self):
sl = FakeBuildSlave("sl1")
self.botmaster.slaves = dict(sl1=sl)
sl.setServiceParent(self.botmaster)
sl.master = self.master
sl.botmaster = self.botmaster
sl_new = FakeBuildSlave("sl1")
self.new_config.slaves = [sl_new]
wfd = defer.waitForDeferred(self.botmaster.reconfigServiceSlaves(self.new_config))
yield wfd
wfd.getResult()
# sl was not replaced..
self.assertIdentical(self.botmaster.slaves["sl1"], sl)
@defer.deferredGenerator
def test_reconfigServiceSlaves_class_changes(self):
sl = FakeBuildSlave("sl1")
self.botmaster.slaves = dict(sl1=sl)
sl.setServiceParent(self.botmaster)
sl.master = self.master
sl.botmaster = self.botmaster
sl_new = FakeBuildSlave2("sl1")
self.new_config.slaves = [sl_new]
wfd = defer.waitForDeferred(self.botmaster.reconfigServiceSlaves(self.new_config))
yield wfd
wfd.getResult()
# sl *was* replaced (different class)
self.assertIdentical(self.botmaster.slaves["sl1"], sl_new)
@defer.deferredGenerator
def test_reconfigServiceBuilders_add_remove(self):
bc = config.BuilderConfig(name="bldr", factory=mock.Mock(), slavename="f")
self.new_config.builders = [bc]
wfd = defer.waitForDeferred(self.botmaster.reconfigServiceBuilders(self.new_config))
yield wfd
wfd.getResult()
bldr = self.botmaster.builders["bldr"]
self.assertIdentical(bldr.parent, self.botmaster)
self.assertIdentical(bldr.master, self.master)
self.assertEqual(self.botmaster.builderNames, ["bldr"])
self.new_config.builders = []
wfd = defer.waitForDeferred(self.botmaster.reconfigServiceBuilders(self.new_config))
yield wfd
wfd.getResult()
self.assertIdentical(bldr.parent, None)
self.assertIdentical(bldr.master, None)
self.assertEqual(self.botmaster.builders, {})
self.assertEqual(self.botmaster.builderNames, [])
def test_maybeStartBuildsForBuilder(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.maybeStartBuildsForBuilder("frank")
brd.maybeStartBuildsOn.assert_called_once_with(["frank"])
def test_maybeStartBuildsForSlave(self):
brd = self.botmaster.brd = mock.Mock()
b1 = mock.Mock(name="frank")
b1.name = "frank"
b2 = mock.Mock(name="larry")
b2.name = "larry"
self.botmaster.getBuildersForSlave = mock.Mock(return_value=[b1, b2])
self.botmaster.maybeStartBuildsForSlave("centos")
self.botmaster.getBuildersForSlave.assert_called_once_with("centos")
brd.maybeStartBuildsOn.assert_called_once_with(["frank", "larry"])
def test_maybeStartBuildsForAll(self):
brd = self.botmaster.brd = mock.Mock()
self.botmaster.builderNames = ["frank", "larry"]
self.botmaster.maybeStartBuildsForAllBuilders()
brd.maybeStartBuildsOn.assert_called_once_with(["frank", "larry"])
