class TestCleanShutdown(unittest.TestCase): def setUp(self): self.botmaster = BotMaster(mock.Mock()) self.reactor = mock.Mock() self.botmaster.startService() def assertReactorStopped(self, _=None): self.assertTrue(self.reactor.stop.called) def assertReactorNotStopped(self, _=None): self.assertFalse(self.reactor.stop.called) def makeFakeBuild(self): self.fake_builder = builder = mock.Mock() build = mock.Mock() builder.builder_status.getCurrentBuilds.return_value = [build] self.build_deferred = defer.Deferred() build.waitUntilFinished.return_value = self.build_deferred self.botmaster.builders = mock.Mock() self.botmaster.builders.values.return_value = [builder] def finishFakeBuild(self): self.fake_builder.builder_status.getCurrentBuilds.return_value = [] self.build_deferred.callback(None) # tests def test_shutdown_idle(self): """Test that the master shuts down when it's idle""" self.botmaster.cleanShutdown(_reactor=self.reactor) self.assertReactorStopped() def test_shutdown_busy(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild() self.botmaster.cleanShutdown(_reactor=self.reactor) # check that we haven't stopped yet, since there's a running build self.assertReactorNotStopped() # try to shut it down again, just to check that this does not fail self.botmaster.cleanShutdown(_reactor=self.reactor) # Now we cause the build to finish self.finishFakeBuild() # And now we should be stopped self.assertReactorStopped() def test_shutdown_cancel_not_shutting_down(self): """Test that calling cancelCleanShutdown when none is in progress works""" # this just shouldn't fail.. self.botmaster.cancelCleanShutdown() def test_shutdown_cancel(self): """Test that we can cancel a shutdown""" self.makeFakeBuild() self.botmaster.cleanShutdown(_reactor=self.reactor) # Next we check that we haven't stopped yet, since there's a running # build. self.assertReactorNotStopped() # but the BuildRequestDistributor should not be running self.assertFalse(self.botmaster.brd.running) # Cancel the shutdown self.botmaster.cancelCleanShutdown() # Now we cause the build to finish self.finishFakeBuild() # We should still be running! self.assertReactorNotStopped() # and the BuildRequestDistributor should be, as well self.assertTrue(self.botmaster.brd.running)
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService, WorkerAPICompatMixin): # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered # unclaimed; this should be at least 2 to avoid false positives UNCLAIMED_BUILD_FACTOR = 6 def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None): service.AsyncMultiService.__init__(self) if reactor is None: from twisted.internet import reactor self.reactor = reactor self.setName("buildmaster") self.umask = umask self.basedir = basedir if basedir is not None: # None is used in tests assert os.path.isdir(self.basedir) if config_loader is not None and configFileName is not None: raise config.ConfigErrors([ "Can't specify both `config_loader` and `configFilename`.", ]) elif config_loader is None: if configFileName is None: configFileName = 'master.cfg' config_loader = config.FileLoader(self.basedir, configFileName) self.config_loader = config_loader self.configFileName = configFileName # flag so we don't try to do fancy things before the master is ready self._master_initialized = False self.initLock = defer.DeferredLock() # set up child services self.create_child_services() # db configured values self.configured_db_url = None # configuration / reconfiguration handling self.config = config.MasterConfig() self.reconfig_active = False self.reconfig_requested = False self.reconfig_notifier = None # this stores parameters used in the tac file, and is accessed by the # WebStatus to duplicate those values. self.log_rotation = LogRotation() # local cache for this master's object ID self._object_id = None # Check environment is sensible check_functional_environment(self.config) # figure out local hostname try: self.hostname = os.uname()[1] # only on unix except AttributeError: self.hostname = socket.getfqdn() # public attributes self.name = ("%s:%s" % (self.hostname, os.path.abspath(self.basedir or '.'))) if isinstance(self.name, bytes): self.name = self.name.decode('ascii', 'replace') self.masterid = None def create_child_services(self): # note that these are order-dependent. If you get the order wrong, # you'll know it, as the master will fail to start. self.metrics = metrics.MetricLogObserver() self.metrics.setServiceParent(self) self.caches = cache.CacheManager() self.caches.setServiceParent(self) self.pbmanager = buildbot.pbmanager.PBManager() self.pbmanager.setServiceParent(self) self.workers = workermanager.WorkerManager(self) self.workers.setServiceParent(self) self.change_svc = ChangeManager() self.change_svc.setServiceParent(self) self.botmaster = BotMaster() self.botmaster.setServiceParent(self) self.scheduler_manager = SchedulerManager() self.scheduler_manager.setServiceParent(self) self.user_manager = UserManagerManager(self) self.user_manager.setServiceParent(self) self.db = dbconnector.DBConnector(self.basedir) self.db.setServiceParent(self) self.wamp = wampconnector.WampConnector() self.wamp.setServiceParent(self) self.mq = mqconnector.MQConnector() self.mq.setServiceParent(self) self.data = dataconnector.DataConnector() self.data.setServiceParent(self) self.www = wwwservice.WWWService() self.www.setServiceParent(self) self.debug = debug.DebugServices() self.debug.setServiceParent(self) self.status = Status() self.status.setServiceParent(self) self.secrets_manager = SecretManager() self.secrets_manager.setServiceParent(self) self.secrets_manager.reconfig_priority = 2000 self.service_manager = service.BuildbotServiceManager() self.service_manager.setServiceParent(self) self.service_manager.reconfig_priority = 1000 self.masterHouskeepingTimer = 0 @defer.inlineCallbacks def heartbeat(): if self.masterid is not None: yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) yield self.data.updates.expireMasters() self.masterHeartbeatService = internet.TimerService(60, heartbeat) # we do setServiceParent only when the master is configured # master should advertise itself only at that time # setup and reconfig handling _already_started = False @defer.inlineCallbacks def startService(self): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() self.reactor.callWhenRunning(d.callback, None) yield d startup_succeed = False try: yield self.initLock.acquire() # load the configuration file, treating errors as fatal try: # run the master.cfg in thread, so that it can use blocking # code self.config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) except config.ConfigErrors as e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") self.reactor.stop() return except Exception: log.err(failure.Failure(), 'while starting BuildMaster') self.reactor.stop() return # set up services that need access to the config before everything # else gets told to reconfig try: yield self.db.setup() except exceptions.DatabaseNotReadyError: # (message was already logged) self.reactor.stop() return self.mq.setup() if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) if hasattr(signal, "SIGUSR1"): def sigusr1(*args): eventually(self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # get the masterid so other services can use it in # startup/reconfig. This goes directly to the DB since the data # API isn't initialized yet, and anyway, this method is aware of # the DB API since it just called its setup function self.masterid = yield self.db.masters.findMasterId( name=self.name) # mark this master as stopped, in case it crashed before yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) # call the parent method yield service.AsyncMultiService.startService(self) # We make sure the housekeeping is done before configuring in order to cleanup # any remaining claimed schedulers or change sources from zombie # masters yield self.data.updates.expireMasters(forceHouseKeeping=True) # give all services a chance to load the new configuration, rather # than the base configuration yield self.reconfigServiceWithBuildbotConfig(self.config) # Mark the master as active now that mq is running yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) # Start the heartbeat timer yield self.masterHeartbeatService.setServiceParent(self) # send the statistics to buildbot.net, without waiting self.sendBuildbotNetUsageData() startup_succeed = True except Exception: f = failure.Failure() log.err(f, 'while starting BuildMaster') self.reactor.stop() finally: if startup_succeed: log.msg("BuildMaster is running") else: log.msg("BuildMaster startup failed") yield self.initLock.release() self._master_initialized = True def sendBuildbotNetUsageData(self): if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None: raise RuntimeError( "Should not enable buildbotNetUsageData in trial tests!") sendBuildbotNetUsageData(self) @defer.inlineCallbacks def stopService(self): try: yield self.initLock.acquire() if self.masterid is not None: yield self.data.updates.masterStopped( name=self.name, masterid=self.masterid) if self.running: yield self.botmaster.cleanShutdown( quickMode=True, stopReactor=False) yield service.AsyncMultiService.stopService(self) log.msg("BuildMaster is stopped") self._master_initialized = False finally: yield self.initLock.release() def reconfig(self): # this method wraps doConfig, ensuring it is only ever called once at # a time, and alerting the user if the reconfig takes too long if self.reconfig_active: log.msg("reconfig already active; will reconfig again after") self.reconfig_requested = True return self.reconfig_active = self.reactor.seconds() metrics.MetricCountEvent.log("loaded_config", 1) # notify every 10 seconds that the reconfig is still going on, although # reconfigs should not take that long! self.reconfig_notifier = task.LoopingCall(lambda: log.msg("reconfig is ongoing for %d s" % (self.reactor.seconds() - self.reconfig_active))) self.reconfig_notifier.start(10, now=False) timer = metrics.Timer("BuildMaster.reconfig") timer.start() d = self.doReconfig() @d.addBoth def cleanup(res): timer.stop() self.reconfig_notifier.stop() self.reconfig_notifier = None self.reconfig_active = False if self.reconfig_requested: self.reconfig_requested = False self.reconfig() return res d.addErrback(log.err, 'while reconfiguring') return d # for tests @defer.inlineCallbacks def doReconfig(self): log.msg("beginning configuration update") changes_made = False failed = False try: yield self.initLock.acquire() # Run the master.cfg in thread, so that it can use blocking code new_config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) changes_made = True self.config = new_config yield self.reconfigServiceWithBuildbotConfig(new_config) except config.ConfigErrors as e: for msg in e.errors: log.msg(msg) failed = True except Exception: log.err(failure.Failure(), 'during reconfig:') failed = True finally: yield self.initLock.release() if failed: if changes_made: log.msg("WARNING: reconfig partially applied; master " "may malfunction") else: log.msg("reconfig aborted without making any changes") else: log.msg("configuration update complete") def reconfigServiceWithBuildbotConfig(self, new_config): if self.configured_db_url is None: self.configured_db_url = new_config.db['db_url'] elif (self.configured_db_url != new_config.db['db_url']): config.error( "Cannot change c['db']['db_url'] after the master has started", ) if self.config.mq['type'] != new_config.mq['type']: raise config.ConfigErrors([ "Cannot change c['mq']['type'] after the master has started", ]) return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig(self, new_config) # informational methods def allSchedulers(self): return list(self.scheduler_manager) def getStatus(self): """ @rtype: L{buildbot.status.builder.Status} """ return self.status # state maintenance (private) def getObjectId(self): """ Return the object id for this master, for associating state with the master. @returns: ID, via Deferred """ # try to get the cached value if self._object_id is not None: return defer.succeed(self._object_id) # failing that, get it from the DB; multiple calls to this function # at the same time will not hurt d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster") @d.addCallback def keep(id): self._object_id = id return id return d def _getState(self, name, default=None): "private wrapper around C{self.db.state.getState}" d = self.getObjectId() @d.addCallback def get(objectid): return self.db.state.getState(objectid, name, default) return d def _setState(self, name, value): "private wrapper around C{self.db.state.setState}" d = self.getObjectId() @d.addCallback def set(objectid): return self.db.state.setState(objectid, name, value) return d
class TestCleanShutdown(TestReactorMixin, unittest.TestCase): @defer.inlineCallbacks def setUp(self): self.setUpTestReactor() self.master = fakemaster.make_master(self, wantData=True) self.botmaster = BotMaster() yield self.botmaster.setServiceParent(self.master) self.botmaster.startService() def assertReactorStopped(self, _=None): self.assertTrue(self.reactor.stop_called) def assertReactorNotStopped(self, _=None): self.assertFalse(self.reactor.stop_called) def makeFakeBuild(self, waitedFor=False): self.fake_builder = builder = mock.Mock() build_status = mock.Mock() builder.builder_status.getCurrentBuilds.return_value = [build_status] self.build_deferred = defer.Deferred() request = mock.Mock() request.waitedFor = waitedFor build = mock.Mock() build.stopBuild = self.stopFakeBuild build.waitUntilFinished.return_value = self.build_deferred build.requests = [request] builder.building = [build] self.botmaster.builders = mock.Mock() self.botmaster.builders.values.return_value = [builder] def stopFakeBuild(self, reason, results): self.reason = reason self.results = results self.finishFakeBuild() def finishFakeBuild(self): self.fake_builder.building = [] self.build_deferred.callback(None) # tests def test_shutdown_idle(self): """Test that the master shuts down when it's idle""" self.botmaster.cleanShutdown() self.assertReactorStopped() def test_shutdown_busy(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild() self.botmaster.cleanShutdown() # check that we haven't stopped yet, since there's a running build self.assertReactorNotStopped() # try to shut it down again, just to check that this does not fail self.botmaster.cleanShutdown() # Now we cause the build to finish self.finishFakeBuild() # And now we should be stopped self.assertReactorStopped() def test_shutdown_busy_quick(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild() self.botmaster.cleanShutdown(quickMode=True) # And now we should be stopped self.assertReactorStopped() self.assertEqual(self.results, RETRY) def test_shutdown_busy_quick_cancelled(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild(waitedFor=True) self.botmaster.cleanShutdown(quickMode=True) # And now we should be stopped self.assertReactorStopped() self.assertEqual(self.results, CANCELLED) def test_shutdown_cancel_not_shutting_down(self): """Test that calling cancelCleanShutdown when none is in progress works""" # this just shouldn't fail.. self.botmaster.cancelCleanShutdown() def test_shutdown_cancel(self): """Test that we can cancel a shutdown""" self.makeFakeBuild() self.botmaster.cleanShutdown() # Next we check that we haven't stopped yet, since there's a running # build. self.assertReactorNotStopped() # but the BuildRequestDistributor should not be running self.assertFalse(self.botmaster.brd.running) # Cancel the shutdown self.botmaster.cancelCleanShutdown() # Now we cause the build to finish self.finishFakeBuild() # We should still be running! self.assertReactorNotStopped() # and the BuildRequestDistributor should be, as well self.assertTrue(self.botmaster.brd.running)
class BuildMaster(service.ReconfigurableServiceMixin, service.MasterService, WorkerAPICompatMixin): # multiplier on RECLAIM_BUILD_INTERVAL at which a build is considered # unclaimed; this should be at least 2 to avoid false positives UNCLAIMED_BUILD_FACTOR = 6 def __init__(self, basedir, configFileName=None, umask=None, reactor=None, config_loader=None): service.AsyncMultiService.__init__(self) if reactor is None: from twisted.internet import reactor self.reactor = reactor self.setName("buildmaster") self.umask = umask self.basedir = basedir if basedir is not None: # None is used in tests assert os.path.isdir(self.basedir) if config_loader is not None and configFileName is not None: raise config.ConfigErrors([ "Can't specify both `config_loader` and `configFilename`.", ]) elif config_loader is None: if configFileName is None: configFileName = 'master.cfg' config_loader = config.FileLoader(self.basedir, configFileName) self.config_loader = config_loader self.configFileName = configFileName # flag so we don't try to do fancy things before the master is ready self._master_initialized = False self.initLock = defer.DeferredLock() # set up child services self.create_child_services() # db configured values self.configured_db_url = None # configuration / reconfiguration handling self.config = config.MasterConfig() self.reconfig_active = False self.reconfig_requested = False self.reconfig_notifier = None # this stores parameters used in the tac file, and is accessed by the # WebStatus to duplicate those values. self.log_rotation = LogRotation() # local cache for this master's object ID self._object_id = None # Check environment is sensible check_functional_environment(self.config) # figure out local hostname try: self.hostname = os.uname()[1] # only on unix except AttributeError: self.hostname = socket.getfqdn() # public attributes self.name = ("%s:%s" % (self.hostname, os.path.abspath(self.basedir or '.'))) if isinstance(self.name, bytes): self.name = self.name.decode('ascii', 'replace') self.masterid = None def create_child_services(self): # note that these are order-dependent. If you get the order wrong, # you'll know it, as the master will fail to start. self.metrics = metrics.MetricLogObserver() self.metrics.setServiceParent(self) self.caches = cache.CacheManager() self.caches.setServiceParent(self) self.pbmanager = buildbot.pbmanager.PBManager() self.pbmanager.setServiceParent(self) self.workers = workermanager.WorkerManager(self) self.workers.setServiceParent(self) self.change_svc = ChangeManager() self.change_svc.setServiceParent(self) self.botmaster = BotMaster() self.botmaster.setServiceParent(self) self.scheduler_manager = SchedulerManager() self.scheduler_manager.setServiceParent(self) self.user_manager = UserManagerManager(self) self.user_manager.setServiceParent(self) self.db = dbconnector.DBConnector(self.basedir) self.db.setServiceParent(self) self.wamp = wampconnector.WampConnector() self.wamp.setServiceParent(self) self.mq = mqconnector.MQConnector() self.mq.setServiceParent(self) self.data = dataconnector.DataConnector() self.data.setServiceParent(self) self.www = wwwservice.WWWService() self.www.setServiceParent(self) self.debug = debug.DebugServices() self.debug.setServiceParent(self) self.status = Status() self.status.setServiceParent(self) self.secrets_manager = SecretManager() self.secrets_manager.setServiceParent(self) self.secrets_manager.reconfig_priority = 2000 self.service_manager = service.BuildbotServiceManager() self.service_manager.setServiceParent(self) self.service_manager.reconfig_priority = 1000 self.masterHouskeepingTimer = 0 @defer.inlineCallbacks def heartbeat(): if self.masterid is not None: yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) yield self.data.updates.expireMasters() self.masterHeartbeatService = internet.TimerService(60, heartbeat) self.masterHeartbeatService.clock = self.reactor # we do setServiceParent only when the master is configured # master should advertise itself only at that time # setup and reconfig handling _already_started = False @defer.inlineCallbacks def startService(self): assert not self._already_started, "can only start the master once" self._already_started = True log.msg("Starting BuildMaster -- buildbot.version: %s" % buildbot.version) # Set umask if self.umask is not None: os.umask(self.umask) # first, apply all monkeypatches monkeypatches.patch_all() # we want to wait until the reactor is running, so we can call # reactor.stop() for fatal errors d = defer.Deferred() self.reactor.callWhenRunning(d.callback, None) yield d startup_succeed = False try: yield self.initLock.acquire() # load the configuration file, treating errors as fatal try: # run the master.cfg in thread, so that it can use blocking # code self.config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) except config.ConfigErrors as e: log.msg("Configuration Errors:") for msg in e.errors: log.msg(" " + msg) log.msg("Halting master.") self.reactor.stop() return except Exception: log.err(failure.Failure(), 'while starting BuildMaster') self.reactor.stop() return # set up services that need access to the config before everything # else gets told to reconfig try: yield self.db.setup() except exceptions.DatabaseNotReadyError: # (message was already logged) self.reactor.stop() return self.mq.setup() if hasattr(signal, "SIGHUP"): def sighup(*args): eventually(self.reconfig) signal.signal(signal.SIGHUP, sighup) if hasattr(signal, "SIGUSR1"): def sigusr1(*args): eventually(self.botmaster.cleanShutdown) signal.signal(signal.SIGUSR1, sigusr1) # get the masterid so other services can use it in # startup/reconfig. This goes directly to the DB since the data # API isn't initialized yet, and anyway, this method is aware of # the DB API since it just called its setup function self.masterid = yield self.db.masters.findMasterId(name=self.name) # mark this master as stopped, in case it crashed before yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) # call the parent method yield service.AsyncMultiService.startService(self) # We make sure the housekeeping is done before configuring in order to cleanup # any remaining claimed schedulers or change sources from zombie # masters yield self.data.updates.expireMasters(forceHouseKeeping=True) # give all services a chance to load the new configuration, rather # than the base configuration yield self.reconfigServiceWithBuildbotConfig(self.config) # Mark the master as active now that mq is running yield self.data.updates.masterActive(name=self.name, masterid=self.masterid) # Start the heartbeat timer yield self.masterHeartbeatService.setServiceParent(self) # send the statistics to buildbot.net, without waiting self.sendBuildbotNetUsageData() startup_succeed = True except Exception: f = failure.Failure() log.err(f, 'while starting BuildMaster') self.reactor.stop() finally: if startup_succeed: log.msg("BuildMaster is running") else: log.msg("BuildMaster startup failed") yield self.initLock.release() self._master_initialized = True def sendBuildbotNetUsageData(self): if "TRIAL_PYTHONPATH" in os.environ and self.config.buildbotNetUsageData is not None: raise RuntimeError( "Should not enable buildbotNetUsageData in trial tests!") sendBuildbotNetUsageData(self) @defer.inlineCallbacks def stopService(self): try: yield self.initLock.acquire() if self.masterid is not None: yield self.data.updates.masterStopped(name=self.name, masterid=self.masterid) if self.running: yield self.botmaster.cleanShutdown(quickMode=True, stopReactor=False) yield service.AsyncMultiService.stopService(self) log.msg("BuildMaster is stopped") self._master_initialized = False finally: yield self.initLock.release() def reconfig(self): # this method wraps doConfig, ensuring it is only ever called once at # a time, and alerting the user if the reconfig takes too long if self.reconfig_active: log.msg("reconfig already active; will reconfig again after") self.reconfig_requested = True return self.reconfig_active = self.reactor.seconds() metrics.MetricCountEvent.log("loaded_config", 1) # notify every 10 seconds that the reconfig is still going on, although # reconfigs should not take that long! self.reconfig_notifier = task.LoopingCall( lambda: log.msg("reconfig is ongoing for %d s" % (self.reactor.seconds() - self.reconfig_active))) self.reconfig_notifier.start(10, now=False) timer = metrics.Timer("BuildMaster.reconfig") timer.start() d = self.doReconfig() @d.addBoth def cleanup(res): timer.stop() self.reconfig_notifier.stop() self.reconfig_notifier = None self.reconfig_active = False if self.reconfig_requested: self.reconfig_requested = False self.reconfig() return res d.addErrback(log.err, 'while reconfiguring') return d # for tests @defer.inlineCallbacks def doReconfig(self): log.msg("beginning configuration update") changes_made = False failed = False try: yield self.initLock.acquire() # Run the master.cfg in thread, so that it can use blocking code new_config = yield threads.deferToThreadPool( self.reactor, self.reactor.getThreadPool(), self.config_loader.loadConfig) changes_made = True self.config = new_config yield self.reconfigServiceWithBuildbotConfig(new_config) except config.ConfigErrors as e: for msg in e.errors: log.msg(msg) failed = True except Exception: log.err(failure.Failure(), 'during reconfig:') failed = True finally: yield self.initLock.release() if failed: if changes_made: log.msg("WARNING: reconfig partially applied; master " "may malfunction") else: log.msg("reconfig aborted without making any changes") else: log.msg("configuration update complete") def reconfigServiceWithBuildbotConfig(self, new_config): if self.configured_db_url is None: self.configured_db_url = new_config.db['db_url'] elif (self.configured_db_url != new_config.db['db_url']): config.error( "Cannot change c['db']['db_url'] after the master has started", ) if self.config.mq['type'] != new_config.mq['type']: raise config.ConfigErrors([ "Cannot change c['mq']['type'] after the master has started", ]) return service.ReconfigurableServiceMixin.reconfigServiceWithBuildbotConfig( self, new_config) # informational methods def allSchedulers(self): return list(self.scheduler_manager) def getStatus(self): """ @rtype: L{buildbot.status.builder.Status} """ return self.status # state maintenance (private) def getObjectId(self): """ Return the object id for this master, for associating state with the master. @returns: ID, via Deferred """ # try to get the cached value if self._object_id is not None: return defer.succeed(self._object_id) # failing that, get it from the DB; multiple calls to this function # at the same time will not hurt d = self.db.state.getObjectId(self.name, "buildbot.master.BuildMaster") @d.addCallback def keep(id): self._object_id = id return id return d def _getState(self, name, default=None): "private wrapper around C{self.db.state.getState}" d = self.getObjectId() @d.addCallback def get(objectid): return self.db.state.getState(objectid, name, default) return d def _setState(self, name, value): "private wrapper around C{self.db.state.setState}" d = self.getObjectId() @d.addCallback def set(objectid): return self.db.state.setState(objectid, name, value) return d
class TestCleanShutdown(unittest.TestCase): def setUp(self): self.master = BotMaster(Mock()) self.master.reactor = Mock() self.master.startService() def test_shutdown_idle(self): """Test that the master shuts down when it's idle""" d = self.master.cleanShutdown() def _check(ign): self.assertEquals(self.master.reactor.stop.called, True) d.addCallback(_check) return d def test_shutdown_busy(self): """Test that the master shuts down after builds finish""" # Fake some builds builder = Mock() build = Mock() builder.builder_status.getCurrentBuilds.return_value = [build] d_finished = defer.Deferred() build.waitUntilFinished.return_value = d_finished self.master.builders = Mock() self.master.builders.values.return_value = [builder] d_shutdown = self.master.cleanShutdown() # Trigger the loop to get things going self.master.loop.trigger() # First we wait for it to quiet down again d = self.master.loop.when_quiet() # Next we check that we haven't stopped yet, since there's a running # build def _check1(ign): self.assertEquals(self.master.reactor.stop.called, False) d.addCallback(_check1) # Now we cause the build to finish, then kick the loop again, # empty out the list of running builds, and wait for the shutdown # process to finish def _finish_build(ign): d_finished.callback(None) self.master.loop.trigger() self.master.builders.values.return_value = [] return d_shutdown d.addCallback(_finish_build) # And now we should be done def _check2(ign): self.assertEquals(self.master.reactor.stop.called, True) d.addCallback(_check2) return d def test_shutdown_cancel(self): """Test that we can cancel a shutdown""" # Fake some builds builder = Mock() build = Mock() builder.builder_status.getCurrentBuilds.return_value = [build] d_finished = defer.Deferred() build.waitUntilFinished.return_value = d_finished self.master.builders = Mock() self.master.builders.values.return_value = [builder] d_shutdown = self.master.cleanShutdown() # Trigger the loop to get things going self.master.loop.trigger() # First we wait for it to quiet down again d = self.master.loop.when_quiet() # Next we check that we haven't stopped yet, since there's a running # build. # We cancel the shutdown here too def _check1(ign): self.assertEquals(self.master.reactor.stop.called, False) self.master.cancelCleanShutdown() d.addCallback(_check1) # Now we cause the build to finish, then kick the loop again, # empty out the list of running builds, and wait for the shutdown # process to finish def _finish_build(ign): d_finished.callback(None) self.master.loop.trigger() self.master.builders.values.return_value = [] return d_shutdown d.addCallback(_finish_build) # We should still be running! def _check2(ign): self.assertEquals(self.master.reactor.stop.called, False) d.addCallback(_check2) return d def test_shutdown_no_new_builds(self): """Test that no new builds get handed out when we're shutting down""" # Fake some builds builder = Mock() build = Mock() builder.builder_status.getCurrentBuilds.return_value = [build] d_finished = defer.Deferred() build.waitUntilFinished.return_value = d_finished self.master.builders = Mock() self.master.builders.values.return_value = [builder] self.assertEquals(self.master._get_processors(), [builder.run]) d_shutdown = self.master.cleanShutdown() assert d_shutdown # Trigger the loop to get things going self.master.loop.trigger() # First we wait for it to quiet down again d = self.master.loop.when_quiet() # Next we check that we haven't stopped yet, since there's a running # build. # Also check that we're not trying to hand out new builds! def _check1(ign): self.assertEquals(self.master.reactor.stop.called, False) self.assertEquals(self.master._get_processors(), []) d.addCallback(_check1) return d
class TestCleanShutdown(TestReactorMixin, unittest.TestCase): def setUp(self): self.setUpTestReactor() self.master = fakemaster.make_master(self, wantData=True) self.botmaster = BotMaster() self.botmaster.setServiceParent(self.master) self.botmaster.startService() def assertReactorStopped(self, _=None): self.assertTrue(self.reactor.stop_called) def assertReactorNotStopped(self, _=None): self.assertFalse(self.reactor.stop_called) def makeFakeBuild(self, waitedFor=False): self.fake_builder = builder = mock.Mock() build_status = mock.Mock() builder.builder_status.getCurrentBuilds.return_value = [build_status] self.build_deferred = defer.Deferred() request = mock.Mock() request.waitedFor = waitedFor build = mock.Mock() build.stopBuild = self.stopFakeBuild build.waitUntilFinished.return_value = self.build_deferred build.requests = [request] builder.building = [build] self.botmaster.builders = mock.Mock() self.botmaster.builders.values.return_value = [builder] def stopFakeBuild(self, reason, results): self.reason = reason self.results = results self.finishFakeBuild() def finishFakeBuild(self): self.fake_builder.building = [] self.build_deferred.callback(None) # tests def test_shutdown_idle(self): """Test that the master shuts down when it's idle""" self.botmaster.cleanShutdown(_reactor=self.reactor) self.assertReactorStopped() def test_shutdown_busy(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild() self.botmaster.cleanShutdown(_reactor=self.reactor) # check that we haven't stopped yet, since there's a running build self.assertReactorNotStopped() # try to shut it down again, just to check that this does not fail self.botmaster.cleanShutdown(_reactor=self.reactor) # Now we cause the build to finish self.finishFakeBuild() # And now we should be stopped self.assertReactorStopped() def test_shutdown_busy_quick(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild() self.botmaster.cleanShutdown(quickMode=True, _reactor=self.reactor) # And now we should be stopped self.assertReactorStopped() self.assertEqual(self.results, RETRY) def test_shutdown_busy_quick_cancelled(self): """Test that the master shuts down after builds finish""" self.makeFakeBuild(waitedFor=True) self.botmaster.cleanShutdown(quickMode=True, _reactor=self.reactor) # And now we should be stopped self.assertReactorStopped() self.assertEqual(self.results, CANCELLED) def test_shutdown_cancel_not_shutting_down(self): """Test that calling cancelCleanShutdown when none is in progress works""" # this just shouldn't fail.. self.botmaster.cancelCleanShutdown() def test_shutdown_cancel(self): """Test that we can cancel a shutdown""" self.makeFakeBuild() self.botmaster.cleanShutdown(_reactor=self.reactor) # Next we check that we haven't stopped yet, since there's a running # build. self.assertReactorNotStopped() # but the BuildRequestDistributor should not be running self.assertFalse(self.botmaster.brd.running) # Cancel the shutdown self.botmaster.cancelCleanShutdown() # Now we cause the build to finish self.finishFakeBuild() # We should still be running! self.assertReactorNotStopped() # and the BuildRequestDistributor should be, as well self.assertTrue(self.botmaster.brd.running)