def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=10 * 60, # Ten minutes properties=None, locks=None, keepalive_interval=3600): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary """ self.name = name = ascii2unicode(name) if properties is None: properties = {} self.password = password # protocol registration self.registration = None # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error( 'notify_on_missing arg %r is not a string' % (i,)) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None self._old_builder_list = None
class AbstractWorker(service.BuildbotService, object): """This is the master-side representative for a remote buildbot worker. There is exactly one for each worker described in the config file (the c['workers'] list). When buildbots connect in (.attach), they get a reference to this instance. The BotMaster object is stashed as the .botmaster attribute. The BotMaster is also our '.parent' Service. I represent a worker -- a remote machine capable of running builds. I am instantiated by the configuration file, and can be subclassed to add extra functionality.""" implements(IWorker) # reconfig workers after builders reconfig_priority = 64 def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=10 * 60, # Ten minutes properties=None, locks=None, keepalive_interval=3600): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary """ self.name = name = ascii2unicode(name) if properties is None: properties = {} self.password = password # protocol registration self.registration = None # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error( 'notify_on_missing arg %r is not a string' % (i,)) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None self._old_builder_list = None def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.name) @property def workername(self): # workername is now an alias to twisted.Service's name return self.name deprecatedWorkerClassProperty(locals(), workername) @property def botmaster(self): if self.master is None: return None return self.master.botmaster def updateLocks(self): """Convert the L{LockAccess} objects in C{self.locks} into real lock objects, while also maintaining the subscriptions to lock releases.""" # unsubscribe from any old locks for s in self.lock_subscriptions: s.unsubscribe() # convert locks into their real form locks = [(self.botmaster.getLockFromLockAccess(a), a) for a in self.access] self.locks = [(l.getLock(self), la) for l, la in locks] self.lock_subscriptions = [l.subscribeToReleases(self._lockReleased) for l, la in self.locks] def locksAvailable(self): """ I am called to see if all the locks I depend on are available, in which I return True, otherwise I return False """ if not self.locks: return True for lock, access in self.locks: if not lock.isAvailable(self, access): return False return True def acquireLocks(self): """ I am called when a build is preparing to run. I try to claim all the locks that are needed for a build to happen. If I can't, then my caller should give up the build and try to get another worker to look at it. """ log.msg("acquireLocks(worker %s, locks %s)" % (self, self.locks)) if not self.locksAvailable(): log.msg("worker %s can't lock, giving up" % (self, )) return False # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) return True def releaseLocks(self): """ I am called to release any locks after a build has finished """ log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: lock.release(self, access) def _lockReleased(self): """One of the locks for this worker was released; try scheduling builds.""" if not self.botmaster: return # oh well.. self.botmaster.maybeStartBuildsForWorker(self.name) def _applyWorkerInfo(self, info): if not info: return self.worker_status.setAdmin(info.get("admin")) self.worker_status.setHost(info.get("host")) self.worker_status.setAccessURI(info.get("access_uri", None)) self.worker_status.setVersion(info.get("version", "(unknown)")) @defer.inlineCallbacks def _getWorkerInfo(self): worker = yield self.master.data.get( ('workers', self.workerid)) self._applyWorkerInfo(worker['workerinfo']) def setServiceParent(self, parent): # botmaster needs to set before setServiceParent which calls # startService self.manager = parent return service.BuildbotService.setServiceParent(self, parent) @defer.inlineCallbacks def startService(self): self.updateLocks() self.startMissingTimer() self.workerid = yield self.master.data.updates.findWorkerId( self.name) yield self._getWorkerInfo() yield service.BuildbotService.startService(self) @defer.inlineCallbacks def reconfigService(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=3600, properties=None, locks=None, keepalive_interval=3600): # Given a Worker config arguments, configure this one identically. # Because Worker objects are remotely referenced, we can't replace them # without disconnecting the worker, yet there's no reason to do that. assert self.name == name self.password = password # adopt new instance's configuration parameters self.max_builds = max_builds self.access = [] if locks: self.access = locks self.notify_on_missing = notify_on_missing if self.missing_timeout != missing_timeout: running_missing_timer = self.missing_timer self.stopMissingTimer() self.missing_timeout = missing_timeout if running_missing_timer: self.startMissingTimer() if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") # update our records with the worker manager if not self.registration: self.registration = yield self.master.workers.register(self) yield self.registration.update(self, self.master.config) self.updateLocks() bids = [ b._builderid for b in self.botmaster.getBuildersForWorker(self.name)] yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, bids) # update the attached worker's notion of which builders are attached. # This assumes that the relevant builders have already been configured, # which is why the reconfig_priority is set low in this class. yield self.updateWorker() @defer.inlineCallbacks def stopService(self): if self.registration: yield self.registration.unregister() self.registration = None self.stopMissingTimer() # mark this worker as configured for zero builders in this master yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, []) yield service.BuildbotService.stopService(self) def startMissingTimer(self): if self.notify_on_missing and self.missing_timeout and self.parent: self.stopMissingTimer() # in case it's already running self.missing_timer = self.master.reactor.callLater(self.missing_timeout, self._missing_timer_fired) def stopMissingTimer(self): if self.missing_timer: self.missing_timer.cancel() self.missing_timer = None def isConnected(self): return self.conn def _missing_timer_fired(self): self.missing_timer = None # notify people, but only if we're still in the config if not self.parent: return buildmaster = self.botmaster.master status = buildmaster.getStatus() text = "The Buildbot working for '%s'\n" % status.getTitle() text += ("has noticed that the worker named %s went away\n" % self.name) text += "\n" text += ("It last disconnected at %s (buildmaster-local time)\n" % time.ctime(time.time() - self.missing_timeout)) # approx text += "\n" text += "The admin on record (as reported by WORKER:info/admin)\n" text += "was '%s'.\n" % self.worker_status.getAdmin() text += "\n" text += "Sincerely,\n" text += " The Buildbot\n" text += " %s\n" % status.getTitleURL() text += "\n" text += "%s\n" % status.getURLForThing(self.worker_status) subject = "Buildbot: worker %s was lost" % (self.name,) return self._mail_missing_message(subject, text) def updateWorker(self): """Called to add or remove builders after the worker has connected. @return: a Deferred that indicates when an attached worker has accepted the new builders and/or released the old ones.""" if self.conn: return self.sendBuilderList() else: return defer.succeed(None) @defer.inlineCallbacks def attached(self, conn): """This is called when the worker connects.""" metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1) # now we go through a sequence of calls, gathering information, then # tell the Botmaster that it can finally give this worker to all the # Builders that care about it. # Reset graceful shutdown status self.worker_status.setGraceful(False) # We want to know when the graceful shutdown flag changes self.worker_status.addGracefulWatcher(self._gracefulChanged) self.conn = conn self._old_builder_list = None # clear builder list before proceed self.worker_status.addPauseWatcher(self._pauseChanged) self.worker_status.setConnected(True) self._applyWorkerInfo(conn.info) self.worker_commands = conn.info.get("slave_commands", {}) self.worker_environ = conn.info.get("environ", {}) self.worker_basedir = conn.info.get("basedir", None) self.worker_system = conn.info.get("system", None) self.conn.notifyOnDisconnect(self.detached) workerinfo = { 'admin': conn.info.get('admin'), 'host': conn.info.get('host'), 'access_uri': conn.info.get('access_uri'), 'version': conn.info.get('version') } yield self.master.data.updates.workerConnected( workerid=self.workerid, masterid=self.master.masterid, workerinfo=workerinfo ) if self.worker_system == "nt": self.path_module = namedModule("ntpath") else: # most everything accepts / as separator, so posix should be a # reasonable fallback self.path_module = namedModule("posixpath") log.msg("bot attached") self.messageReceivedFromWorker() self.stopMissingTimer() self.master.status.workerConnected(self.name) yield self.updateWorker() yield self.botmaster.maybeStartBuildsForWorker(self.name) def messageReceivedFromWorker(self): now = time.time() self.lastMessageReceived = now self.worker_status.setLastMessageReceived(now) @defer.inlineCallbacks def detached(self): metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1) self.conn = None self._old_builder_list = [] self.worker_status.removeGracefulWatcher(self._gracefulChanged) self.worker_status.removePauseWatcher(self._pauseChanged) self.worker_status.setConnected(False) log.msg("Worker.detached(%s)" % (self.name,)) self.master.status.workerDisconnected(self.name) self.releaseLocks() yield self.master.data.updates.workerDisconnected( workerid=self.workerid, masterid=self.master.masterid, ) def disconnect(self): """Forcibly disconnect the worker. This severs the TCP connection and returns a Deferred that will fire (with None) when the connection is probably gone. If the worker is still alive, they will probably try to reconnect again in a moment. This is called in two circumstances. The first is when a worker is removed from the config file. In this case, when they try to reconnect, they will be rejected as an unknown worker. The second is when we wind up with two connections for the same worker, in which case we disconnect the older connection. """ if self.conn is None: return defer.succeed(None) log.msg("disconnecting old worker %s now" % (self.name,)) # When this Deferred fires, we'll be ready to accept the new worker return self._disconnect(self.conn) def _disconnect(self, conn): # all kinds of teardown will happen as a result of # loseConnection(), but it happens after a reactor iteration or # two. Hook the actual disconnect so we can know when it is safe # to connect the new worker. We have to wait one additional # iteration (with callLater(0)) to make sure the *other* # notifyOnDisconnect handlers have had a chance to run. d = defer.Deferred() # notifyOnDisconnect runs the callback def _disconnected(): eventually(d.callback, None) conn.notifyOnDisconnect(_disconnected) conn.loseConnection() log.msg("waiting for worker to finish disconnecting") return d def sendBuilderList(self): our_builders = self.botmaster.getBuildersForWorker(self.name) blist = [(b.name, b.config.workerbuilddir) for b in our_builders] if blist == self._old_builder_list: return defer.succeed(None) d = self.conn.remoteSetBuilderList(builders=blist) @d.addCallback def sentBuilderList(ign): self._old_builder_list = blist return ign return d def shutdownRequested(self): log.msg("worker %s wants to shut down" % (self.name,)) self.worker_status.setGraceful(True) def addWorkerForBuilder(self, wfb): self.workerforbuilders[wfb.builder_name] = wfb def removeWorkerForBuilder(self, wfb): try: del self.workerforbuilders[wfb.builder_name] except KeyError: pass def buildFinished(self, wfb): """This is called when a build on this worker is finished.""" self.botmaster.maybeStartBuildsForWorker(self.name) def canStartBuild(self): """ I am called when a build is requested to see if this worker can start a build. This function can be used to limit overall concurrency on the worker. Note for subclassers: if a worker can become willing to start a build without any action on that worker (for example, by a resource in use on another worker becoming available), then you must arrange for L{maybeStartBuildsForWorker} to be called at that time, or builds on this worker will not start. """ if self.worker_status.isPaused(): return False # If we're waiting to shutdown gracefully, then we shouldn't # accept any new jobs. if self.worker_status.getGraceful(): return False if self.max_builds: active_builders = [sb for sb in itervalues(self.workerforbuilders) if sb.isBusy()] if len(active_builders) >= self.max_builds: return False if not self.locksAvailable(): return False return True def _mail_missing_message(self, subject, text): # FIXME: This should be handled properly via the event api # we should send a missing message on the mq, and let any reporter # handle that # first, see if we have a MailNotifier we can use. This gives us a # fromaddr and a relayhost. buildmaster = self.botmaster.master for st in buildmaster.services: if isinstance(st, MailNotifier): break else: # if not, they get a default MailNotifier, which always uses SMTP # to localhost and uses a dummy fromaddr of "buildbot". log.msg("worker-missing msg using default MailNotifier") st = MailNotifier("buildbot") # now construct the mail m = Message() m.set_payload(text) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = st.fromaddr recipients = self.notify_on_missing m['To'] = ", ".join(recipients) d = st.sendMessage(m, recipients) # return the Deferred for testing purposes return d def _gracefulChanged(self, graceful): """This is called when our graceful shutdown setting changes""" self.maybeShutdown() @defer.inlineCallbacks def shutdown(self): """Shutdown the worker""" if not self.conn: log.msg("no remote; worker is already shut down") return yield self.conn.remoteShutdown() def maybeShutdown(self): """Shut down this worker if it has been asked to shut down gracefully, and has no active builders.""" if not self.worker_status.getGraceful(): return active_builders = [sb for sb in itervalues(self.workerforbuilders) if sb.isBusy()] if active_builders: return d = self.shutdown() d.addErrback(log.err, 'error while shutting down worker') def _pauseChanged(self, paused): if paused is True: self.botmaster.master.status.workerPaused(self.name) else: self.botmaster.master.status.workerUnpaused(self.name) def pause(self): """Stop running new builds on the worker.""" self.worker_status.setPaused(True) def unpause(self): """Restart running new builds on the worker.""" self.worker_status.setPaused(False) self.botmaster.maybeStartBuildsForWorker(self.name) def isPaused(self): return self.worker_status.isPaused()
class AbstractWorker(service.BuildbotService): """This is the master-side representative for a remote buildbot worker. There is exactly one for each worker described in the config file (the c['workers'] list). When buildbots connect in (.attach), they get a reference to this instance. The BotMaster object is stashed as the .botmaster attribute. The BotMaster is also our '.parent' Service. I represent a worker -- a remote machine capable of running builds. I am instantiated by the configuration file, and can be subclassed to add extra functionality.""" # reconfig workers after builders reconfig_priority = 64 quarantine_timer = None quarantine_timeout = quarantine_initial_timeout = 10 quarantine_max_timeout = 60 * 60 start_missing_on_startup = True DEFAULT_MISSING_TIMEOUT = 3600 DEFAULT_KEEPALIVE_INTERVAL = 3600 # override to True if isCompatibleWithBuild may return False builds_may_be_incompatible = False def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=None, properties=None, defaultProperties=None, locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL, machine_name=None): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param defaultProperties: properties that will be applied to builds run on this worker only if the property has not been set by another source @type defaultProperties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary @param machine_name: The name of the machine to associate with the worker. """ self.name = name = bytes2unicode(name) self.machine_name = machine_name self.password = password # protocol registration self.registration = None self._graceful = False self._paused = False # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties or {}, "Worker") self.properties.setProperty("workername", name, "Worker") self.defaultProperties = Properties() self.defaultProperties.update(defaultProperties or {}, "Worker") if self.machine_name is not None: self.properties.setProperty('machine_name', self.machine_name, 'Worker') self.machine = None self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error('notify_on_missing arg %r is not a string' % (i, )) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None # during disconnection self.conn will be set to None before all disconnection notifications # are delivered. During that period _pending_disconnection_delivery_notifier will be set to # a notifier and allows interested users to wait until all disconnection notifications are # delivered. self._pending_disconnection_delivery_notifier = None self._old_builder_list = None self._configured_builderid_list = None def __repr__(self): return "<{} {}>".format(self.__class__.__name__, repr(self.name)) @property def workername(self): # workername is now an alias to twisted.Service's name return self.name @property def botmaster(self): if self.master is None: return None return self.master.botmaster @defer.inlineCallbacks def updateLocks(self): """Convert the L{LockAccess} objects in C{self.locks} into real lock objects, while also maintaining the subscriptions to lock releases.""" # unsubscribe from any old locks for s in self.lock_subscriptions: s.unsubscribe() # convert locks into their real form locks = yield self.botmaster.getLockFromLockAccesses( self.access, self.config_version) self.locks = [(l.getLockForWorker(self.workername), la) for l, la in locks] self.lock_subscriptions = [ l.subscribeToReleases(self._lockReleased) for l, la in self.locks ] def locksAvailable(self): """ I am called to see if all the locks I depend on are available, in which I return True, otherwise I return False """ if not self.locks: return True for lock, access in self.locks: if not lock.isAvailable(self, access): return False return True def acquireLocks(self): """ I am called when a build is preparing to run. I try to claim all the locks that are needed for a build to happen. If I can't, then my caller should give up the build and try to get another worker to look at it. """ log.msg("acquireLocks(worker {}, locks {})".format(self, self.locks)) if not self.locksAvailable(): log.msg("worker {} can't lock, giving up".format(self)) return False # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) return True def releaseLocks(self): """ I am called to release any locks after a build has finished """ log.msg("releaseLocks({}): {}".format(self, self.locks)) for lock, access in self.locks: lock.release(self, access) def _lockReleased(self): """One of the locks for this worker was released; try scheduling builds.""" if not self.botmaster: return # oh well.. self.botmaster.maybeStartBuildsForWorker(self.name) def _applyWorkerInfo(self, info): if not info: return self.worker_status.setAdmin(info.get("admin")) self.worker_status.setHost(info.get("host")) self.worker_status.setAccessURI(info.get("access_uri", None)) self.worker_status.setVersion(info.get("version", "(unknown)")) # store everything as Properties for k, v in info.items(): if k in ('environ', 'worker_commands'): continue self.worker_status.info.setProperty(k, v, "Worker") @defer.inlineCallbacks def _getWorkerInfo(self): worker = yield self.master.data.get(('workers', self.workerid)) self._applyWorkerInfo(worker['workerinfo']) def setServiceParent(self, parent): # botmaster needs to set before setServiceParent which calls # startService self.manager = parent return super().setServiceParent(parent) @defer.inlineCallbacks def startService(self): # tracks config version for locks self.config_version = self.master.config_version self.updateLocks() self.workerid = yield self.master.data.updates.findWorkerId(self.name) self.workerActionConsumer = yield self.master.mq.startConsuming( self.controlWorker, ("control", "worker", str(self.workerid), None)) yield self._getWorkerInfo() yield super().startService() # startMissingTimer wants the service to be running to really start if self.start_missing_on_startup: self.startMissingTimer() @defer.inlineCallbacks def reconfigService(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=DEFAULT_MISSING_TIMEOUT, properties=None, defaultProperties=None, locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL, machine_name=None): # Given a Worker config arguments, configure this one identically. # Because Worker objects are remotely referenced, we can't replace them # without disconnecting the worker, yet there's no reason to do that. assert self.name == name self.password = password # adopt new instance's configuration parameters self.max_builds = max_builds self.access = [] if locks: self.access = locks if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing if self.missing_timeout != missing_timeout: running_missing_timer = self.missing_timer self.stopMissingTimer() self.missing_timeout = missing_timeout if running_missing_timer: self.startMissingTimer() self.properties = Properties() self.properties.update(properties or {}, "Worker") self.properties.setProperty("workername", name, "Worker") self.defaultProperties = Properties() self.defaultProperties.update(defaultProperties or {}, "Worker") # Note that before first reconfig self.machine will always be None and # out of sync with self.machine_name, thus more complex logic is needed. if self.machine is not None and self.machine_name != machine_name: self.machine.unregisterWorker(self) self.machine = None self.machine_name = machine_name if self.machine is None and self.machine_name is not None: self.machine = self.master.machine_manager.getMachineByName( self.machine_name) if self.machine is not None: self.machine.registerWorker(self) self.properties.setProperty("machine_name", self.machine_name, "Worker") else: log.err("Unknown machine '{}' for worker '{}'".format( self.machine_name, self.name)) # update our records with the worker manager if not self.registration: self.registration = yield self.master.workers.register(self) yield self.registration.update(self, self.master.config) # tracks config version for locks self.config_version = self.master.config_version self.updateLocks() @defer.inlineCallbacks def reconfigServiceWithSibling(self, sibling): # reconfigServiceWithSibling will only reconfigure the worker when it is configured # differently. # However, the worker configuration depends on which builder it is configured yield super().reconfigServiceWithSibling(sibling) # update the attached worker's notion of which builders are attached. # This assumes that the relevant builders have already been configured, # which is why the reconfig_priority is set low in this class. bids = [ b.getBuilderId() for b in self.botmaster.getBuildersForWorker(self.name) ] bids = yield defer.gatherResults(bids, consumeErrors=True) if self._configured_builderid_list != bids: yield self.master.data.updates.workerConfigured( self.workerid, self.master.masterid, bids) yield self.updateWorker() self._configured_builderid_list = bids @defer.inlineCallbacks def stopService(self): if self.registration: yield self.registration.unregister() self.registration = None self.workerActionConsumer.stopConsuming() self.stopMissingTimer() self.stopQuarantineTimer() # mark this worker as configured for zero builders in this master yield self.master.data.updates.workerConfigured( self.workerid, self.master.masterid, []) # during master shutdown we need to wait until the disconnection notification deliveries # are completed, otherwise some of the events may still be firing long after the master # is completely shut down. yield self.disconnect() yield self.waitForCompleteShutdown() yield super().stopService() def isCompatibleWithBuild(self, build_props): # given a build properties object, determines whether the build is # compatible with the currently running worker or not. This is most # often useful for latent workers where it's possible to request # different kinds of workers. return defer.succeed(True) def startMissingTimer(self): if self.missing_timeout and self.parent and self.running: self.stopMissingTimer() # in case it's already running self.missing_timer = self.master.reactor.callLater( self.missing_timeout, self._missing_timer_fired) def stopMissingTimer(self): if self.missing_timer: if self.missing_timer.active(): self.missing_timer.cancel() self.missing_timer = None def isConnected(self): return self.conn def _missing_timer_fired(self): self.missing_timer = None # notify people, but only if we're still in the config if not self.parent: return last_connection = time.ctime(time.time() - self.missing_timeout) self.master.data.updates.workerMissing(workerid=self.workerid, masterid=self.master.masterid, last_connection=last_connection, notify=self.notify_on_missing) def updateWorker(self): """Called to add or remove builders after the worker has connected. @return: a Deferred that indicates when an attached worker has accepted the new builders and/or released the old ones.""" if self.conn: return self.sendBuilderList() # else: return defer.succeed(None) @defer.inlineCallbacks def attached(self, conn): """This is called when the worker connects.""" assert self.conn is None metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1) # now we go through a sequence of calls, gathering information, then # tell the Botmaster that it can finally give this worker to all the # Builders that care about it. # Reset graceful shutdown status self._graceful = False self.conn = conn self._old_builder_list = None # clear builder list before proceed self.worker_status.setConnected(True) self._applyWorkerInfo(conn.info) self.worker_commands = conn.info.get("worker_commands", {}) self.worker_environ = conn.info.get("environ", {}) self.worker_basedir = conn.info.get("basedir", None) self.worker_system = conn.info.get("system", None) self.conn.notifyOnDisconnect(self.detached) workerinfo = { 'admin': conn.info.get('admin'), 'host': conn.info.get('host'), 'access_uri': conn.info.get('access_uri'), 'version': conn.info.get('version') } yield self.master.data.updates.workerConnected( workerid=self.workerid, masterid=self.master.masterid, workerinfo=workerinfo) if self.worker_system == "nt": self.path_module = namedModule("ntpath") else: # most everything accepts / as separator, so posix should be a # reasonable fallback self.path_module = namedModule("posixpath") log.msg("bot attached") self.messageReceivedFromWorker() self.stopMissingTimer() yield self.updateWorker() yield self.botmaster.maybeStartBuildsForWorker(self.name) self.updateState() def messageReceivedFromWorker(self): now = time.time() self.lastMessageReceived = now self.worker_status.setLastMessageReceived(now) def setupProperties(self, props): for name in self.properties.properties: props.setProperty(name, self.properties.getProperty(name), "Worker") for name in self.defaultProperties.properties: if name not in props: props.setProperty(name, self.defaultProperties.getProperty(name), "Worker") @defer.inlineCallbacks def _handle_disconnection_delivery_notifier(self): self._pending_disconnection_delivery_notifier = Notifier() yield self.conn.waitForNotifyDisconnectedDelivered() self._pending_disconnection_delivery_notifier.notify(None) self._pending_disconnection_delivery_notifier = None @defer.inlineCallbacks def detached(self): # protect against race conditions in conn disconnect path and someone # calling detached directly. At the moment the null worker does that. if self.conn is None: return metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1) self._handle_disconnection_delivery_notifier() yield self.conn.waitShutdown() self.conn = None self._old_builder_list = [] self.worker_status.setConnected(False) log.msg("Worker.detached({})".format(self.name)) self.releaseLocks() yield self.master.data.updates.workerDisconnected( workerid=self.workerid, masterid=self.master.masterid, ) def disconnect(self): """Forcibly disconnect the worker. This severs the TCP connection and returns a Deferred that will fire (with None) when the connection is probably gone. If the worker is still alive, they will probably try to reconnect again in a moment. This is called in two circumstances. The first is when a worker is removed from the config file. In this case, when they try to reconnect, they will be rejected as an unknown worker. The second is when we wind up with two connections for the same worker, in which case we disconnect the older connection. """ if self.conn is None: return defer.succeed(None) log.msg("disconnecting old worker {} now".format(self.name)) # When this Deferred fires, we'll be ready to accept the new worker return self._disconnect(self.conn) def waitForCompleteShutdown(self): # This function waits until the disconnection to happen and the disconnection # notifications have been delivered and acted upon. return self._waitForCompleteShutdownImpl(self.conn) @defer.inlineCallbacks def _waitForCompleteShutdownImpl(self, conn): if conn: d = defer.Deferred() def _disconnected(): eventually(d.callback, None) conn.notifyOnDisconnect(_disconnected) yield d yield conn.waitForNotifyDisconnectedDelivered() elif self._pending_disconnection_delivery_notifier is not None: yield self._pending_disconnection_delivery_notifier.wait() @defer.inlineCallbacks def _disconnect(self, conn): # This function waits until the disconnection to happen and the disconnection # notifications have been delivered and acted upon d = self._waitForCompleteShutdownImpl(conn) conn.loseConnection() log.msg("waiting for worker to finish disconnecting") yield d @defer.inlineCallbacks def sendBuilderList(self): our_builders = self.botmaster.getBuildersForWorker(self.name) blist = [(b.name, b.config.workerbuilddir) for b in our_builders] if blist == self._old_builder_list: return slist = yield self.conn.remoteSetBuilderList(builders=blist) self._old_builder_list = blist # Nothing has changed, so don't need to re-attach to everything if not slist: return dl = [] for name in slist: # use get() since we might have changed our mind since then b = self.botmaster.builders.get(name) if b: d1 = self.attachBuilder(b) dl.append(d1) yield defer.DeferredList(dl) def attachBuilder(self, builder): return builder.attached(self, self.worker_commands) def controlWorker(self, key, params): log.msg("worker {} wants to {}: {}".format(self.name, key[-1], params)) if key[-1] == "stop": return self.shutdownRequested() if key[-1] == "pause": self.pause() if key[-1] == "unpause": self.unpause() if key[-1] == "kill": self.shutdown() def shutdownRequested(self): self._graceful = True self.maybeShutdown() self.updateState() def addWorkerForBuilder(self, wfb): self.workerforbuilders[wfb.builder_name] = wfb def removeWorkerForBuilder(self, wfb): try: del self.workerforbuilders[wfb.builder_name] except KeyError: pass def buildFinished(self, wfb): """This is called when a build on this worker is finished.""" self.botmaster.maybeStartBuildsForWorker(self.name) def canStartBuild(self): """ I am called when a build is requested to see if this worker can start a build. This function can be used to limit overall concurrency on the worker. Note for subclassers: if a worker can become willing to start a build without any action on that worker (for example, by a resource in use on another worker becoming available), then you must arrange for L{maybeStartBuildsForWorker} to be called at that time, or builds on this worker will not start. """ # If we're waiting to shutdown gracefully or paused, then we shouldn't # accept any new jobs. if self._graceful or self._paused: return False if self.max_builds: active_builders = [ wfb for wfb in self.workerforbuilders.values() if wfb.isBusy() ] if len(active_builders) >= self.max_builds: return False if not self.locksAvailable(): return False return True @defer.inlineCallbacks def shutdown(self): """Shutdown the worker""" if not self.conn: log.msg("no remote; worker is already shut down") return yield self.conn.remoteShutdown() def maybeShutdown(self): """Shut down this worker if it has been asked to shut down gracefully, and has no active builders.""" if not self._graceful: return active_builders = [ wfb for wfb in self.workerforbuilders.values() if wfb.isBusy() ] if active_builders: return d = self.shutdown() d.addErrback(log.err, 'error while shutting down worker') def updateState(self): self.master.data.updates.setWorkerState(self.workerid, self._paused, self._graceful) def pause(self): """Stop running new builds on the worker.""" self._paused = True self.updateState() def unpause(self): """Restart running new builds on the worker.""" self._paused = False self.botmaster.maybeStartBuildsForWorker(self.name) self.updateState() def isPaused(self): return self._paused def resetQuarantine(self): self.quarantine_timeout = self.quarantine_initial_timeout def putInQuarantine(self): if self.quarantine_timer: # already in quarantine return self.pause() self.quarantine_timer = self.master.reactor.callLater( self.quarantine_timeout, self.exitQuarantine) log.msg("{} has been put in quarantine for {}s".format( self.name, self.quarantine_timeout)) # next we will wait twice as long self.quarantine_timeout *= 2 if self.quarantine_timeout > self.quarantine_max_timeout: # unless we hit the max timeout self.quarantine_timeout = self.quarantine_max_timeout def exitQuarantine(self): self.quarantine_timer = None self.unpause() def stopQuarantineTimer(self): if self.quarantine_timer is not None: self.quarantine_timer.cancel() self.quarantine_timer = None self.unpause()
def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=None, properties=None, defaultProperties=None, locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL, machine_name=None): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param defaultProperties: properties that will be applied to builds run on this worker only if the property has not been set by another source @type defaultProperties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary @param machine_name: The name of the machine to associate with the worker. """ self.name = name = bytes2unicode(name) self.machine_name = machine_name self.password = password # protocol registration self.registration = None self._graceful = False self._paused = False # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties or {}, "Worker") self.properties.setProperty("workername", name, "Worker") self.defaultProperties = Properties() self.defaultProperties.update(defaultProperties or {}, "Worker") if self.machine_name is not None: self.properties.setProperty('machine_name', self.machine_name, 'Worker') self.machine = None self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error('notify_on_missing arg %r is not a string' % (i, )) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None # during disconnection self.conn will be set to None before all disconnection notifications # are delivered. During that period _pending_disconnection_delivery_notifier will be set to # a notifier and allows interested users to wait until all disconnection notifications are # delivered. self._pending_disconnection_delivery_notifier = None self._old_builder_list = None self._configured_builderid_list = None
class AbstractWorker(service.BuildbotService, object): """This is the master-side representative for a remote buildbot worker. There is exactly one for each worker described in the config file (the c['workers'] list). When buildbots connect in (.attach), they get a reference to this instance. The BotMaster object is stashed as the .botmaster attribute. The BotMaster is also our '.parent' Service. I represent a worker -- a remote machine capable of running builds. I am instantiated by the configuration file, and can be subclassed to add extra functionality.""" # reconfig workers after builders reconfig_priority = 64 quarantine_timer = None quarantine_timeout = quarantine_initial_timeout = 10 quarantine_max_timeout = 60 * 60 start_missing_on_startup = True def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=10 * 60, # Ten minutes properties=None, locks=None, keepalive_interval=3600): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary """ self.name = name = ascii2unicode(name) if properties is None: properties = {} self.password = password # protocol registration self.registration = None self._graceful = False self._paused = False # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error( 'notify_on_missing arg %r is not a string' % (i,)) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None self._old_builder_list = None self._configured_builderid_list = None def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.name) @property def workername(self): # workername is now an alias to twisted.Service's name return self.name deprecatedWorkerClassProperty(locals(), workername) @property def botmaster(self): if self.master is None: return None return self.master.botmaster def updateLocks(self): """Convert the L{LockAccess} objects in C{self.locks} into real lock objects, while also maintaining the subscriptions to lock releases.""" # unsubscribe from any old locks for s in self.lock_subscriptions: s.unsubscribe() # convert locks into their real form locks = [(self.botmaster.getLockFromLockAccess(a), a) for a in self.access] self.locks = [(l.getLock(self), la) for l, la in locks] self.lock_subscriptions = [l.subscribeToReleases(self._lockReleased) for l, la in self.locks] def locksAvailable(self): """ I am called to see if all the locks I depend on are available, in which I return True, otherwise I return False """ if not self.locks: return True for lock, access in self.locks: if not lock.isAvailable(self, access): return False return True def acquireLocks(self): """ I am called when a build is preparing to run. I try to claim all the locks that are needed for a build to happen. If I can't, then my caller should give up the build and try to get another worker to look at it. """ log.msg("acquireLocks(worker %s, locks %s)" % (self, self.locks)) if not self.locksAvailable(): log.msg("worker %s can't lock, giving up" % (self, )) return False # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) return True def releaseLocks(self): """ I am called to release any locks after a build has finished """ log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: lock.release(self, access) def _lockReleased(self): """One of the locks for this worker was released; try scheduling builds.""" if not self.botmaster: return # oh well.. self.botmaster.maybeStartBuildsForWorker(self.name) def _applyWorkerInfo(self, info): if not info: return self.worker_status.setAdmin(info.get("admin")) self.worker_status.setHost(info.get("host")) self.worker_status.setAccessURI(info.get("access_uri", None)) self.worker_status.setVersion(info.get("version", "(unknown)")) @defer.inlineCallbacks def _getWorkerInfo(self): worker = yield self.master.data.get( ('workers', self.workerid)) self._applyWorkerInfo(worker['workerinfo']) def setServiceParent(self, parent): # botmaster needs to set before setServiceParent which calls # startService self.manager = parent return service.BuildbotService.setServiceParent(self, parent) @defer.inlineCallbacks def startService(self): self.updateLocks() self.workerid = yield self.master.data.updates.findWorkerId( self.name) self.workerActionConsumer = yield self.master.mq.startConsuming(self.controlWorker, ("control", "worker", str(self.workerid), None)) yield self._getWorkerInfo() yield service.BuildbotService.startService(self) # startMissingTimer wants the service to be running to really start if self.start_missing_on_startup: self.startMissingTimer() @defer.inlineCallbacks def reconfigService(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=3600, properties=None, locks=None, keepalive_interval=3600): # Given a Worker config arguments, configure this one identically. # Because Worker objects are remotely referenced, we can't replace them # without disconnecting the worker, yet there's no reason to do that. assert self.name == name self.password = password # adopt new instance's configuration parameters self.max_builds = max_builds self.access = [] if locks: self.access = locks self.notify_on_missing = notify_on_missing if self.missing_timeout != missing_timeout: running_missing_timer = self.missing_timer self.stopMissingTimer() self.missing_timeout = missing_timeout if running_missing_timer: self.startMissingTimer() if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") # update our records with the worker manager if not self.registration: self.registration = yield self.master.workers.register(self) yield self.registration.update(self, self.master.config) self.updateLocks() @defer.inlineCallbacks def reconfigServiceWithSibling(self, sibling): # reconfigServiceWithSibling will only reconfigure the worker when it is configured differently. # However, the worker configuration depends on which builder it is configured yield service.BuildbotService.reconfigServiceWithSibling(self, sibling) # update the attached worker's notion of which builders are attached. # This assumes that the relevant builders have already been configured, # which is why the reconfig_priority is set low in this class. bids = [ b.getBuilderId() for b in self.botmaster.getBuildersForWorker(self.name)] bids = yield defer.gatherResults(bids, consumeErrors=True) if self._configured_builderid_list != bids: yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, bids) yield self.updateWorker() self._configured_builderid_list = bids @defer.inlineCallbacks def stopService(self): if self.registration: yield self.registration.unregister() self.registration = None self.workerActionConsumer.stopConsuming() self.stopMissingTimer() self.stopQuarantineTimer() # mark this worker as configured for zero builders in this master yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, []) yield service.BuildbotService.stopService(self) def startMissingTimer(self): if self.missing_timeout and self.parent and self.running: self.stopMissingTimer() # in case it's already running self.missing_timer = self.master.reactor.callLater(self.missing_timeout, self._missing_timer_fired) def stopMissingTimer(self): if self.missing_timer: if self.missing_timer.active(): self.missing_timer.cancel() self.missing_timer = None def isConnected(self): return self.conn def _missing_timer_fired(self): self.missing_timer = None # notify people, but only if we're still in the config if not self.parent: return last_connection = time.ctime(time.time() - self.missing_timeout) self.master.data.updates.workerMissing( workerid=self.workerid, masterid=self.master.masterid, last_connection=last_connection, notify=self.notify_on_missing ) def updateWorker(self): """Called to add or remove builders after the worker has connected. @return: a Deferred that indicates when an attached worker has accepted the new builders and/or released the old ones.""" if self.conn: return self.sendBuilderList() # else: return defer.succeed(None) @defer.inlineCallbacks def attached(self, conn): """This is called when the worker connects.""" metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1) # now we go through a sequence of calls, gathering information, then # tell the Botmaster that it can finally give this worker to all the # Builders that care about it. # Reset graceful shutdown status self._graceful = False self.conn = conn self._old_builder_list = None # clear builder list before proceed self.worker_status.setConnected(True) self._applyWorkerInfo(conn.info) self.worker_commands = conn.info.get("worker_commands", {}) self.worker_environ = conn.info.get("environ", {}) self.worker_basedir = conn.info.get("basedir", None) self.worker_system = conn.info.get("system", None) self.conn.notifyOnDisconnect(self.detached) workerinfo = { 'admin': conn.info.get('admin'), 'host': conn.info.get('host'), 'access_uri': conn.info.get('access_uri'), 'version': conn.info.get('version') } yield self.master.data.updates.workerConnected( workerid=self.workerid, masterid=self.master.masterid, workerinfo=workerinfo ) if self.worker_system == "nt": self.path_module = namedModule("ntpath") else: # most everything accepts / as separator, so posix should be a # reasonable fallback self.path_module = namedModule("posixpath") log.msg("bot attached") self.messageReceivedFromWorker() self.stopMissingTimer() yield self.updateWorker() yield self.botmaster.maybeStartBuildsForWorker(self.name) def messageReceivedFromWorker(self): now = time.time() self.lastMessageReceived = now self.worker_status.setLastMessageReceived(now) @defer.inlineCallbacks def detached(self): metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1) self.conn = None self._old_builder_list = [] self.worker_status.setConnected(False) log.msg("Worker.detached(%s)" % (self.name,)) self.releaseLocks() yield self.master.data.updates.workerDisconnected( workerid=self.workerid, masterid=self.master.masterid, ) def disconnect(self): """Forcibly disconnect the worker. This severs the TCP connection and returns a Deferred that will fire (with None) when the connection is probably gone. If the worker is still alive, they will probably try to reconnect again in a moment. This is called in two circumstances. The first is when a worker is removed from the config file. In this case, when they try to reconnect, they will be rejected as an unknown worker. The second is when we wind up with two connections for the same worker, in which case we disconnect the older connection. """ if self.conn is None: return defer.succeed(None) log.msg("disconnecting old worker %s now" % (self.name,)) # When this Deferred fires, we'll be ready to accept the new worker return self._disconnect(self.conn) def _disconnect(self, conn): # all kinds of teardown will happen as a result of # loseConnection(), but it happens after a reactor iteration or # two. Hook the actual disconnect so we can know when it is safe # to connect the new worker. We have to wait one additional # iteration (with callLater(0)) to make sure the *other* # notifyOnDisconnect handlers have had a chance to run. d = defer.Deferred() # notifyOnDisconnect runs the callback def _disconnected(): eventually(d.callback, None) conn.notifyOnDisconnect(_disconnected) conn.loseConnection() log.msg("waiting for worker to finish disconnecting") return d @defer.inlineCallbacks def sendBuilderList(self): our_builders = self.botmaster.getBuildersForWorker(self.name) blist = [(b.name, b.config.workerbuilddir) for b in our_builders] if blist == self._old_builder_list: return slist = yield self.conn.remoteSetBuilderList(builders=blist) self._old_builder_list = blist # Nothing has changed, so don't need to re-attach to everything if not slist: return dl = [] for name in slist: # use get() since we might have changed our mind since then b = self.botmaster.builders.get(name) if b: d1 = self.attachBuilder(b) dl.append(d1) yield defer.DeferredList(dl) def attachBuilder(self, builder): return builder.attached(self, self.worker_commands) def controlWorker(self, key, params): log.msg("worker {} wants to {}: {}".format(self.name, key[-1], params)) if key[-1] == "stop": return self.shutdownRequested() if key[-1] == "pause": self.pause() if key[-1] == "unpause": self.unpause() if key[-1] == "kill": self.shutdown() def shutdownRequested(self): self._graceful = True self.maybeShutdown() def addWorkerForBuilder(self, wfb): self.workerforbuilders[wfb.builder_name] = wfb def removeWorkerForBuilder(self, wfb): try: del self.workerforbuilders[wfb.builder_name] except KeyError: pass def buildFinished(self, wfb): """This is called when a build on this worker is finished.""" self.botmaster.maybeStartBuildsForWorker(self.name) def canStartBuild(self): """ I am called when a build is requested to see if this worker can start a build. This function can be used to limit overall concurrency on the worker. Note for subclassers: if a worker can become willing to start a build without any action on that worker (for example, by a resource in use on another worker becoming available), then you must arrange for L{maybeStartBuildsForWorker} to be called at that time, or builds on this worker will not start. """ if self.quarantine_timer: return False # If we're waiting to shutdown gracefully or paused, then we shouldn't # accept any new jobs. if self._graceful or self._paused: return False if self.max_builds: active_builders = [wfb for wfb in itervalues(self.workerforbuilders) if wfb.isBusy()] if len(active_builders) >= self.max_builds: return False if not self.locksAvailable(): return False return True @defer.inlineCallbacks def shutdown(self): """Shutdown the worker""" if not self.conn: log.msg("no remote; worker is already shut down") return yield self.conn.remoteShutdown() def maybeShutdown(self): """Shut down this worker if it has been asked to shut down gracefully, and has no active builders.""" if not self._graceful: return active_builders = [wfb for wfb in itervalues(self.workerforbuilders) if wfb.isBusy()] if active_builders: return d = self.shutdown() d.addErrback(log.err, 'error while shutting down worker') def pause(self): """Stop running new builds on the worker.""" self._paused = True def unpause(self): """Restart running new builds on the worker.""" self._paused = False self.botmaster.maybeStartBuildsForWorker(self.name) def isPaused(self): return self._paused def resetQuarantine(self): self.quarantine_timeout = self.quarantine_initial_timeout def putInQuarantine(self): if self.quarantine_timer: # already in quarantine return self.quarantine_timer = self.master.reactor.callLater( self.quarantine_timeout, self.exitQuarantine) log.msg("{} has been put in quarantine for {}s".format( self.name, self.quarantine_timeout)) # next we will wait twice as long self.quarantine_timeout *= 2 if self.quarantine_timeout > self.quarantine_max_timeout: # unless we hit the max timeout self.quarantine_timeout = self.quarantine_max_timeout def exitQuarantine(self): self.quarantine_timer = None self.botmaster.maybeStartBuildsForWorker(self.name) def stopQuarantineTimer(self): if self.quarantine_timer is not None: self.quarantine_timer.cancel() self.quarantine_timer = None