class AbstractWorker(service.BuildbotService, object): """This is the master-side representative for a remote buildbot worker. There is exactly one for each worker described in the config file (the c['workers'] list). When buildbots connect in (.attach), they get a reference to this instance. The BotMaster object is stashed as the .botmaster attribute. The BotMaster is also our '.parent' Service. I represent a worker -- a remote machine capable of running builds. I am instantiated by the configuration file, and can be subclassed to add extra functionality.""" # reconfig workers after builders reconfig_priority = 64 quarantine_timer = None quarantine_timeout = quarantine_initial_timeout = 10 quarantine_max_timeout = 60 * 60 start_missing_on_startup = True DEFAULT_MISSING_TIMEOUT = 3600 DEFAULT_KEEPALIVE_INTERVAL = 3600 # override to True if isCompatibleWithBuild may return False builds_may_be_incompatible = False def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=None, properties=None, locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary """ self.name = name = bytes2unicode(name) if properties is None: properties = {} self.password = password # protocol registration self.registration = None self._graceful = False self._paused = False # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error('notify_on_missing arg %r is not a string' % (i, )) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None self._old_builder_list = None self._configured_builderid_list = None def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.name) @property def workername(self): # workername is now an alias to twisted.Service's name return self.name deprecatedWorkerClassProperty(locals(), workername) @property def botmaster(self): if self.master is None: return None return self.master.botmaster def updateLocks(self): """Convert the L{LockAccess} objects in C{self.locks} into real lock objects, while also maintaining the subscriptions to lock releases.""" # unsubscribe from any old locks for s in self.lock_subscriptions: s.unsubscribe() # convert locks into their real form locks = [(self.botmaster.getLockFromLockAccess(a), a) for a in self.access] self.locks = [(l.getLock(self), la) for l, la in locks] self.lock_subscriptions = [ l.subscribeToReleases(self._lockReleased) for l, la in self.locks ] def locksAvailable(self): """ I am called to see if all the locks I depend on are available, in which I return True, otherwise I return False """ if not self.locks: return True for lock, access in self.locks: if not lock.isAvailable(self, access): return False return True def acquireLocks(self): """ I am called when a build is preparing to run. I try to claim all the locks that are needed for a build to happen. If I can't, then my caller should give up the build and try to get another worker to look at it. """ log.msg("acquireLocks(worker %s, locks %s)" % (self, self.locks)) if not self.locksAvailable(): log.msg("worker %s can't lock, giving up" % (self, )) return False # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) return True def releaseLocks(self): """ I am called to release any locks after a build has finished """ log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: lock.release(self, access) def _lockReleased(self): """One of the locks for this worker was released; try scheduling builds.""" if not self.botmaster: return # oh well.. self.botmaster.maybeStartBuildsForWorker(self.name) def _applyWorkerInfo(self, info): if not info: return self.worker_status.setAdmin(info.get("admin")) self.worker_status.setHost(info.get("host")) self.worker_status.setAccessURI(info.get("access_uri", None)) self.worker_status.setVersion(info.get("version", "(unknown)")) @defer.inlineCallbacks def _getWorkerInfo(self): worker = yield self.master.data.get(('workers', self.workerid)) self._applyWorkerInfo(worker['workerinfo']) def setServiceParent(self, parent): # botmaster needs to set before setServiceParent which calls # startService self.manager = parent return service.BuildbotService.setServiceParent(self, parent) @defer.inlineCallbacks def startService(self): self.updateLocks() self.workerid = yield self.master.data.updates.findWorkerId(self.name) self.workerActionConsumer = yield self.master.mq.startConsuming( self.controlWorker, ("control", "worker", str(self.workerid), None)) yield self._getWorkerInfo() yield service.BuildbotService.startService(self) # startMissingTimer wants the service to be running to really start if self.start_missing_on_startup: self.startMissingTimer() @defer.inlineCallbacks def reconfigService(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=DEFAULT_MISSING_TIMEOUT, properties=None, locks=None, keepalive_interval=DEFAULT_KEEPALIVE_INTERVAL): # Given a Worker config arguments, configure this one identically. # Because Worker objects are remotely referenced, we can't replace them # without disconnecting the worker, yet there's no reason to do that. assert self.name == name self.password = password # adopt new instance's configuration parameters self.max_builds = max_builds self.access = [] if locks: self.access = locks if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing if self.missing_timeout != missing_timeout: running_missing_timer = self.missing_timer self.stopMissingTimer() self.missing_timeout = missing_timeout if running_missing_timer: self.startMissingTimer() if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") # update our records with the worker manager if not self.registration: self.registration = yield self.master.workers.register(self) yield self.registration.update(self, self.master.config) self.updateLocks() @defer.inlineCallbacks def reconfigServiceWithSibling(self, sibling): # reconfigServiceWithSibling will only reconfigure the worker when it is configured differently. # However, the worker configuration depends on which builder it is configured yield service.BuildbotService.reconfigServiceWithSibling(self, sibling) # update the attached worker's notion of which builders are attached. # This assumes that the relevant builders have already been configured, # which is why the reconfig_priority is set low in this class. bids = [ b.getBuilderId() for b in self.botmaster.getBuildersForWorker(self.name) ] bids = yield defer.gatherResults(bids, consumeErrors=True) if self._configured_builderid_list != bids: yield self.master.data.updates.workerConfigured( self.workerid, self.master.masterid, bids) yield self.updateWorker() self._configured_builderid_list = bids @defer.inlineCallbacks def stopService(self): if self.registration: yield self.registration.unregister() self.registration = None self.workerActionConsumer.stopConsuming() self.stopMissingTimer() self.stopQuarantineTimer() # mark this worker as configured for zero builders in this master yield self.master.data.updates.workerConfigured( self.workerid, self.master.masterid, []) yield service.BuildbotService.stopService(self) def isCompatibleWithBuild(self, build_props): # given a build properties object, determines whether the build is # compatible with the currently running worker or not. This is most # often useful for latent workers where it's possible to request # different kinds of workers. return defer.succeed(True) def startMissingTimer(self): if self.missing_timeout and self.parent and self.running: self.stopMissingTimer() # in case it's already running self.missing_timer = self.master.reactor.callLater( self.missing_timeout, self._missing_timer_fired) def stopMissingTimer(self): if self.missing_timer: if self.missing_timer.active(): self.missing_timer.cancel() self.missing_timer = None def isConnected(self): return self.conn def _missing_timer_fired(self): self.missing_timer = None # notify people, but only if we're still in the config if not self.parent: return last_connection = time.ctime(time.time() - self.missing_timeout) self.master.data.updates.workerMissing(workerid=self.workerid, masterid=self.master.masterid, last_connection=last_connection, notify=self.notify_on_missing) def updateWorker(self): """Called to add or remove builders after the worker has connected. @return: a Deferred that indicates when an attached worker has accepted the new builders and/or released the old ones.""" if self.conn: return self.sendBuilderList() # else: return defer.succeed(None) @defer.inlineCallbacks def attached(self, conn): """This is called when the worker connects.""" metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1) # now we go through a sequence of calls, gathering information, then # tell the Botmaster that it can finally give this worker to all the # Builders that care about it. # Reset graceful shutdown status self._graceful = False self.conn = conn self._old_builder_list = None # clear builder list before proceed self.worker_status.setConnected(True) self._applyWorkerInfo(conn.info) self.worker_commands = conn.info.get("worker_commands", {}) self.worker_environ = conn.info.get("environ", {}) self.worker_basedir = conn.info.get("basedir", None) self.worker_system = conn.info.get("system", None) self.conn.notifyOnDisconnect(self.detached) workerinfo = { 'admin': conn.info.get('admin'), 'host': conn.info.get('host'), 'access_uri': conn.info.get('access_uri'), 'version': conn.info.get('version') } yield self.master.data.updates.workerConnected( workerid=self.workerid, masterid=self.master.masterid, workerinfo=workerinfo) if self.worker_system == "nt": self.path_module = namedModule("ntpath") else: # most everything accepts / as separator, so posix should be a # reasonable fallback self.path_module = namedModule("posixpath") log.msg("bot attached") self.messageReceivedFromWorker() self.stopMissingTimer() yield self.updateWorker() yield self.botmaster.maybeStartBuildsForWorker(self.name) self.updateState() def messageReceivedFromWorker(self): now = time.time() self.lastMessageReceived = now self.worker_status.setLastMessageReceived(now) @defer.inlineCallbacks def detached(self): metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1) self.conn = None self._old_builder_list = [] self.worker_status.setConnected(False) log.msg("Worker.detached(%s)" % (self.name, )) self.releaseLocks() yield self.master.data.updates.workerDisconnected( workerid=self.workerid, masterid=self.master.masterid, ) def disconnect(self): """Forcibly disconnect the worker. This severs the TCP connection and returns a Deferred that will fire (with None) when the connection is probably gone. If the worker is still alive, they will probably try to reconnect again in a moment. This is called in two circumstances. The first is when a worker is removed from the config file. In this case, when they try to reconnect, they will be rejected as an unknown worker. The second is when we wind up with two connections for the same worker, in which case we disconnect the older connection. """ if self.conn is None: return defer.succeed(None) log.msg("disconnecting old worker %s now" % (self.name, )) # When this Deferred fires, we'll be ready to accept the new worker return self._disconnect(self.conn) def _disconnect(self, conn): # all kinds of teardown will happen as a result of # loseConnection(), but it happens after a reactor iteration or # two. Hook the actual disconnect so we can know when it is safe # to connect the new worker. We have to wait one additional # iteration (with callLater(0)) to make sure the *other* # notifyOnDisconnect handlers have had a chance to run. d = defer.Deferred() # notifyOnDisconnect runs the callback def _disconnected(): eventually(d.callback, None) conn.notifyOnDisconnect(_disconnected) conn.loseConnection() log.msg("waiting for worker to finish disconnecting") return d @defer.inlineCallbacks def sendBuilderList(self): our_builders = self.botmaster.getBuildersForWorker(self.name) blist = [(b.name, b.config.workerbuilddir) for b in our_builders] if blist == self._old_builder_list: return slist = yield self.conn.remoteSetBuilderList(builders=blist) self._old_builder_list = blist # Nothing has changed, so don't need to re-attach to everything if not slist: return dl = [] for name in slist: # use get() since we might have changed our mind since then b = self.botmaster.builders.get(name) if b: d1 = self.attachBuilder(b) dl.append(d1) yield defer.DeferredList(dl) def attachBuilder(self, builder): return builder.attached(self, self.worker_commands) def controlWorker(self, key, params): log.msg("worker {} wants to {}: {}".format(self.name, key[-1], params)) if key[-1] == "stop": return self.shutdownRequested() if key[-1] == "pause": self.pause() if key[-1] == "unpause": self.unpause() if key[-1] == "kill": self.shutdown() def shutdownRequested(self): self._graceful = True self.maybeShutdown() self.updateState() def addWorkerForBuilder(self, wfb): self.workerforbuilders[wfb.builder_name] = wfb def removeWorkerForBuilder(self, wfb): try: del self.workerforbuilders[wfb.builder_name] except KeyError: pass def buildFinished(self, wfb): """This is called when a build on this worker is finished.""" self.botmaster.maybeStartBuildsForWorker(self.name) def canStartBuild(self): """ I am called when a build is requested to see if this worker can start a build. This function can be used to limit overall concurrency on the worker. Note for subclassers: if a worker can become willing to start a build without any action on that worker (for example, by a resource in use on another worker becoming available), then you must arrange for L{maybeStartBuildsForWorker} to be called at that time, or builds on this worker will not start. """ # If we're waiting to shutdown gracefully or paused, then we shouldn't # accept any new jobs. if self._graceful or self._paused: return False if self.max_builds: active_builders = [ wfb for wfb in itervalues(self.workerforbuilders) if wfb.isBusy() ] if len(active_builders) >= self.max_builds: return False if not self.locksAvailable(): return False return True @defer.inlineCallbacks def shutdown(self): """Shutdown the worker""" if not self.conn: log.msg("no remote; worker is already shut down") return yield self.conn.remoteShutdown() def maybeShutdown(self): """Shut down this worker if it has been asked to shut down gracefully, and has no active builders.""" if not self._graceful: return active_builders = [ wfb for wfb in itervalues(self.workerforbuilders) if wfb.isBusy() ] if active_builders: return d = self.shutdown() d.addErrback(log.err, 'error while shutting down worker') def updateState(self): self.master.data.updates.setWorkerState(self.workerid, self._paused, self._graceful) def pause(self): """Stop running new builds on the worker.""" self._paused = True self.updateState() def unpause(self): """Restart running new builds on the worker.""" self._paused = False self.botmaster.maybeStartBuildsForWorker(self.name) self.updateState() def isPaused(self): return self._paused def resetQuarantine(self): self.quarantine_timeout = self.quarantine_initial_timeout def putInQuarantine(self): if self.quarantine_timer: # already in quarantine return self.pause() self.quarantine_timer = self.master.reactor.callLater( self.quarantine_timeout, self.exitQuarantine) log.msg("{} has been put in quarantine for {}s".format( self.name, self.quarantine_timeout)) # next we will wait twice as long self.quarantine_timeout *= 2 if self.quarantine_timeout > self.quarantine_max_timeout: # unless we hit the max timeout self.quarantine_timeout = self.quarantine_max_timeout def exitQuarantine(self): self.quarantine_timer = None self.unpause() def stopQuarantineTimer(self): if self.quarantine_timer is not None: self.quarantine_timer.cancel() self.quarantine_timer = None self.unpause()
class AbstractWorker(service.BuildbotService, object): """This is the master-side representative for a remote buildbot worker. There is exactly one for each worker described in the config file (the c['workers'] list). When buildbots connect in (.attach), they get a reference to this instance. The BotMaster object is stashed as the .botmaster attribute. The BotMaster is also our '.parent' Service. I represent a worker -- a remote machine capable of running builds. I am instantiated by the configuration file, and can be subclassed to add extra functionality.""" implements(IWorker) # reconfig workers after builders reconfig_priority = 64 def checkConfig(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=10 * 60, # Ten minutes properties=None, locks=None, keepalive_interval=3600): """ @param name: botname this machine will supply when it connects @param password: password this machine will supply when it connects @param max_builds: maximum number of simultaneous builds that will be run concurrently on this worker (the default is None for no limit) @param properties: properties that will be applied to builds run on this worker @type properties: dictionary @param locks: A list of locks that must be acquired before this worker can be used @type locks: dictionary """ self.name = name = ascii2unicode(name) if properties is None: properties = {} self.password = password # protocol registration self.registration = None # these are set when the service is started self.manager = None self.workerid = None self.worker_status = WorkerStatus(name) self.worker_commands = None self.workerforbuilders = {} self.max_builds = max_builds self.access = [] if locks: self.access = locks self.lock_subscriptions = [] self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") self.lastMessageReceived = 0 if notify_on_missing is None: notify_on_missing = [] if isinstance(notify_on_missing, str): notify_on_missing = [notify_on_missing] self.notify_on_missing = notify_on_missing for i in notify_on_missing: if not isinstance(i, str): config.error( 'notify_on_missing arg %r is not a string' % (i,)) self.missing_timeout = missing_timeout self.missing_timer = None # a protocol connection, if we're currently connected self.conn = None self._old_builder_list = None def __repr__(self): return "<%s %r>" % (self.__class__.__name__, self.name) @property def workername(self): # workername is now an alias to twisted.Service's name return self.name deprecatedWorkerClassProperty(locals(), workername) @property def botmaster(self): if self.master is None: return None return self.master.botmaster def updateLocks(self): """Convert the L{LockAccess} objects in C{self.locks} into real lock objects, while also maintaining the subscriptions to lock releases.""" # unsubscribe from any old locks for s in self.lock_subscriptions: s.unsubscribe() # convert locks into their real form locks = [(self.botmaster.getLockFromLockAccess(a), a) for a in self.access] self.locks = [(l.getLock(self), la) for l, la in locks] self.lock_subscriptions = [l.subscribeToReleases(self._lockReleased) for l, la in self.locks] def locksAvailable(self): """ I am called to see if all the locks I depend on are available, in which I return True, otherwise I return False """ if not self.locks: return True for lock, access in self.locks: if not lock.isAvailable(self, access): return False return True def acquireLocks(self): """ I am called when a build is preparing to run. I try to claim all the locks that are needed for a build to happen. If I can't, then my caller should give up the build and try to get another worker to look at it. """ log.msg("acquireLocks(worker %s, locks %s)" % (self, self.locks)) if not self.locksAvailable(): log.msg("worker %s can't lock, giving up" % (self, )) return False # all locks are available, claim them all for lock, access in self.locks: lock.claim(self, access) return True def releaseLocks(self): """ I am called to release any locks after a build has finished """ log.msg("releaseLocks(%s): %s" % (self, self.locks)) for lock, access in self.locks: lock.release(self, access) def _lockReleased(self): """One of the locks for this worker was released; try scheduling builds.""" if not self.botmaster: return # oh well.. self.botmaster.maybeStartBuildsForWorker(self.name) def _applyWorkerInfo(self, info): if not info: return self.worker_status.setAdmin(info.get("admin")) self.worker_status.setHost(info.get("host")) self.worker_status.setAccessURI(info.get("access_uri", None)) self.worker_status.setVersion(info.get("version", "(unknown)")) @defer.inlineCallbacks def _getWorkerInfo(self): worker = yield self.master.data.get( ('workers', self.workerid)) self._applyWorkerInfo(worker['workerinfo']) def setServiceParent(self, parent): # botmaster needs to set before setServiceParent which calls # startService self.manager = parent return service.BuildbotService.setServiceParent(self, parent) @defer.inlineCallbacks def startService(self): self.updateLocks() self.startMissingTimer() self.workerid = yield self.master.data.updates.findWorkerId( self.name) yield self._getWorkerInfo() yield service.BuildbotService.startService(self) @defer.inlineCallbacks def reconfigService(self, name, password, max_builds=None, notify_on_missing=None, missing_timeout=3600, properties=None, locks=None, keepalive_interval=3600): # Given a Worker config arguments, configure this one identically. # Because Worker objects are remotely referenced, we can't replace them # without disconnecting the worker, yet there's no reason to do that. assert self.name == name self.password = password # adopt new instance's configuration parameters self.max_builds = max_builds self.access = [] if locks: self.access = locks self.notify_on_missing = notify_on_missing if self.missing_timeout != missing_timeout: running_missing_timer = self.missing_timer self.stopMissingTimer() self.missing_timeout = missing_timeout if running_missing_timer: self.startMissingTimer() if properties is None: properties = {} self.properties = Properties() self.properties.update(properties, "Worker") self.properties.setProperty("slavename", name, "Worker (deprecated)") self.properties.setProperty("workername", name, "Worker") # update our records with the worker manager if not self.registration: self.registration = yield self.master.workers.register(self) yield self.registration.update(self, self.master.config) self.updateLocks() bids = [ b._builderid for b in self.botmaster.getBuildersForWorker(self.name)] yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, bids) # update the attached worker's notion of which builders are attached. # This assumes that the relevant builders have already been configured, # which is why the reconfig_priority is set low in this class. yield self.updateWorker() @defer.inlineCallbacks def stopService(self): if self.registration: yield self.registration.unregister() self.registration = None self.stopMissingTimer() # mark this worker as configured for zero builders in this master yield self.master.data.updates.workerConfigured(self.workerid, self.master.masterid, []) yield service.BuildbotService.stopService(self) def startMissingTimer(self): if self.notify_on_missing and self.missing_timeout and self.parent: self.stopMissingTimer() # in case it's already running self.missing_timer = self.master.reactor.callLater(self.missing_timeout, self._missing_timer_fired) def stopMissingTimer(self): if self.missing_timer: self.missing_timer.cancel() self.missing_timer = None def isConnected(self): return self.conn def _missing_timer_fired(self): self.missing_timer = None # notify people, but only if we're still in the config if not self.parent: return buildmaster = self.botmaster.master status = buildmaster.getStatus() text = "The Buildbot working for '%s'\n" % status.getTitle() text += ("has noticed that the worker named %s went away\n" % self.name) text += "\n" text += ("It last disconnected at %s (buildmaster-local time)\n" % time.ctime(time.time() - self.missing_timeout)) # approx text += "\n" text += "The admin on record (as reported by WORKER:info/admin)\n" text += "was '%s'.\n" % self.worker_status.getAdmin() text += "\n" text += "Sincerely,\n" text += " The Buildbot\n" text += " %s\n" % status.getTitleURL() text += "\n" text += "%s\n" % status.getURLForThing(self.worker_status) subject = "Buildbot: worker %s was lost" % (self.name,) return self._mail_missing_message(subject, text) def updateWorker(self): """Called to add or remove builders after the worker has connected. @return: a Deferred that indicates when an attached worker has accepted the new builders and/or released the old ones.""" if self.conn: return self.sendBuilderList() else: return defer.succeed(None) @defer.inlineCallbacks def attached(self, conn): """This is called when the worker connects.""" metrics.MetricCountEvent.log("AbstractWorker.attached_workers", 1) # now we go through a sequence of calls, gathering information, then # tell the Botmaster that it can finally give this worker to all the # Builders that care about it. # Reset graceful shutdown status self.worker_status.setGraceful(False) # We want to know when the graceful shutdown flag changes self.worker_status.addGracefulWatcher(self._gracefulChanged) self.conn = conn self._old_builder_list = None # clear builder list before proceed self.worker_status.addPauseWatcher(self._pauseChanged) self.worker_status.setConnected(True) self._applyWorkerInfo(conn.info) self.worker_commands = conn.info.get("slave_commands", {}) self.worker_environ = conn.info.get("environ", {}) self.worker_basedir = conn.info.get("basedir", None) self.worker_system = conn.info.get("system", None) self.conn.notifyOnDisconnect(self.detached) workerinfo = { 'admin': conn.info.get('admin'), 'host': conn.info.get('host'), 'access_uri': conn.info.get('access_uri'), 'version': conn.info.get('version') } yield self.master.data.updates.workerConnected( workerid=self.workerid, masterid=self.master.masterid, workerinfo=workerinfo ) if self.worker_system == "nt": self.path_module = namedModule("ntpath") else: # most everything accepts / as separator, so posix should be a # reasonable fallback self.path_module = namedModule("posixpath") log.msg("bot attached") self.messageReceivedFromWorker() self.stopMissingTimer() self.master.status.workerConnected(self.name) yield self.updateWorker() yield self.botmaster.maybeStartBuildsForWorker(self.name) def messageReceivedFromWorker(self): now = time.time() self.lastMessageReceived = now self.worker_status.setLastMessageReceived(now) @defer.inlineCallbacks def detached(self): metrics.MetricCountEvent.log("AbstractWorker.attached_workers", -1) self.conn = None self._old_builder_list = [] self.worker_status.removeGracefulWatcher(self._gracefulChanged) self.worker_status.removePauseWatcher(self._pauseChanged) self.worker_status.setConnected(False) log.msg("Worker.detached(%s)" % (self.name,)) self.master.status.workerDisconnected(self.name) self.releaseLocks() yield self.master.data.updates.workerDisconnected( workerid=self.workerid, masterid=self.master.masterid, ) def disconnect(self): """Forcibly disconnect the worker. This severs the TCP connection and returns a Deferred that will fire (with None) when the connection is probably gone. If the worker is still alive, they will probably try to reconnect again in a moment. This is called in two circumstances. The first is when a worker is removed from the config file. In this case, when they try to reconnect, they will be rejected as an unknown worker. The second is when we wind up with two connections for the same worker, in which case we disconnect the older connection. """ if self.conn is None: return defer.succeed(None) log.msg("disconnecting old worker %s now" % (self.name,)) # When this Deferred fires, we'll be ready to accept the new worker return self._disconnect(self.conn) def _disconnect(self, conn): # all kinds of teardown will happen as a result of # loseConnection(), but it happens after a reactor iteration or # two. Hook the actual disconnect so we can know when it is safe # to connect the new worker. We have to wait one additional # iteration (with callLater(0)) to make sure the *other* # notifyOnDisconnect handlers have had a chance to run. d = defer.Deferred() # notifyOnDisconnect runs the callback def _disconnected(): eventually(d.callback, None) conn.notifyOnDisconnect(_disconnected) conn.loseConnection() log.msg("waiting for worker to finish disconnecting") return d def sendBuilderList(self): our_builders = self.botmaster.getBuildersForWorker(self.name) blist = [(b.name, b.config.workerbuilddir) for b in our_builders] if blist == self._old_builder_list: return defer.succeed(None) d = self.conn.remoteSetBuilderList(builders=blist) @d.addCallback def sentBuilderList(ign): self._old_builder_list = blist return ign return d def shutdownRequested(self): log.msg("worker %s wants to shut down" % (self.name,)) self.worker_status.setGraceful(True) def addWorkerForBuilder(self, wfb): self.workerforbuilders[wfb.builder_name] = wfb def removeWorkerForBuilder(self, wfb): try: del self.workerforbuilders[wfb.builder_name] except KeyError: pass def buildFinished(self, wfb): """This is called when a build on this worker is finished.""" self.botmaster.maybeStartBuildsForWorker(self.name) def canStartBuild(self): """ I am called when a build is requested to see if this worker can start a build. This function can be used to limit overall concurrency on the worker. Note for subclassers: if a worker can become willing to start a build without any action on that worker (for example, by a resource in use on another worker becoming available), then you must arrange for L{maybeStartBuildsForWorker} to be called at that time, or builds on this worker will not start. """ if self.worker_status.isPaused(): return False # If we're waiting to shutdown gracefully, then we shouldn't # accept any new jobs. if self.worker_status.getGraceful(): return False if self.max_builds: active_builders = [sb for sb in itervalues(self.workerforbuilders) if sb.isBusy()] if len(active_builders) >= self.max_builds: return False if not self.locksAvailable(): return False return True def _mail_missing_message(self, subject, text): # FIXME: This should be handled properly via the event api # we should send a missing message on the mq, and let any reporter # handle that # first, see if we have a MailNotifier we can use. This gives us a # fromaddr and a relayhost. buildmaster = self.botmaster.master for st in buildmaster.services: if isinstance(st, MailNotifier): break else: # if not, they get a default MailNotifier, which always uses SMTP # to localhost and uses a dummy fromaddr of "buildbot". log.msg("worker-missing msg using default MailNotifier") st = MailNotifier("buildbot") # now construct the mail m = Message() m.set_payload(text) m['Date'] = formatdate(localtime=True) m['Subject'] = subject m['From'] = st.fromaddr recipients = self.notify_on_missing m['To'] = ", ".join(recipients) d = st.sendMessage(m, recipients) # return the Deferred for testing purposes return d def _gracefulChanged(self, graceful): """This is called when our graceful shutdown setting changes""" self.maybeShutdown() @defer.inlineCallbacks def shutdown(self): """Shutdown the worker""" if not self.conn: log.msg("no remote; worker is already shut down") return yield self.conn.remoteShutdown() def maybeShutdown(self): """Shut down this worker if it has been asked to shut down gracefully, and has no active builders.""" if not self.worker_status.getGraceful(): return active_builders = [sb for sb in itervalues(self.workerforbuilders) if sb.isBusy()] if active_builders: return d = self.shutdown() d.addErrback(log.err, 'error while shutting down worker') def _pauseChanged(self, paused): if paused is True: self.botmaster.master.status.workerPaused(self.name) else: self.botmaster.master.status.workerUnpaused(self.name) def pause(self): """Stop running new builds on the worker.""" self.worker_status.setPaused(True) def unpause(self): """Restart running new builds on the worker.""" self.worker_status.setPaused(False) self.botmaster.maybeStartBuildsForWorker(self.name) def isPaused(self): return self.worker_status.isPaused()
class C(object): @property def workername(self): return "name" deprecatedWorkerClassProperty(locals(), workername)