def __init__(self, builder, maxSlots): AbstractDispatcher.__init__(self, builder, maxSlots) Thread.__init__(self) self._commitSlots = util.BoundedCounter(0, 1, 1) self._cooker = LocalGroupCooker(self._builder) self._committer = LocalChangeSetCommitter(self._builder) self._order = [] self._started = False self._done = False self.daemon = True
class LocalDispatcher(AbstractDispatcher, Thread): """ Coordinate multiple local builds. """ statusClass = Status _slotdone = ( JobStatus.JOB_BUILT, ) _completed = ( JobStatus.JOB_FAILED, JobStatus.JOB_COMMITTED, ) def __init__(self, builder, maxSlots): AbstractDispatcher.__init__(self, builder, maxSlots) Thread.__init__(self) self._commitSlots = util.BoundedCounter(0, 1, 1) self._cooker = LocalGroupCooker(self._builder) self._committer = LocalChangeSetCommitter(self._builder) self._order = [] self._started = False self._done = False self.daemon = True def _getNotStarted(self): """ Get the list of jobs that have not yet been started. """ return [ x for x in self._jobs.itervalues() if x[1] == JobStatus.JOB_NOT_STARTED ] def run(self): """ Polling loop to monitor and update job status. """ totalWait = 0 committed = [] while not self._done: # Start any jobs that are waiting to be started as long as there # are available slots. notStarted = self._getNotStarted() while self._slots and notStarted: self._slots -= 1 trove, status, res = notStarted[0] self._cooker.buildGroup((trove, res.data.flavorFilter)) self._jobs[trove][1] = JobStatus.JOB_BUILDING res.setStatus('building') notStarted = self._getNotStarted() # Check cooker status. for trove, csFileName, result in self._cooker.getStatus(): self._slots += 1 trove, status, res = self._jobs[trove] self._jobs[trove][1] = JobStatus.JOB_BUILT res.setStatus('built') res.data.buildResults = result res.data.csFileName = csFileName # Check for cooker errors. for trove, error in self._cooker.getErrors(): self._slots += 1 self._jobs[trove][1] = JobStatus.JOB_FAILED self._jobs[trove][2].setStatus('build failed') self._jobs[trove][2].setError(error) # Find jobs that are ready to be committed. Jobs must be committed # in the order that they were submitted. for trove in self._order: # continue if this trove has already been committed. if self._jobs[trove][1] in self._completed: continue # Make sure everything has committed in order. if self._jobs[trove][1] not in self._slotdone: break # Make sure there are open slots. if self._commitSlots == 0: break # Commit the job. if self._commitSlots > 0: self._commitSlots -= 1 self._jobs[trove][2].setStatus('committing') self._jobs[trove][1] = JobStatus.JOB_COMMITTING changeSetFile = self._jobs[trove][2].data.csFileName self._committer.commitChangeSet((trove, changeSetFile)) assert self._commitSlots > -1 # Check for commit results. for trove, results in self._committer.getStatus(): self._commitSlots += 1 self._jobs[trove][1] = JobStatus.JOB_COMMITTED res = self._jobs[trove][2] res.setStatus('committed') res.setResults(res.data.buildResults) committed.append(trove) # Check for commit errors. for trove, error in self._committer.getErrors(): self._jobs[trove][1] = JobStatus.JOB_FAILED self._jobs[trove][2].setStatus('commit failed') self._jobs[trove][2].setError(error) time.sleep(3) totalWait += 3 # Only log slot status once a minute if not totalWait % 60: log.info('build slots: %s' % self._slots) log.info('commit slots: %s' % self._commitSlots) def build(self, troveSpec, flavorFilter=None): """ Add one trove spec to the build queue. """ # Make sure this instance hasn't been marked as done. assert not self._done # Require at least one trove. if not troveSpec: return None # Wait for an available slot. while not self._slots: time.sleep(3) if troveSpec not in self._jobs: status = self.statusClass(troveSpec) status.data.flavorFilter = frozenset(flavorFilter) self._jobs[troveSpec] = [troveSpec, JobStatus.JOB_NOT_STARTED, status] self._order.append(troveSpec) else: log.warn('already building/built requested trove: %s=%s' % (troveSpec[0], troveSpec[1])) return self._jobs[troveSpec][2] if not self._started: self.start() self._started = True return status def done(self): """ Mark this worker as complete. This stops the worker thread. Note that once this is called this instance may no longer be used for building. """ self._done = True