Esempio n. 1
0
    def __init__(self, builder, maxSlots):
        AbstractDispatcher.__init__(self, builder, maxSlots)
        Thread.__init__(self)

        self._commitSlots = util.BoundedCounter(0, 1, 1)

        self._cooker = LocalGroupCooker(self._builder)
        self._committer = LocalChangeSetCommitter(self._builder)

        self._order = []
        self._started = False
        self._done = False

        self.daemon = True
Esempio n. 2
0
class LocalDispatcher(AbstractDispatcher, Thread):
    """
    Coordinate multiple local builds.
    """

    statusClass = Status

    _slotdone = (
        JobStatus.JOB_BUILT,
    )

    _completed = (
        JobStatus.JOB_FAILED,
        JobStatus.JOB_COMMITTED,
    )

    def __init__(self, builder, maxSlots):
        AbstractDispatcher.__init__(self, builder, maxSlots)
        Thread.__init__(self)

        self._commitSlots = util.BoundedCounter(0, 1, 1)

        self._cooker = LocalGroupCooker(self._builder)
        self._committer = LocalChangeSetCommitter(self._builder)

        self._order = []
        self._started = False
        self._done = False

        self.daemon = True

    def _getNotStarted(self):
        """
        Get the list of jobs that have not yet been started.
        """

        return [ x for x in self._jobs.itervalues()
                 if x[1] == JobStatus.JOB_NOT_STARTED ]

    def run(self):
        """
        Polling loop to monitor and update job status.
        """

        totalWait = 0
        committed = []
        while not self._done:
            # Start any jobs that are waiting to be started as long as there
            # are available slots.
            notStarted = self._getNotStarted()
            while self._slots and notStarted:
                self._slots -= 1
                trove, status, res = notStarted[0]
                self._cooker.buildGroup((trove, res.data.flavorFilter))

                self._jobs[trove][1] = JobStatus.JOB_BUILDING
                res.setStatus('building')

                notStarted = self._getNotStarted()

            # Check cooker status.
            for trove, csFileName, result in self._cooker.getStatus():
                self._slots += 1
                trove, status, res = self._jobs[trove]

                self._jobs[trove][1] = JobStatus.JOB_BUILT
                res.setStatus('built')

                res.data.buildResults = result
                res.data.csFileName = csFileName

            # Check for cooker errors.
            for trove, error in self._cooker.getErrors():
                self._slots += 1
                self._jobs[trove][1] = JobStatus.JOB_FAILED
                self._jobs[trove][2].setStatus('build failed')
                self._jobs[trove][2].setError(error)

            # Find jobs that are ready to be committed. Jobs must be committed
            # in the order that they were submitted.
            for trove in self._order:
                # continue if this trove has already been committed.
                if self._jobs[trove][1] in self._completed:
                    continue

                # Make sure everything has committed in order.
                if self._jobs[trove][1] not in self._slotdone:
                    break

                # Make sure there are open slots.
                if self._commitSlots == 0:
                    break

                # Commit the job.
                if self._commitSlots > 0:
                    self._commitSlots -= 1
                    self._jobs[trove][2].setStatus('committing')
                    self._jobs[trove][1] = JobStatus.JOB_COMMITTING

                    changeSetFile = self._jobs[trove][2].data.csFileName
                    self._committer.commitChangeSet((trove, changeSetFile))

                assert self._commitSlots > -1

            # Check for commit results.
            for trove, results in self._committer.getStatus():
                self._commitSlots += 1
                self._jobs[trove][1] = JobStatus.JOB_COMMITTED
                res = self._jobs[trove][2]
                res.setStatus('committed')
                res.setResults(res.data.buildResults)
                committed.append(trove)

            # Check for commit errors.
            for trove, error in self._committer.getErrors():
                self._jobs[trove][1] = JobStatus.JOB_FAILED
                self._jobs[trove][2].setStatus('commit failed')
                self._jobs[trove][2].setError(error)

            time.sleep(3)
            totalWait += 3

            # Only log slot status once a minute
            if not totalWait % 60:
                log.info('build slots: %s' % self._slots)
                log.info('commit slots: %s' % self._commitSlots)

    def build(self, troveSpec, flavorFilter=None):
        """
        Add one trove spec to the build queue.
        """

        # Make sure this instance hasn't been marked as done.
        assert not self._done

        # Require at least one trove.
        if not troveSpec:
            return None

        # Wait for an available slot.
        while not self._slots:
            time.sleep(3)

        if troveSpec not in self._jobs:
            status = self.statusClass(troveSpec)
            status.data.flavorFilter = frozenset(flavorFilter)

            self._jobs[troveSpec] = [troveSpec, JobStatus.JOB_NOT_STARTED,
                status]
            self._order.append(troveSpec)
        else:
            log.warn('already building/built requested trove: %s=%s'
                     % (troveSpec[0], troveSpec[1]))
            return self._jobs[troveSpec][2]

        if not self._started:
            self.start()
            self._started = True

        return status

    def done(self):
        """
        Mark this worker as complete. This stops the worker thread. Note that
        once this is called this instance may no longer be used for building.
        """

        self._done = True