Exemplo n.º 1
0
 def submit(self, fn, *args, **kwargs):
     """Submit work to be executed and capture statistics."""
     if self._start_before_submit:
         started_at = _utils.now()
     fut = self._submit_func(fn, *args, **kwargs)
     if not self._start_before_submit:
         started_at = _utils.now()
     fut.add_done_callback(functools.partial(self._capture_stats,
                                             started_at))
     return fut
Exemplo n.º 2
0
 def submit(self, fn, *args, **kwargs):
     """Submit work to be executed and capture statistics."""
     if self._start_before_submit:
         started_at = _utils.now()
     fut = self._submit_func(fn, *args, **kwargs)
     if not self._start_before_submit:
         started_at = _utils.now()
     fut.add_done_callback(
         functools.partial(self._capture_stats, started_at))
     return fut
Exemplo n.º 3
0
    def _capture_stats(self, started_at, fut):
        """Capture statistics

        :param started_at: when the activity the future has performed
                           was started at
        :param fut: future object
        """
        # If time somehow goes backwards, make sure we cap it at 0.0 instead
        # of having negative elapsed time...
        elapsed = max(0.0, _utils.now() - started_at)
        with self._stats_lock:
            # Use a new collection and lock so that all mutations are seen as
            # atomic and not overlapping and corrupting with other
            # mutations (the clone ensures that others reading the current
            # values will not see a mutated/corrupted one). Since futures may
            # be completed by different threads we need to be extra careful to
            # gather this data in a way that is thread-safe...
            (failures, executed, runtime,
             cancelled) = (self._stats.failures, self._stats.executed,
                           self._stats.runtime, self._stats.cancelled)
            if fut.cancelled():
                cancelled += 1
            else:
                executed += 1
                if fut.exception() is not None:
                    failures += 1
                runtime += elapsed
            self._stats = ExecutorStatistics(failures=failures,
                                             executed=executed,
                                             runtime=runtime,
                                             cancelled=cancelled)
Exemplo n.º 4
0
    def _capture_stats(self, started_at, fut):
        """Capture statistics

        :param started_at: when the activity the future has performed
                           was started at
        :param fut: future object
        """
        # If time somehow goes backwards, make sure we cap it at 0.0 instead
        # of having negative elapsed time...
        elapsed = max(0.0, _utils.now() - started_at)
        with self._stats_lock:
            # Use a new collection and lock so that all mutations are seen as
            # atomic and not overlapping and corrupting with other
            # mutations (the clone ensures that others reading the current
            # values will not see a mutated/corrupted one). Since futures may
            # be completed by different threads we need to be extra careful to
            # gather this data in a way that is thread-safe...
            (failures, executed, runtime, cancelled) = (self._stats.failures,
                                                        self._stats.executed,
                                                        self._stats.runtime,
                                                        self._stats.cancelled)
            if fut.cancelled():
                cancelled += 1
            else:
                executed += 1
                if fut.exception() is not None:
                    failures += 1
                runtime += elapsed
            self._stats = ExecutorStatistics(failures=failures,
                                             executed=executed,
                                             runtime=runtime,
                                             cancelled=cancelled)
Exemplo n.º 5
0
def _build(callables):
    schedule = _Schedule()
    now = None
    immediates = []
    # Reverse order is used since these are later popped off (and to
    # ensure the popping order is first -> last we need to append them
    # in the opposite ordering last -> first).
    for i, (cb, args, kwargs) in _utils.reverse_enumerate(callables):
        if cb._periodic_run_immediately:
            immediates.append(i)
        else:
            if now is None:
                now = _utils.now()
            schedule.push_next(cb, i, now=now)
    return immediates, schedule
Exemplo n.º 6
0
 def push_next(self, cb, index, now=None):
     if now is None:
         now = _utils.now()
     next_run = now + cb._periodic_spacing
     self.push(next_run, index)
Exemplo n.º 7
0
    def _run(self, executor):
        """Main worker run loop."""

        def _on_done(kind, cb, index, fut, now=None):
            try:
                # NOTE(harlowja): accessing the result will cause it to
                # raise (so that we can log if it had/has failed).
                _r = fut.result()  # noqa
            except Exception:
                how_often = cb._periodic_spacing
                self._log.exception("Failed to call %s '%r' (it runs every"
                                    " %0.2f seconds)", kind, cb, how_often)
            finally:
                with self._waiter:
                    self._schedule.push_next(cb, index, now=now)
                    self._waiter.notify_all()

        while not self._tombstone.is_set():
            if self._immediates:
                # Run & schedule its next execution.
                try:
                    index = self._immediates.pop()
                except IndexError:
                    pass
                else:
                    cb, args, kwargs = self._callables[index]
                    now = _utils.now()
                    fut = executor.submit(cb, *args, **kwargs)
                    # Note that we are providing the time that it started
                    # at, so this implies that the rescheduling time will
                    # *not* take into account how long it took to actually
                    # run the callback... (for better or worse).
                    fut.add_done_callback(functools.partial(_on_done,
                                                            'immediate',
                                                            cb, index,
                                                            now=now))
            else:
                # Figure out when we should run next (by selecting the
                # minimum item from the heap, where the minimum should be
                # the callable that needs to run next and has the lowest
                # next desired run time).
                with self._waiter:
                    while (not self._schedule and
                           not self._tombstone.is_set()):
                        self._waiter.wait(self.MAX_LOOP_IDLE)
                    if self._tombstone.is_set():
                        break
                    now = _utils.now()
                    next_run, index = self._schedule.pop()
                    when_next = next_run - now
                    if when_next <= 0:
                        # Run & schedule its next execution.
                        cb, args, kwargs = self._callables[index]
                        fut = executor.submit(cb, *args, **kwargs)
                        # Note that we are providing the time that it started
                        # at, so this implies that the rescheduling time will
                        # *not* take into account how long it took to actually
                        # run the callback... (for better or worse).
                        fut.add_done_callback(functools.partial(_on_done,
                                                                'periodic',
                                                                cb, index,
                                                                now=now))
                    else:
                        # Gotta wait...
                        self._schedule.push(next_run, index)
                        when_next = min(when_next, self.MAX_LOOP_IDLE)
                        self._waiter.wait(when_next)