def submit(self, fn, *args, **kwargs): """Submit work to be executed and capture statistics.""" if self._start_before_submit: started_at = _utils.now() fut = self._submit_func(fn, *args, **kwargs) if not self._start_before_submit: started_at = _utils.now() fut.add_done_callback(functools.partial(self._capture_stats, started_at)) return fut
def submit(self, fn, *args, **kwargs): """Submit work to be executed and capture statistics.""" if self._start_before_submit: started_at = _utils.now() fut = self._submit_func(fn, *args, **kwargs) if not self._start_before_submit: started_at = _utils.now() fut.add_done_callback( functools.partial(self._capture_stats, started_at)) return fut
def _capture_stats(self, started_at, fut): """Capture statistics :param started_at: when the activity the future has performed was started at :param fut: future object """ # If time somehow goes backwards, make sure we cap it at 0.0 instead # of having negative elapsed time... elapsed = max(0.0, _utils.now() - started_at) with self._stats_lock: # Use a new collection and lock so that all mutations are seen as # atomic and not overlapping and corrupting with other # mutations (the clone ensures that others reading the current # values will not see a mutated/corrupted one). Since futures may # be completed by different threads we need to be extra careful to # gather this data in a way that is thread-safe... (failures, executed, runtime, cancelled) = (self._stats.failures, self._stats.executed, self._stats.runtime, self._stats.cancelled) if fut.cancelled(): cancelled += 1 else: executed += 1 if fut.exception() is not None: failures += 1 runtime += elapsed self._stats = ExecutorStatistics(failures=failures, executed=executed, runtime=runtime, cancelled=cancelled)
def _build(callables): schedule = _Schedule() now = None immediates = [] # Reverse order is used since these are later popped off (and to # ensure the popping order is first -> last we need to append them # in the opposite ordering last -> first). for i, (cb, args, kwargs) in _utils.reverse_enumerate(callables): if cb._periodic_run_immediately: immediates.append(i) else: if now is None: now = _utils.now() schedule.push_next(cb, i, now=now) return immediates, schedule
def push_next(self, cb, index, now=None): if now is None: now = _utils.now() next_run = now + cb._periodic_spacing self.push(next_run, index)
def _run(self, executor): """Main worker run loop.""" def _on_done(kind, cb, index, fut, now=None): try: # NOTE(harlowja): accessing the result will cause it to # raise (so that we can log if it had/has failed). _r = fut.result() # noqa except Exception: how_often = cb._periodic_spacing self._log.exception("Failed to call %s '%r' (it runs every" " %0.2f seconds)", kind, cb, how_often) finally: with self._waiter: self._schedule.push_next(cb, index, now=now) self._waiter.notify_all() while not self._tombstone.is_set(): if self._immediates: # Run & schedule its next execution. try: index = self._immediates.pop() except IndexError: pass else: cb, args, kwargs = self._callables[index] now = _utils.now() fut = executor.submit(cb, *args, **kwargs) # Note that we are providing the time that it started # at, so this implies that the rescheduling time will # *not* take into account how long it took to actually # run the callback... (for better or worse). fut.add_done_callback(functools.partial(_on_done, 'immediate', cb, index, now=now)) else: # Figure out when we should run next (by selecting the # minimum item from the heap, where the minimum should be # the callable that needs to run next and has the lowest # next desired run time). with self._waiter: while (not self._schedule and not self._tombstone.is_set()): self._waiter.wait(self.MAX_LOOP_IDLE) if self._tombstone.is_set(): break now = _utils.now() next_run, index = self._schedule.pop() when_next = next_run - now if when_next <= 0: # Run & schedule its next execution. cb, args, kwargs = self._callables[index] fut = executor.submit(cb, *args, **kwargs) # Note that we are providing the time that it started # at, so this implies that the rescheduling time will # *not* take into account how long it took to actually # run the callback... (for better or worse). fut.add_done_callback(functools.partial(_on_done, 'periodic', cb, index, now=now)) else: # Gotta wait... self._schedule.push(next_run, index) when_next = min(when_next, self.MAX_LOOP_IDLE) self._waiter.wait(when_next)