def submit(self, fn, *args, **kwargs):
"""Submit work to be executed and capture statistics."""
if self._start_before_submit:
started_at = _utils.now()
fut = self._submit_func(fn, *args, **kwargs)
if not self._start_before_submit:
started_at = _utils.now()
fut.add_done_callback(functools.partial(self._capture_stats,
started_at))
return fut
def submit(self, fn, *args, **kwargs):
"""Submit work to be executed and capture statistics."""
if self._start_before_submit:
started_at = _utils.now()
fut = self._submit_func(fn, *args, **kwargs)
if not self._start_before_submit:
started_at = _utils.now()
fut.add_done_callback(
functools.partial(self._capture_stats, started_at))
return fut
def _capture_stats(self, started_at, fut):
"""Capture statistics
:param started_at: when the activity the future has performed
was started at
:param fut: future object
"""
# If time somehow goes backwards, make sure we cap it at 0.0 instead
# of having negative elapsed time...
elapsed = max(0.0, _utils.now() - started_at)
with self._stats_lock:
# Use a new collection and lock so that all mutations are seen as
# atomic and not overlapping and corrupting with other
# mutations (the clone ensures that others reading the current
# values will not see a mutated/corrupted one). Since futures may
# be completed by different threads we need to be extra careful to
# gather this data in a way that is thread-safe...
(failures, executed, runtime,
cancelled) = (self._stats.failures, self._stats.executed,
self._stats.runtime, self._stats.cancelled)
if fut.cancelled():
cancelled += 1
else:
executed += 1
if fut.exception() is not None:
failures += 1
runtime += elapsed
self._stats = ExecutorStatistics(failures=failures,
executed=executed,
runtime=runtime,
cancelled=cancelled)
def _capture_stats(self, started_at, fut):
"""Capture statistics
:param started_at: when the activity the future has performed
was started at
:param fut: future object
"""
# If time somehow goes backwards, make sure we cap it at 0.0 instead
# of having negative elapsed time...
elapsed = max(0.0, _utils.now() - started_at)
with self._stats_lock:
# Use a new collection and lock so that all mutations are seen as
# atomic and not overlapping and corrupting with other
# mutations (the clone ensures that others reading the current
# values will not see a mutated/corrupted one). Since futures may
# be completed by different threads we need to be extra careful to
# gather this data in a way that is thread-safe...
(failures, executed, runtime, cancelled) = (self._stats.failures,
self._stats.executed,
self._stats.runtime,
self._stats.cancelled)
if fut.cancelled():
cancelled += 1
else:
executed += 1
if fut.exception() is not None:
failures += 1
runtime += elapsed
self._stats = ExecutorStatistics(failures=failures,
executed=executed,
runtime=runtime,
cancelled=cancelled)
def _build(callables):
schedule = _Schedule()
now = None
immediates = []
# Reverse order is used since these are later popped off (and to
# ensure the popping order is first -> last we need to append them
# in the opposite ordering last -> first).
for i, (cb, args, kwargs) in _utils.reverse_enumerate(callables):
if cb._periodic_run_immediately:
immediates.append(i)
else:
if now is None:
now = _utils.now()
schedule.push_next(cb, i, now=now)
return immediates, schedule
def push_next(self, cb, index, now=None):
if now is None:
now = _utils.now()
next_run = now + cb._periodic_spacing
self.push(next_run, index)
def _run(self, executor):
"""Main worker run loop."""
def _on_done(kind, cb, index, fut, now=None):
try:
# NOTE(harlowja): accessing the result will cause it to
# raise (so that we can log if it had/has failed).
_r = fut.result() # noqa
except Exception:
how_often = cb._periodic_spacing
self._log.exception("Failed to call %s '%r' (it runs every"
" %0.2f seconds)", kind, cb, how_often)
finally:
with self._waiter:
self._schedule.push_next(cb, index, now=now)
self._waiter.notify_all()
while not self._tombstone.is_set():
if self._immediates:
# Run & schedule its next execution.
try:
index = self._immediates.pop()
except IndexError:
pass
else:
cb, args, kwargs = self._callables[index]
now = _utils.now()
fut = executor.submit(cb, *args, **kwargs)
# Note that we are providing the time that it started
# at, so this implies that the rescheduling time will
# *not* take into account how long it took to actually
# run the callback... (for better or worse).
fut.add_done_callback(functools.partial(_on_done,
'immediate',
cb, index,
now=now))
else:
# Figure out when we should run next (by selecting the
# minimum item from the heap, where the minimum should be
# the callable that needs to run next and has the lowest
# next desired run time).
with self._waiter:
while (not self._schedule and
not self._tombstone.is_set()):
self._waiter.wait(self.MAX_LOOP_IDLE)
if self._tombstone.is_set():
break
now = _utils.now()
next_run, index = self._schedule.pop()
when_next = next_run - now
if when_next <= 0:
# Run & schedule its next execution.
cb, args, kwargs = self._callables[index]
fut = executor.submit(cb, *args, **kwargs)
# Note that we are providing the time that it started
# at, so this implies that the rescheduling time will
# *not* take into account how long it took to actually
# run the callback... (for better or worse).
fut.add_done_callback(functools.partial(_on_done,
'periodic',
cb, index,
now=now))
else:
# Gotta wait...
self._schedule.push(next_run, index)
when_next = min(when_next, self.MAX_LOOP_IDLE)
self._waiter.wait(when_next)