async def finalize_task(task: Task) -> Any:
try:
return await task
finally:
if not task.cancelled():
task.cancel()
def _task_done_callback(self, task_id: t.Hashable,
done_task: asyncio.Task) -> None:
"""
Delete the task and raise its exception if one exists.
If `done_task` and the task associated with `task_id` are different, then the latter
will not be deleted. In this case, a new task was likely rescheduled with the same ID.
"""
self._log.info(
f"Performing done callback for task #{task_id} {id(done_task)}.")
scheduled_task = self._scheduled_tasks.get(task_id)
if scheduled_task and done_task is scheduled_task:
# A task for the ID exists and is the same as the done task.
# Since this is the done callback, the task is already done so no need to cancel it.
self._log.info(f"Deleting task #{task_id} {id(done_task)}.")
del self._scheduled_tasks[task_id]
elif scheduled_task:
# A new task was likely rescheduled with the same ID.
self._log.debug(
f"The scheduled task #{task_id} {id(scheduled_task)} "
f"and the done task {id(done_task)} differ.")
elif not done_task.cancelled():
self._log.warning(
f"Task #{task_id} not found while handling task {id(done_task)}! "
f"A task somehow got unscheduled improperly (i.e. deleted but not cancelled)."
)
with contextlib.suppress(asyncio.CancelledError):
exception = done_task.exception()
# Log the exception if one exists.
if exception:
self._log.error(f"Error in task #{task_id} {id(done_task)}!",
exc_info=exception)
def task_complete(self, task: asyncio.Task):
if task.cancelled():
return
if task.exception():
task.print_stack()
self.keep_alive = self.loop.create_task(self.rebooter())
self.keep_alive.add_done_callback(self.task_complete)
def background_task_callback(task: asyncio.Task) -> None:
"""Check if the finished background task failed to make sure we log errors."""
if task.cancelled():
log.info(f"Background task `{task.get_name()}` was cancelled.")
elif exception := task.exception():
log.error(f"Background task `{task.get_name()}` failed:",
exc_info=exception)
def _worker_done(self, task: asyncio.Task) -> None:
assert task is self._worker_task
if task.cancelled():
self._connection_task.cancel()
elif task.exception():
f = io.StringIO()
task.print_stack(file=f)
self.logger.error(f.getvalue())
now = time.time()
self._worker_task_failure_timestamps.append(time.time())
if len(self._worker_task_failure_timestamps) == 5:
if self._worker_task_failure_timestamps.pop(0) >= now - 10:
self.logger.error(
"Worker task exceeded exception threshold; terminating"
)
self._close("Exception threshold exceeded")
return
self.logger.warning("Restarting worker task")
self._worker_task = self.loop.create_task(self._worker())
self._worker_task.add_done_callback(self._worker_done)
else:
self.logger.debug("Worker task exited gracefully")
return
async def _async_run_long_action(self, long_task: asyncio.Task) -> None:
"""Run a long task while monitoring for stop request."""
async def async_cancel_long_task() -> None:
# Stop long task and wait for it to finish.
long_task.cancel()
with suppress(Exception):
await long_task
# Wait for long task while monitoring for a stop request.
stop_task = self._hass.async_create_task(self._stop.wait())
try:
await asyncio.wait({long_task, stop_task},
return_when=asyncio.FIRST_COMPLETED)
# If our task is cancelled, then cancel long task, too. Note that if long task
# is cancelled otherwise the CancelledError exception will not be raised to
# here due to the call to asyncio.wait(). Rather we'll check for that below.
except asyncio.CancelledError:
await async_cancel_long_task()
raise
finally:
stop_task.cancel()
if long_task.cancelled():
raise asyncio.CancelledError
if long_task.done():
# Propagate any exceptions that occurred.
long_task.result()
else:
# Stopped before long task completed, so cancel it.
await async_cancel_long_task()
async def cancel(task: asyncio.Task):
if not (task.done() or task.cancelled()):
try:
task.cancel()
await task
except asyncio.CancelledError:
pass
def _on_done(self, task: asyncio.Task) -> None:
if not task.cancelled():
try:
task.result()
except Exception:
log.exception("Async-iterator task ended with error")
pass # TODO: log it or something...
def _task_done(self, task: asyncio.Task) -> None:
self._tasks.remove(task)
if task.cancelled():
return
exc = task.exception()
if exc is not None:
raise exc
def task_result_callback(task: asyncio.Task) -> None:
"""Check if our sleep task ended in an expected way."""
name = task.get_name()
if task.cancelled():
log.debug(f"{name} task callback: the task was cancelled")
return
if exc := task.exception():
log.exception(f"{name} task callback: the task failed!", exc_info=exc)
def _task_done(self, actx: ActivityContext, task: asyncio.Task) -> None:
actx.tasks.remove(task)
if task.cancelled():
return
exc = task.exception()
if exc is not None:
msg = f"Task failed. {exc.__class__.__name__}: {exc}"
self.set_status(("error message", msg))
return
def _default_cb(self, t: asyncio.Task) -> None:
"""A simple callback for tasks to log & call exc handler."""
if not t.cancelled():
exc = t.exception()
if exc and not isinstance(exc, (SystemExit, KeyboardInterrupt)):
self.exceptions += 1
loop = asyncio.get_running_loop()
loop.default_exception_handler({'exception': exc})
def _recv_task_finalizer(self, task: Task):
self._logger.debug("%s Recv task closed" % repr(self))
self._recv_task = None
# noinspection PyUnresolvedReferences
self._recv_queue._queue.clear(
) # By some reason this method isn't exposed. Not sure if it is a good idea.
if not task.cancelled() and task.exception():
self._logger.error("%s: RECV err %s" %
(repr(self), task.exception()))
def track_done(fut: asyncio.Task):
if fut.cancelled():
return # todo sth to do in this case?
trackExc = fut.exception()
if trackExc is not None:
logger.error(
f"Failed tracking job for ips={ips!r} - {trackExc!s}\n{track_task.get_stack()!s}"
)
worker.peer.stop()
def _task_finished(self, task: Task) -> None:
# If a task raised an exception (other than CancelledError), cancel all other tasks
if not task.cancelled() and task.exception() is not None:
if self._status is NurseryStatus.running:
self._status = NurseryStatus.closed
self._host_task.cancel()
for task in self._tasks:
task.cancel()
else:
self._tasks.discard(task)
def _done_callback(future: asyncio.Future, task: asyncio.Task):
if task.cancelled():
future.cancel()
return
exception = task.exception()
if exception is not None:
future.set_exception(exception)
return
future.set_result(task.result())
def _on_done(self, task: asyncio.Task) -> None:
if task in self._tasks:
# Probably task was removed from _cleanup
self._tasks.remove(task)
self._done -= 1
if self._done <= 0:
self._queue.close()
if not task.cancelled():
try:
task.result()
except Exception:
pass # TODO: log exception
def _reraise_if_necessary(task: asyncio.Task) -> None:
try:
if not task.cancelled():
task.result()
except Exception as error: # pylint: disable=broad-except
self.logger.error(
f"Exiting from servo main loop do to error: {error} (task={task})"
)
self.logger.opt(exception=error).trace(
f"Exception raised by task {task}"
)
raise error # Ensure that we surface the error for handling
def _on_reader_done(self, task: asyncio.Task) -> None:
log.debug("Reader exited for %r", self)
if not self._writer_task.done():
self._writer_task.cancel()
if not task.cancelled() and task.exception() is not None:
log.debug("Cancelling cause reader exited abnormally")
self.set_close_reason(
reply_code=500, reply_text="reader unexpected closed",
)
self.create_task(self.close(task.exception()))
def __callback_connected(self, task: asyncio.Task) -> None:
if task.cancelled():
return
assert task.done()
exc = task.exception()
if exc is not None:
logger.error("Session %016x: Failed to connect to callback client: %s", self.id, exc)
self.close()
return
assert self.__callback_stub.connected
self.callback_connected()
def _download_done(
self, fileinfo: FileInfo, statusw: urwid.WidgetPlaceholder, task: asyncio.Task
) -> None:
del self._dl_in_progress[fileinfo.name]
if task.cancelled():
statusw.original_widget = urwid.Text(("error message", " Cancelled "))
return
exc = task.exception()
if exc is not None:
statusw.original_widget = urwid.Text(("error banner", " Failed "))
return
statusw.original_widget = urwid.Text(("success banner", " Done "))
async def stop_supervisor(supervisor_task: asyncio.Task, logger: Logger):
if not supervisor_task.done():
supervisor_task.cancel()
done, _ = await asyncio.wait([supervisor_task])
if not supervisor_task.done():
logger.warning("unable to stop")
else:
if not supervisor_task.cancelled():
exc = supervisor_task.exception()
if exc is not None:
logger.exception('exception thrown on stop: %s',
exc,
exc_info=exc)
def __callback_done(self, cmd: str, task: asyncio.Task) -> None:
if task.cancelled():
logger.info("Session %016x: %s was cancelled.", self.id, cmd)
return
exc = task.exception()
if isinstance(exc, ConnectionClosed):
logger.warning("Session %016x: callback connection closed.", self.id)
self.close()
elif exc is not None:
buf = io.StringIO()
task.print_stack(file=buf)
logger.error(
"Session %016x: %s failed with exception: %s\n%s",
self.id, cmd, exc, buf.getvalue())
self.close()
def async_activate_callback(self, task: asyncio.Task) -> None:
if task.cancelled():
log.debug(_("%s has been stopped due user request"),
task.get_coro())
return
exception = task.exception()
if exception and not isinstance(exception, asyncio.CancelledError):
stack = task.get_stack()
for frame in stack:
log.critical(f"{type(exception).__name__} at {frame}")
log.critical(f"Fatal Error: {str(exception)}")
loop = asyncio.get_running_loop()
loop.stop()
self.on_quit()
async def cancel_and_stop_task(task: Task):
if task.cancelled():
logger.debug('Task already canceled')
return
task.cancel()
try:
await task
except CancelledError:
logger.debug('Task has been canceled')
except Exception as err:
logger.exception(f'Task finished with error ({err}):')
else:
logger.debug('Task finished successfully')
def task_done(self, task_id: Any, event: str, task: asyncio.Task) -> None:
"""
Called when self.callback is an async function/method and it has finished. This
calls self.process_events() so that the C callback is invoked again with the
same arguments (request_id, event) and we can return the actual result.
"""
if task.cancelled():
self.tasks.pop(task_id, None)
return
try:
if event in ("update", "change", "rpc", "oper"):
# The task result will be evaluated in the C callback.
# It will return the result to sysrepo.
self.process_events()
else:
# Sysrepo does not care about the result of the callback.
# This will raise the exception here if any occured in the task
# and will be logged (i.e. not lost).
self.tasks.pop(task_id, None)
task.result()
except Exception:
LOG.exception("failure in task: %r", task)
def get_task_info(task: asyncio.Task) -> Dict:
def _format_frame(f):
keys = ["f_code", "f_lineno"]
return OrderedDict([(k, str(getattr(f, k))) for k in keys])
info = OrderedDict(
txt=str(task),
type=str(type(task)),
done=task.done(),
cancelled=False,
stack=[],
exception=None,
)
if not task.done():
info["stack"] = [_format_frame(x) for x in task.get_stack()]
else:
if task.cancelled():
info["cancelled"] = True
else:
# WARNING: raise if not done or cancelled
exc = task.exception()
info["exception"] = f"{type(exc)}: {str(exc)}" if exc else None
return info
def cb(task: asyncio.Task) -> None:
if current_task:
current_task.remove_done_callback(cb)
if task.cancelled():
self.future.cancel()
def propagate_failure(other: asyncio.Task, t: asyncio.Task):
if t.cancelled():
other.cancel()
elif t.exception() is not None:
other.cancel()
def __instrument_loader_done(self, task: asyncio.Task) -> None:
if task.cancelled():
return
task.result()
