def _worker_done(self, task: asyncio.Task) -> None:
assert task is self._worker_task
if task.cancelled():
self._connection_task.cancel()
elif task.exception():
f = io.StringIO()
task.print_stack(file=f)
self.logger.error(f.getvalue())
now = time.time()
self._worker_task_failure_timestamps.append(time.time())
if len(self._worker_task_failure_timestamps) == 5:
if self._worker_task_failure_timestamps.pop(0) >= now - 10:
self.logger.error(
"Worker task exceeded exception threshold; terminating"
)
self._close("Exception threshold exceeded")
return
self.logger.warning("Restarting worker task")
self._worker_task = self.loop.create_task(self._worker())
self._worker_task.add_done_callback(self._worker_done)
else:
self.logger.debug("Worker task exited gracefully")
return
def task_complete(self, task: asyncio.Task):
if task.cancelled():
return
if task.exception():
task.print_stack()
self.keep_alive = self.loop.create_task(self.rebooter())
self.keep_alive.add_done_callback(self.task_complete)
def _task_done(self, task: Task) -> None:
with self._lock:
coroutine = self._task_map.pop(task)
try:
if task.exception() and self.stack_limit:
task.print_stack(limit=None if self.stack_limit is True else self.stack_limit, file=self.stack_file)
except CancelledError:
coroutine.close()
def __call_async_cb(self, task: asyncio.Task,
callback: Optional[Callable[[Any], None]]) -> None:
if task.exception() is not None:
buf = io.StringIO()
task.print_stack(file=buf)
self.__app.crashWithMessage("Exception in callback",
buf.getvalue())
raise task.exception()
if callback is not None:
callback(task.result())
def __callback_done(self, cmd: str, task: asyncio.Task) -> None:
if task.cancelled():
logger.info("Session %016x: %s was cancelled.", self.id, cmd)
return
exc = task.exception()
if isinstance(exc, ConnectionClosed):
logger.warning("Session %016x: callback connection closed.", self.id)
self.close()
elif exc is not None:
buf = io.StringIO()
task.print_stack(file=buf)
logger.error(
"Session %016x: %s failed with exception: %s\n%s",
self.id, cmd, exc, buf.getvalue())
self.close()
def done_callback(task: asyncio.Task):
if task.exception():
task.print_stack()
def unmute_error(task: asyncio.Task):
if task.exception():
task.print_stack()
else:
print(f"A Unmute has been done sucssefully")
def done_callback(done: asyncio.Task):
pendings.remove(done)
exception = done.exception()
if exception and not done.cancelled():
done.print_stack(file=sys.stderr)
self.logger.error(exception)
