def task_callback(task: asyncio.Task) -> None: with contextlib.suppress(asyncio.CancelledError, asyncio.InvalidStateError): if exc := task.exception(): log.exception("%s raised an Exception", task.get_name(), exc_info=exc)
def is_background_task(task: Task): # Only in Python 3.8+ will we have a get_name function name = ( task.get_name() if hasattr(task, "get_name") else getattr(task, "name", f"Task-{id(task)}") ) return name.endswith("_check_tasks")
def _create_task_info(task: asyncio.Task) -> TaskInfo: coro = task._coro # type: ignore if _task_names is None: name = task.get_name() # type: ignore else: name = _task_names.get(task) return TaskInfo(id(task), _task_parents.get(task), name, coro)
def _done(_task: asyncio.Task): _host = _task.get_name() _res = _task.result() if not len(_res): print(f"IPF device not found: {_host}") return callback(_host, _res[0])
def task_result_callback(task: asyncio.Task) -> None: """Check if our sleep task ended in an expected way.""" name = task.get_name() if task.cancelled(): log.debug(f"{name} task callback: the task was cancelled") return if exc := task.exception(): log.exception(f"{name} task callback: the task failed!", exc_info=exc)
def _create_task_info(task: asyncio.Task) -> TaskInfo: task_state = _task_states.get(task) if task_state is None: name = task.get_name() if _native_task_names else None # type: ignore parent_id = None else: name = task_state.name parent_id = task_state.parent_id return TaskInfo(id(task), parent_id, name, task._coro) # type: ignore
def task_repr(task: asyncio.Task) -> str: """Get a task representation with debug info.""" name = task.get_name() age = getattr(task, "created", "") if age: age = f" (age: {time.time() - age:.0f}s)" client = getattr(task, "client", "") if client: client = f"{human.format_address(client)}: " return f"{client}{name}{age}"
async def notify_done(t: asyncio.Task): tid = t.get_name() logger.debug(f'start task state notifier: {tid}') try: await socket_io.emit(EventTopics.State, { 'id': tid, 'state': TaskState.Ongoing, 'startedAt': int(time.time()), }) ret = await t logger.info(f"task {tid} completed: {ret}") await socket_io.emit( EventTopics.State, { "id": tid, "state": TaskState.Completed, "endedAt": int(time.time()), }, ) except asyncio.CancelledError: logger.warning(f'task {tid} cancelled') await socket_io.emit( EventTopics.State, { "id": tid, "state": TaskState.Cancelled, "endedAt": int(time.time()), }, ) except Exception as e: logger.exception(f"task {tid} failed: {e}") await socket_io.emit( EventTopics.State, { "id": tid, "state": TaskState.Failed, "endedAt": int(time.time()), "msg": str(e), }, ) finally: logger.debug(f'shutdown task state notifier: {tid}')
def handle_exception(task: asyncio.Task) -> None: """Handle any exceptions that occur in tasks. Log all errors and stop the event loop when any exception other than asyncio.CancelledError is raised. Args: task: The task that raised the exception. """ try: task.result() except asyncio.CancelledError: pass except Exception as e: if logging.getLogger().level <= logging.DEBUG: logging.exception("Exception raised by task %s", task.get_name()) else: logging.error("%s", e) context_loop = asyncio.get_event_loop() task_loop = task.get_loop() if task_loop is not context_loop: task_loop.stop()
async def finalize_batch(batch: Iterable[asyncio.Task], previous_finalizer: asyncio.Task) -> None: nonlocal outstanding_batches task_name = asyncio.current_task().get_name() logger.debug("%s: waiting on semaphore. ", task_name) async with outstanding_batch_semaphore: outstanding_batches += 1 try: logger.debug("%s: Semaphore acquired. Outstanding batches %s", task_name, outstanding_batches) logger.debug("awaiting on %s downloads", len(batch)) try: await asyncio.gather(*batch) except Exception as e: logging.error("Exception from download tasks") logging.exception(e) cancel_all(batch) # Await cancelled tasks or we'll get warnings about # coroutines never having been awaited await asyncio.gather(*batch, return_exceptions=True) raise logger.debug("%s: downloads completed", task_name) logger.info("Downloaded batch of %s pages complete", len(batch)) if previous_finalizer: # We await on the previous finalizer to ensure our writes # happen in order logger.debug( "%s awaiting on finalizing of previous batch %s", task_name, previous_finalizer.get_name()) await previous_finalizer logger.debug("%s: Finished awaiting. Ingesting batch", task_name) finally: outstanding_batches -= 1 logger.debug("%s: Semaphore released. Outstanding batches %s", task_name, outstanding_batches)
async def finalize_batch(batch: Iterable[asyncio.Task], previous_finalizer: asyncio.Task) -> None: nonlocal outstanding_batches task_name = asyncio.current_task().get_name() logger.debug("%s: waiting on semaphore. ", task_name) async with outstanding_batch_semaphore: outstanding_batches += 1 try: logger.debug("%s: Semaphore acquired. Outstanding batches %s", task_name, outstanding_batches) logger.debug("awaiting on %s downloads", len(batch)) # Each invocation of download_products_at_ts in the batch will return a # dict with a key 'timestamp' plus one key for each product. If a product # was not available for download, its value will be None. try: downloads_ary = await asyncio.gather(*batch) except Exception as e: logging.error("Exception from download tasks") logging.exception(e) cancel_all(batch) # Await cancelled tasks or we'll get warnings about coroutines never having been awaited await asyncio.gather(*batch, return_exceptions=True) raise logger.debug("%s: downloads completed", task_name) logger.info("Downloaded batch of %s images complete", len(batch)) if previous_finalizer: # We await on the previous finalizer to ensure our writes happen in order logger.debug("%s awaiting on finalizing of previous batch %s", task_name, previous_finalizer.get_name()) await previous_finalizer logger.debug("%s: Finished awaiting. Ingesting batch", task_name) await write_to_destination(destination, downloads_ary, task_name) finally: outstanding_batches -= 1 logger.debug("%s: Semaphore released. Outstanding batches %s", task_name, outstanding_batches)
def _serialize_task(obj: asyncio.Task) -> str: return obj.get_name()
def unregister_task(self, task: asyncio.Task): task_name = task.get_name() self.tasks.pop(task_name)
def register_task(self, task: asyncio.Task): task_name = task.get_name() self.tasks[task_name] = task