async def background_asyncio_service(
service: ServiceAPI,
loop: asyncio.AbstractEventLoop = None) -> AsyncIterator[ManagerAPI]:
"""
Run a service in the background.
The service is running within the context block and will be properly
cleaned up upon exiting the context block.
"""
manager = AsyncioManager(service, loop=loop)
task = asyncio.ensure_future(manager.run(), loop=loop)
try:
async with cleanup_tasks(task):
await manager.wait_started()
try:
yield manager
finally:
await manager.stop()
finally:
if manager.did_error:
# TODO: better place for this.
raise MultiError(
tuple(
exc_value.with_traceback(exc_tb)
for _, exc_value, exc_tb in manager._errors))
async def _exit(
self,
exc_type: Optional[Type[BaseException]],
exc_value: Optional[BaseException],
traceback: Optional[TracebackType],
) -> None:
if not self.cms_to_exit:
return
# don't use gather() to ensure that we wait for all __aexit__s
# to complete even if one of them raises
done, _pending = await asyncio.wait([
cm.__aexit__(exc_type, exc_value, traceback)
for cm in self.cms_to_exit
])
# This is to ensure we re-raise any exceptions our coroutines raise when exiting.
errors: List[Tuple[Type[BaseException], BaseException,
TracebackType]] = []
for d in done:
try:
d.result()
except BaseException:
errors.append(sys.exc_info())
if errors:
raise MultiError(
tuple(
exc_value.with_traceback(exc_tb)
for _, exc_value, exc_tb in errors))
async def cancel_tasks(tasks: Iterable[asyncio.Task[Any]]) -> None:
"""
Cancel and await for the given tasks, ignoring any asyncio.CancelledErrors.
"""
for task in tasks:
task.cancel()
errors: List[BaseException] = []
# Wait for all tasks in parallel so if any of them catches CancelledError and performs a
# slow cleanup the othders don't have to wait for it. The timeout is long as our component
# tasks can do a lot of stuff during their cleanup.
done, pending = await asyncio.wait(tasks, timeout=5)
if pending:
errors.append(
asyncio.TimeoutError(
"Tasks never returned after being cancelled: %s", pending))
# We use future as the variable name here because that's what asyncio.wait returns above.
for future in done:
# future.exception() will raise a CancelledError if the future was cancelled by us above, so
# we must suppress that here.
with contextlib.suppress(asyncio.CancelledError):
if future.exception():
errors.append(future.exception())
if errors:
raise MultiError(errors)
async def cancel_pending_tasks(*tasks: asyncio.Task[Any],
timeout: int) -> AsyncIterator[None]:
"""
Cancel and await for all of the given tasks that are still pending, in no specific order.
If all cancelled tasks have not completed after the given timeout, raise a TimeoutError.
Ignores any asyncio.CancelledErrors.
"""
try:
yield
finally:
logger = get_logger('p2p.asyncio_utils.cancel_pending_tasks')
cancelled: List[asyncio.Task[Any]] = []
for task in tasks:
if not task.done():
task.cancel()
cancelled.append(task)
# It'd save us one indentation level on the block of code below if we had an early return
# in case there are no cancelled tasks, but it turns out an early return inside a finally:
# block silently cancels an active exception being raised, so we use an if/else to avoid
# having to check if there is an active exception and re-raising it.
if cancelled:
logger.debug("Cancelled tasks %s, now waiting for them to return",
task)
errors: List[BaseException] = []
# Wait for all tasks in parallel so if any of them catches CancelledError and performs a
# slow cleanup the othders don't have to wait for it.
done, pending = await asyncio.wait(cancelled, timeout=timeout)
if pending:
errors.append(
asyncio.TimeoutError(
"Tasks never returned after being cancelled: %s",
pending))
# We use future as the variable name here because that's what asyncio.wait returns
# above.
for future in done:
# future.exception() will raise a CancelledError if the future was cancelled by us
# above, so we must suppress that here.
with contextlib.suppress(asyncio.CancelledError):
if future.exception():
errors.append(future.exception())
if errors:
raise MultiError(errors)
else:
logger.debug("No pending tasks in %s, returning", task)
async def cancel_futures(futures: Iterable[asyncio.Future[None]]) -> None:
"""
Cancel and await for the given futures, ignoring any asyncio.CancelledErrors.
"""
for fut in futures:
fut.cancel()
errors: List[BaseException] = []
# Wait for all futures in parallel so if any of them catches CancelledError and performs a
# slow cleanup the othders don't have to wait for it. The timeout is long as our component
# tasks can do a lot of stuff during their cleanup.
done, pending = await asyncio.wait(futures, timeout=5)
if pending:
errors.append(
asyncio.TimeoutError(
"Tasks never returned after being cancelled: %s", pending))
for task in done:
with contextlib.suppress(asyncio.CancelledError):
if task.exception():
errors.append(task.exception())
if errors:
raise MultiError(errors)
async def run(self) -> None:
if self._run_lock.locked():
raise LifecycleError(
"Cannot run a service with the run lock already engaged. Already started?"
)
elif self.is_started:
raise LifecycleError(
"Cannot run a service which is already started.")
try:
async with self._run_lock:
handle_cancelled_task = asyncio.ensure_future(
self._handle_cancelled(), loop=self._loop)
async with cleanup_tasks(handle_cancelled_task):
self._started.set()
self.run_task(self._service.run)
# This is hack to get the task stats correct. We don't want to
# count the `Service.run` method as a task. This is still
# imperfect as it will still count as a completed task when it
# finishes.
self._total_task_count = 0
await self._wait_all_tasks_done()
finally:
self._finished.set()
self.logger.debug("%s: finished", self)
# Above we rely on run_task() and handle_cancelled() to run the
# service/tasks and swallow/collect exceptions so that they can be
# reported all together here.
if self.did_error:
raise MultiError(
tuple(
exc_value.with_traceback(exc_tb)
for _, exc_value, exc_tb in self._errors))
async def raise_multi():
raise MultiError([ClickException("err1"), ClickException("err2")])