def get_context(loop: asyncio.AbstractEventLoop = None) -> Context:
loop = loop or asyncio.get_event_loop()
if loop.is_closed():
raise RuntimeError("event loop is closed")
return Context._EVENT_OBJECTS[loop]
def event_loop_exception_handler(loop: asyncio.AbstractEventLoop,
context: Dict[str, Any]) -> None:
# Ignore dropped tasks in test.
if loop.is_closed() and isinstance(context.get('task'), asyncio.Task):
return
loop.default_exception_handler(context)
def _do_waitpid(
self,
loop: asyncio.AbstractEventLoop,
expected_pid: int,
callback: _Callback,
args: List[Any],
) -> None:
assert expected_pid > 0
try:
pid, status = os.waitpid(expected_pid, 0)
except ChildProcessError:
# The child process is already reaped
# (may happen if waitpid() is called elsewhere).
pid = expected_pid
returncode = 255
logger.warning(
"Unknown child process pid %d, will report returncode 255",
pid)
else:
returncode = _compute_returncode(status)
if loop.get_debug():
logger.debug("process %s exited with returncode %s",
expected_pid, returncode)
if loop.is_closed():
logger.warning("Loop %r that handles pid %r is closed", loop,
pid)
else:
loop.call_soon_threadsafe(callback, pid, returncode, *args)
self._threads.pop(expected_pid)
def _loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
if not loop.is_running() or loop.is_closed():
loop.run_forever()
pending = asyncio.all_tasks(loop=loop)
if pending:
loop.run_until_complete(asyncio.gather(*pending))
def graceful_shutdown(loop: asyncio.AbstractEventLoop = None):
tasks = asyncio.gather(*asyncio.Task.all_tasks(loop=loop), loop=loop, return_exceptions=True)
tasks.add_done_callback(lambda t: loop.stop())
tasks.cancel()
while not tasks.done() and not loop.is_closed():
loop.run_forever()
tasks.exception()
def _thread_flusher(handler: logging.handlers.MemoryHandler,
flush_interval: Union[float, int],
loop: asyncio.AbstractEventLoop):
while not loop.is_closed():
if handler.buffer:
with suppress(Exception):
handler.flush()
time.sleep(flush_interval)
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None:
if loop.is_closed():
return
try:
loop.stop()
cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
finally:
loop.close()
def teardown_test_loop(loop: asyncio.AbstractEventLoop,
fast: bool = False) -> None:
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None:
"""Teardown and cleanup an event_loop created by setup_test_loop."""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None:
if loop.is_closed():
return
try:
cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
except Exception as exc:
log.warning(f"Error<{type(exc).__name__}> was raised. {exc}")
finally:
loop.close()
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None:
if loop.is_closed():
return
try:
loop.stop()
cancel_all_tasks(loop)
loop.run_until_complete(loop.shutdown_asyncgens())
except RuntimeError as error:
log.warning("Error while shutting loop down: %s", str(error))
finally:
loop.close()
def teardown_test_loop(loop: asyncio.AbstractEventLoop,
fast: bool=False) -> None:
"""Teardown and cleanup an event_loop created
by setup_test_loop.
"""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def __init__(self, *, loop: asyncio.AbstractEventLoop, executor_factory):
"""
The event loop for the program should be launched in the root thread,
preferably early in the application launch.
Whether the WorkflowManager uses it directly,
it is useful to require the client to provide the event loop,
if for no other reason than to ensure that one exists.
Args:
loop: event loop, such as from asyncio.new_event_loop()
executor_factory: Implementation-specific callable to get a run time work
manager.
"""
# We are moving towards a composed rather than a derived WorkflowManager Context.
# Note that we can require the super().__init__() to be called in derived classes,
# so it is not non-sensical for an abc.ABC to have an __init__ method.
if not isinstance(loop, asyncio.AbstractEventLoop):
raise TypeError(
'Workflow manager requires an event loop object compatible with '
'asyncio.AbstractEventLoop.')
if loop.is_closed():
raise ProtocolError(
'Event loop does not appear to be ready to use.')
logger.debug(
f'{repr(self)} acquired event loop {repr(loop)} at loop time '
f'{loop.time()}.')
self._asyncio_event_loop = loop
if not callable(executor_factory):
raise TypeError('*executor_factory* argument must be a callable.')
self._executor_factory = executor_factory
# Basic Context implementation details
# TODO: Tasks should only writable within a WorkflowEditor context.
self.tasks = TaskMap() # Map UIDs to task Futures.
self._dispatcher: typing.Union[weakref.ref, None] = None
self._dispatcher_lock = asyncio.Lock()
self._event_hooks: typing.Mapping[
str, typing.MutableSet[AddItemCallback]] = {
'add_item': set()
}
def _loop_mgr(loop: asyncio.AbstractEventLoop) -> None:
"""起一个线程执行事件循环的`run_forever`方法.
[Start up a thread for running the eventloop's `run_forever` method.]
当它被终止时会清理未完结的协程,但不会关闭事件循环
[When it shutdown, all the coroutines will be closed.but the eventloop will not close.]
Params:
loop (asyncio.AbstractEventLoop) : - 事件循环[the eventloop]
"""
if loop.is_closed():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_forever()
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
def _thread_flusher(handler: logging.handlers.MemoryHandler,
flush_interval: Union[float, int],
loop: asyncio.AbstractEventLoop):
def has_no_target():
return True
def has_target():
return bool(handler.target)
is_target = has_no_target
if isinstance(handler, logging.handlers.MemoryHandler):
is_target = has_target
while not loop.is_closed() and is_target():
with suppress(Exception):
if handler.buffer:
handler.flush()
time.sleep(flush_interval)
async def _start(
self,
strategy: Callable[["RecurringCallback"], Awaitable[Union[int,
float]], ],
loop: asyncio.AbstractEventLoop,
*,
shield: bool = False,
suppress_exceptions: ExceptionsType = ()) -> None:
runner: Callable[..., Awaitable[Any]]
while True:
if loop.is_closed():
return
runner = self._exec
if shield:
runner = utils.shield(self._exec)
try:
delay: Union[int, float] = await strategy(self)
if not isinstance(delay, (int, float)):
log.warning(
"Strategy %r returns wrong delay %r. Stopping.",
strategy,
delay,
)
return
if delay < 0:
log.warning(
"Strategy %r returns negative delay %r. "
"Zero delay will be used.",
strategy,
delay,
)
delay = 0
except StrategySkip as e:
await asyncio.sleep(e.delay)
continue
except StrategyException:
return
await asyncio.sleep(delay)
future = loop.create_future()
task: asyncio.Task = asyncio.ensure_future(
runner(
loop=loop,
suppress_exceptions=suppress_exceptions,
), )
def on_done(task: asyncio.Task) -> None:
if future.done():
return
future.set_result(task)
task.add_done_callback(on_done)
try:
await future
except asyncio.CancelledError:
if not task.done():
task.cancel()
await asyncio.gather(task, return_exceptions=True)
raise
def run(
func: Optional[Awaitable[None]] = None,
*,
finalize: Optional[Awaitable[None]] = None,
loop: AbstractEventLoop = None,
):
""" Configure the event loop to react to signals and exceptions then
run the provided coroutine and loop forever.
Shutdown the event loop when a signal or exception is received or the
supplied function explicitly requests the loop to stop.
This function provides some of the common boilerplate typically needed
when running asyncio applications. It registers signal handlers that
listen for SIGINT and SIGTERM that will stop the event loop and trigger
application shutdown actions. It registers a global exception handler
that will stop the loop and trigger application shutdown actions. This
helps catch a common problem that occurs in asyncio applications in
which an exception occurs in a task that was spun off but is not
reported until the event loop is stopped. This approach allows users
to be notified about these issues as soon as possible.
:param func: A coroutine to run before looping forever. This coroutine
is typically the "main" coroutine from which all other work is spawned.
The event loop will continue to run after the supplied coroutine
completes.
:param finalize: An optional coroutine to run when shutting down. Use this
to perform any graceful cleanup activities such as finalising log files,
disconnecting from services such as databases, etc.
:param loop: An optional event loop to run. If not supplied the default
event loop is used (i.e., whatever ``asyncio.get_event_loop()`` returns.
"""
logger.debug("Application runner starting")
if func:
if not (inspect.isawaitable(func)
or inspect.iscoroutinefunction(func)):
raise Exception("func must be a coroutine or a coroutine function "
f"that takes no arguments, got {func}")
if finalize:
if not (inspect.isawaitable(finalize)
or inspect.iscoroutinefunction(finalize)):
raise Exception(
"finalize must be a coroutine or a coroutine function "
f"that takes no arguments, got {finalize}")
# Use a supplied loop or the default event loop. If the loop is closed
# (which can happen in unit tests) then create a new event loop.
loop = loop or asyncio.get_event_loop()
if loop.is_closed():
loop = asyncio.new_event_loop()
def signal_handler(loop, sig):
logger.info(f"Caught {sig.name}, stopping.")
loop.call_soon(loop.stop)
loop.add_signal_handler(SIGINT, signal_handler, loop, SIGINT)
loop.add_signal_handler(SIGTERM, signal_handler, loop, SIGTERM)
def exception_handler(loop, context):
exc_msg = context["message"]
exc = context["exception"]
logger.exception(f"Caught exception: {exc_msg}", exc_info=exc)
loop.call_soon(loop.stop)
loop.set_exception_handler(exception_handler)
try:
if func:
if inspect.iscoroutinefunction(func):
func = func() # type: ignore
assert func is not None
loop.create_task(func)
loop.run_forever()
finally:
logger.debug("Application shutdown sequence starting")
if finalize:
if inspect.iscoroutinefunction(finalize):
finalize = finalize() # type: ignore
assert finalize is not None
loop.run_until_complete(finalize)
# Shutdown any outstanding tasks that are left running
pending_tasks = all_tasks(loop=loop)
if pending_tasks:
logger.debug(f"Cancelling {len(pending_tasks)} pending tasks.")
for task in pending_tasks:
logger.debug(f"Cancelling task: {task}")
task.cancel()
try:
loop.run_until_complete(asyncio.gather(*pending_tasks))
except asyncio.CancelledError:
pass
loop.run_until_complete(loop.shutdown_asyncgens())
logger.debug("Application shutdown sequence complete")
loop.close()
logger.debug("Application runner stopped")
