def _loop(event_loop_policy, caplog):
basic_config(
log_format="plain",
stream=caplog.handler.stream,
)
try:
asyncio.set_event_loop_policy(event_loop_policy)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
yield loop
finally:
basic_config(
log_format="plain",
stream=sys.stderr,
)
if loop.is_closed():
return
with suppress(Exception):
loop.run_until_complete(loop.shutdown_asyncgens())
with suppress(Exception):
loop.close()
finally:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def setUp(self):
# Trigger a cleanup of the mapping so we start with a clean slate.
AsyncIOLoop().close()
# If we don't clean up after ourselves other tests may fail on
# py34.
self.orig_policy = asyncio.get_event_loop_policy()
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def set_asyncio_policy():
if os.name == "nt":
# Since IOCP proactor doesn't implement 'create_datagram_endpoint()',
# a Selector event loop should be used instead
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
else:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def test_worker():
try:
result = runner.invoke(cli, ['-s', 'demo.settings', 'worker'])
assert result.exit_code == 0, result.output
assert 'running worker...' in result.output
assert 'running demo worker function, ans: 256' in result.output
finally:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def event_loop(loop_type):
if loop_type == 'default':
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
elif loop_type == 'uvloop':
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop_policy().new_event_loop()
yield loop
gc.collect()
loop.close()
def get_default_policy():
if sys.platform == 'win32':
return asyncio.WindowsProactorEventLoopPolicy()
# TODO: evaluate usage of uvloop
# with contextlib.suppress(ModuleNotFoundError):
# import uvloop
# return uvloop.EventLoopPolicy()
return asyncio.DefaultEventLoopPolicy()
def _hook_uvloop():
"""
Attempts wire uvloop.
"""
import asyncio
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
except:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def __init__(self, application=None):
self._default_loop = None
self._application = application
self._watcher_lock = threading.Lock()
self._watcher = None
self._policy = asyncio.DefaultEventLoopPolicy()
self._policy.new_event_loop = self.new_event_loop
self.get_event_loop = self._policy.get_event_loop
self.set_event_loop = self._policy.set_event_loop
def selector_loop() -> None:
if sys.version_info >= (3, 8):
policy = asyncio.WindowsSelectorEventLoopPolicy()
else:
policy = asyncio.DefaultEventLoopPolicy()
asyncio.set_event_loop_policy(policy)
with loop_context(policy.new_event_loop) as _loop:
asyncio.set_event_loop(_loop)
yield _loop
def test_asyncio_add_watcher_SIGCHLD_nop(self):
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
asyncio.get_event_loop_policy().get_child_watcher()
try:
loop = uvloop.new_event_loop()
with self.assertWarnsRegex(
RuntimeWarning,
"asyncio is trying to install its ChildWatcher"):
asyncio.set_event_loop(loop)
finally:
asyncio.set_event_loop(None)
loop.close()
def selector_loop():
if sys.version_info < (3, 7):
policy = asyncio.get_event_loop_policy()
policy._loop_factory = asyncio.SelectorEventLoop # type: ignore
else:
if sys.version_info >= (3, 8):
policy = asyncio.WindowsSelectorEventLoopPolicy() # type: ignore
else:
policy = asyncio.DefaultEventLoopPolicy()
asyncio.set_event_loop_policy(policy)
with loop_context(policy.new_event_loop) as _loop:
asyncio.set_event_loop(_loop)
yield _loop
def main():
n = get_int_arg()
if n <= 0:
print(
"Function takes exactly 1 argument which should be a positive integer. Example use: python3? "
"sync_requests.py 12")
return
if sys.platform == 'win32':
asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
else:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
asyncio.run(async_get_n(n))
def execute(self, command, completeHook):
while self.managerStatus.prog(command.getRole()) != Status.STOPPED:
runId = uuid.uuid4().hex
self.currentDir = os.path.dirname(os.path.realpath(__file__))
self.runDir = os.path.join(self.currentDir + '/' +
self.metadata.name() + '/' +
self.output_dir + '/' + runId)
if not os.path.exists(self.runDir):
os.makedirs(self.runDir)
# run generator to create input files
self.runGenerator(runId, command, self.generator.completeHook)
# run fuzzer based on the generated input files
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
asyncio.run(self.runFuzzer(runId, command, completeHook))
def _loop(event_loop_policy):
try:
asyncio.set_event_loop_policy(event_loop_policy)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
yield loop
finally:
if loop.is_closed():
return
with suppress(Exception):
loop.run_until_complete(loop.shutdown_asyncgens())
with suppress(Exception):
loop.close()
finally:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def event_loop() -> Iterator[asyncio.AbstractEventLoop]:
"""
On py38 win32 WindowsProactorEventLoopPolicy is default event loop
This loop doesn't implement some methods that aiopg use
So this fixture set WindowsSelectorEventLoopPolicy as default event loop
Set custom event loop policy for pytest-asyncio:
https://github.com/pytest-dev/pytest-asyncio/pull/174#issuecomment-650800383
"""
if sys.platform == 'win32':
policy = asyncio.WindowsSelectorEventLoopPolicy()
else:
policy = asyncio.DefaultEventLoopPolicy()
asyncio.set_event_loop_policy(policy)
loop = asyncio.get_event_loop_policy().new_event_loop()
yield loop
loop.close()
asyncio.set_event_loop_policy(None)
def set_loop_policy(event_loop):
log = structlog.get_logger()
if event_loop == 'uvloop':
try:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
log.info("Using uvloop event loop policy")
except ImportError:
log.warning("uvloop is not available.")
elif event_loop == 'tokio':
try:
import tokio
asyncio.set_event_loop_policy(tokio.EventLoopPolicy())
log.info("Using tokio event loop policy")
except ImportError:
log.warning("tokio is not available.")
else:
# set default policy
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
def asyncio_event_loop():
loop = asyncio.DefaultEventLoopPolicy().new_event_loop()
asyncio.set_event_loop(loop)
yield loop
asyncio.set_event_loop(None)
loop.close()
def event_loop_policy():
if uvloop:
return uvloop.EventLoopPolicy()
return asyncio.DefaultEventLoopPolicy()
def new_policy(self):
return asyncio.DefaultEventLoopPolicy()
def __call__(self, *args, **kwargs):
# You can't call AsyncToSync from a thread with a running event loop
try:
event_loop = get_running_loop()
except RuntimeError:
pass
else:
if event_loop.is_running():
raise RuntimeError(
"You cannot use AsyncToSync in the same thread as "
"an async event loop - "
"just await the async function directly.")
if contextvars is not None:
# Wrapping context in list so it can be reassigned from within
# `main_wrap`.
context = [contextvars.copy_context()]
else:
context = None
# Make a future for the return information
call_result = Future()
# Get the source thread
source_thread = threading.current_thread()
# Make a CurrentThreadExecutor we'll use to idle in this thread - we
# need one for every sync frame, even if there's one above us in the
# same thread.
if hasattr(self.executors, "current"):
old_current_executor = self.executors.current
else:
old_current_executor = None
current_executor = CurrentThreadExecutor()
self.executors.current = current_executor
# Use call_soon_threadsafe to schedule a synchronous callback on the
# main event loop's thread if it's there, otherwise make a new loop
# in this thread.
try:
awaitable = self.main_wrap(args, kwargs, call_result,
source_thread, sys.exc_info(), context)
if not (self.main_event_loop
and self.main_event_loop.is_running()):
# Make our own event loop - in a new thread - and run inside that.
# Note(cosven): There must be only one QEventLoop instance,
# so we should use asyncio.DefaultEventLoopPolicy to create new
# event loops.
policy = asyncio.get_event_loop_policy()
try:
asyncio.set_event_loop_policy(
asyncio.DefaultEventLoopPolicy())
loop = asyncio.new_event_loop()
finally:
asyncio.set_event_loop_policy(policy)
loop_executor = ThreadPoolExecutor(max_workers=1)
loop_future = loop_executor.submit(self._run_event_loop, loop,
awaitable)
if current_executor:
# Run the CurrentThreadExecutor until the future is done
current_executor.run_until_future(loop_future)
# Wait for future and/or allow for exception propagation
loop_future.result()
else:
# Call it inside the existing loop
self.main_event_loop.call_soon_threadsafe(
self.main_event_loop.create_task, awaitable)
if current_executor:
# Run the CurrentThreadExecutor until the future is done
current_executor.run_until_future(call_result)
finally:
# Clean up any executor we were running
if hasattr(self.executors, "current"):
del self.executors.current
if old_current_executor:
self.executors.current = old_current_executor
if contextvars is not None:
_restore_context(context[0])
# Wait for results from the future.
return call_result.result()
aiomisc.ThreadPoolExecutor,
concurrent.futures.ThreadPoolExecutor,
)
thread_pool_ids = (
'aiomisc pool',
'default pool',
)
@pytest.fixture(params=thread_pool_implementation, ids=thread_pool_ids)
def thread_pool_executor(request):
return request.param
policies = (asyncio.DefaultEventLoopPolicy(), )
policy_ids = ('asyncio', )
if uvloop:
policies += (uvloop.EventLoopPolicy(), )
policy_ids += ('uvloop', )
@pytest.fixture(params=policies, ids=policy_ids)
def event_loop_policy(request):
return request.param
@pytest.fixture()
def certs():
return Path(os.path.dirname(os.path.abspath(__file__))) / 'certs'
x INTEGER,
y VARCHAR(255),
z FLOAT
)
""")
await connection.execute("DELETE from test_things")
with ctx(dbapi):
to_do = [insert_data(pool, row) for row in range(1000)]
await asyncio.wait(to_do)
to_do = [select_data(pool) for _ in range(500)]
await asyncio.wait(to_do) # <10>
if __name__ == '__main__':
go(psycopg2, profiled) # sync
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
loop = asyncio.get_event_loop()
loop.run_until_complete(main(asyncpg,profiled)) # async default loop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
loop.run_until_complete(main(uvloop, profiled)) # async uvloop
go(psycopg2, timeonly)
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
loop = asyncio.get_event_loop()
loop.run_until_complete(main(asyncpg, timeonly))
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
loop = asyncio.get_event_loop()
def event_loop() -> Iterator[asyncio.AbstractEventLoop]:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
loop = asyncio.get_event_loop_policy().new_event_loop()
yield loop
gc.collect()
loop.close()
else:
if (hasattr(asyncio.AbstractEventLoop, 'shutdown_default_executor')
and not hasattr(uvloop.loop.Loop, 'shutdown_default_executor')):
uvloop_marks.append(
pytest.mark.skip(
reason='uvloop is missing shutdown_default_executor()'))
else:
uvloop_policy = uvloop.EventLoopPolicy()
pytest_plugins = ['pytester']
@pytest.fixture(params=[
pytest.param(('asyncio', {
'debug': True,
'policy': asyncio.DefaultEventLoopPolicy()
}),
id='asyncio'),
pytest.param(('asyncio', {
'debug': True,
'policy': uvloop_policy
}),
marks=uvloop_marks,
id='asyncio+uvloop'),
pytest.param('curio'),
pytest.param('trio')
])
def anyio_backend(request):
return request.param
def event_loop():
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
loop = asyncio.get_event_loop_policy().new_event_loop()
yield loop
gc.collect()
loop.close()
def run(
coro: "Optional[Coroutine]" = None,
*,
loop: Optional[AbstractEventLoop] = None,
shutdown_handler: Optional[Callable[[AbstractEventLoop], None]] = None,
executor_workers: int = 10,
executor: Optional[Executor] = None,
use_uvloop: bool = False,
stop_on_unhandled_errors: bool = False
) -> None:
"""
Start up the event loop, and wait for a signal to shut down.
:param coro: Optionally supply a coroutine. The loop will still
run if missing. The loop will continue to run after the supplied
coroutine finishes. The supplied coroutine is typically
a "main" coroutine from which all other work is spawned.
:param loop: Optionally supply your own loop. If missing, the
default loop attached to the current thread context will
be used, i.e., whatever ``asyncio.get_event_loop()`` returns.
:param shutdown_handler: By default, SIGINT and SIGTERM will be
handled and will stop the loop, thereby invoking the shutdown
sequence. Alternatively you can supply your own shutdown
handler function. It should conform to the type spec as shown
in the function signature.
:param executor_workers: The number of workers in the executor.
(NOTE: ``run()`` creates a new executor instance internally,
regardless of whether you supply your own loop.)
:param executor: You can decide to use your own executor instance
if you like.
:param use_uvloop: The loop policy will be set to use uvloop. It
is your responsibility to install uvloop. If missing, an
``ImportError`` will be raised.
:param stop_on_unhandled_errors: By default, the event loop will
handle any exceptions that get raised and are not handled. This
means that the event loop will continue running regardless of errors,
and the only way to stop it is to call `loop.stop()`. However, if
this flag is set, any unhandled exceptions will stop the loop, and
be re-raised after the normal shutdown sequence is completed.
"""
logger.debug("Entering run()")
if loop and use_uvloop:
raise Exception(
"'loop' and 'use_uvloop' parameters are mutually "
"exclusive. (Just make your own uvloop and pass it in)."
)
if use_uvloop:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
else:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
if loop and loop.get_exception_handler() and stop_on_unhandled_errors:
raise Exception(
"If you provide a loop instance, and you've configured a custom "
"exception handler on it, then the 'stop_on_unhandled_errors' "
"parameter is unavailable (all exceptions will be handled)."
)
loop_was_supplied = bool(loop)
if not loop_was_supplied:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
pending_exception_to_raise = None
def custom_exception_handler(loop, context: dict):
"""See: https://docs.python.org/3/library/asyncio-eventloop.html#error-handling-api"""
nonlocal pending_exception_to_raise
pending_exception_to_raise = context["exception"]
logger.error("Unhandled exception; stopping loop.")
loop.stop()
if stop_on_unhandled_errors:
loop.set_exception_handler(custom_exception_handler)
if coro:
async def new_coro():
"""During shutdown, run_until_complete() will exit
if a CancelledError bubbles up from anything in the
group. To counteract that, we'll try to handle
any CancelledErrors that bubble up from the given
coro. This isn't fool-proof: if the user doesn't
provide a coro, and instead creates their own with
loop.create_task, that task might bubble
a CancelledError into the run_until_complete()."""
try:
await coro
except asyncio.CancelledError:
pass
loop.create_task(new_coro())
shutdown_handler = shutdown_handler or _shutdown_handler
if WINDOWS: # pragma: no cover
# This is to allow CTRL-C to be detected in a timely fashion,
# see: https://bugs.python.org/issue23057#msg246316
loop.create_task(windows_support_wakeup())
# This is to be able to handle SIGBREAK.
def windows_handler(sig, frame):
# Disable the handler so it won't be called again.
signame = signal.Signals(sig).name
logger.critical("Received signal: %s. Stopping the loop.", signame)
shutdown_handler(loop)
signal.signal(signal.SIGBREAK, windows_handler)
signal.signal(signal.SIGINT, windows_handler)
else:
loop.add_signal_handler(SIGINT, shutdown_handler, loop)
loop.add_signal_handler(SIGTERM, shutdown_handler, loop)
# TODO: We probably don't want to create a different executor if the
# TODO: loop was supplied. (User might have put stuff on that loop's
# TODO: executor).
if not executor:
logger.debug("Creating default executor")
executor = ThreadPoolExecutor(max_workers=executor_workers)
loop.set_default_executor(executor)
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
logger.info("Got KeyboardInterrupt")
if WINDOWS:
# Windows doesn't do any POSIX signal handling, and no
# abstraction layer for signals is currently implemented in
# asyncio. So we fall back to KeyboardInterrupt (triggered
# by the user/environment sending CTRL-C, or signal.CTRL_C_EVENT
shutdown_handler()
logger.info("Entering shutdown phase.")
def sep():
tasks = all_tasks(loop=loop)
do_not_cancel = set()
for t in tasks:
# TODO: we don't need access to the coro. We could simply
# TODO: store the task itself in the weakset.
if t._coro in _DO_NOT_CANCEL_COROS:
do_not_cancel.add(t)
tasks -= do_not_cancel
logger.info("Cancelling pending tasks.")
for t in tasks:
logger.debug("Cancelling task: %s", t)
t.cancel()
return tasks, do_not_cancel
tasks, do_not_cancel = sep()
# Here's a protip: if you group a bunch of tasks, and some of them
# get cancelled, and they DON'T HANDLE THE CANCELLATION, then the
# raised CancelledError will bubble up to, and stop the
# loop.run_until_complete() line: meaning, not all the tasks in
# the gathered group will actually be complete. You need to
# enable this with the ``return_exceptions`` flag.
group = gather(*tasks, *do_not_cancel, loop=loop, return_exceptions=True)
logger.info("Running pending tasks till complete")
# TODO: obtain all the results, and log any results that are exceptions
# other than CancelledError. Will be useful for troubleshooting.
loop.run_until_complete(group)
logger.info("Waiting for executor shutdown.")
executor.shutdown(wait=True)
# If loop was supplied, it's up to the caller to close!
if not loop_was_supplied:
if sys.version_info >= (3, 6):
logger.info("Shutting down async generators")
loop.run_until_complete(loop.shutdown_asyncgens())
logger.info("Closing the loop.")
loop.close()
logger.info("Leaving. Bye!")
if pending_exception_to_raise:
logger.info("Reraising unhandled exception")
raise pending_exception_to_raise
def run(coro: "Optional[Coroutine]" = None,
*,
loop: Optional[AbstractEventLoop] = None,
shutdown_handler: Optional[Callable[[AbstractEventLoop], None]] = None,
shutdown_callback: "ShutdownCallback" = None,
executor_workers: Optional[int] = None,
executor: Optional[Executor] = None,
use_uvloop: bool = False,
stop_on_unhandled_errors: bool = False) -> None:
"""
Start up the event loop, and wait for a signal to shut down.
:param coro: Optionally supply a coroutine. The loop will still
run if missing. The loop will continue to run after the supplied
coroutine finishes. The supplied coroutine is typically
a "main" coroutine from which all other work is spawned.
:param loop: Optionally supply your own loop. If missing, the
default loop attached to the current thread context will
be used, i.e., whatever ``asyncio.get_event_loop()`` returns.
:param shutdown_handler: By default, SIGINT and SIGTERM will be
handled and will stop the loop, thereby invoking the shutdown
sequence. Alternatively you can supply your own shutdown
handler function. It should conform to the type spec as shown
in the function signature.
:param shutdown_callback: Callable, executed after loop is stopped, before
cancelling any tasks.
Useful for graceful shutdown.
:param executor_workers: The number of workers in the executor.
NOTE: ``run()`` creates a new executor instance internally,
regardless of whether you supply your own loop. Note that this
parameter will be ignored if you provide an executor parameter.
:param executor: You can decide to use your own executor instance
if you like. If you provide an executor instance, the
executor_workers parameter will be ignored.
:param use_uvloop: The loop policy will be set to use uvloop. It
is your responsibility to install uvloop. If missing, an
``ImportError`` will be raised.
:param stop_on_unhandled_errors: By default, the event loop will
handle any exceptions that get raised and are not handled. This
means that the event loop will continue running regardless of errors,
and the only way to stop it is to call `loop.stop()`. However, if
this flag is set, any unhandled exceptions will stop the loop, and
be re-raised after the normal shutdown sequence is completed.
"""
_clear_signal_handlers()
logger.debug("Entering run()")
# Disable default signal handling ASAP
if loop and use_uvloop:
raise Exception(
"'loop' and 'use_uvloop' parameters are mutually "
"exclusive. (Just make your own uvloop and pass it in).")
loop_was_supplied = bool(loop)
if not loop_was_supplied:
if use_uvloop:
import uvloop
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
else:
asyncio.set_event_loop_policy(asyncio.DefaultEventLoopPolicy())
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
if loop and loop.get_exception_handler() and stop_on_unhandled_errors:
raise Exception(
"If you provide a loop instance, and you've configured a custom "
"exception handler on it, then the 'stop_on_unhandled_errors' "
"parameter is unavailable (all exceptions will be handled).")
pending_exception_to_raise = None
def custom_exception_handler(loop, context: dict):
"""See: https://docs.python.org/3/library/asyncio-eventloop.html#error-handling-api"""
nonlocal pending_exception_to_raise
pending_exception_to_raise = context.get("exception")
logger.error("Unhandled exception; stopping loop: %r",
context.get("message"),
exc_info=pending_exception_to_raise)
loop.stop()
if stop_on_unhandled_errors:
loop.set_exception_handler(custom_exception_handler)
if coro:
async def new_coro():
"""During shutdown, run_until_complete() will exit
if a CancelledError bubbles up from anything in the
group. To counteract that, we'll try to handle
any CancelledErrors that bubble up from the given
coro. This isn't fool-proof: if the user doesn't
provide a coro, and instead creates their own with
loop.create_task, that task might bubble
a CancelledError into the run_until_complete()."""
try:
await coro
except asyncio.CancelledError:
pass
loop.create_task(new_coro())
shutdown_handler = shutdown_handler or _shutdown_handler
# Setting up signal handlers. The callback configured by the
# underlying system (non-asyncio) API ``signal.signal`` is
# pre-emptive, which means you can't safely do loop manipulation
# with it; yet, aiorun provides an API that allows you to specify
# a ``shutdown_handler`` that takes a loop parameter. This will be
# used to manipulate the loop. How to bridge these two worlds?
# Here we use a private, internal wrapper function that can be
# called from ``signal.signal`` (i.e. pre-emptive interruption)
# but which will call our actual, non-pre-emptive shutdown handler
# in a safe way.
#
# This is supposed to be what loop.add_signal_handler does, but I
# cannot seem get it to work robustly.
sighandler = partial(_signal_wrapper,
loop=loop,
actual_handler=shutdown_handler)
_set_signal_handlers(sighandler)
if WINDOWS: # pragma: no cover
# This is to allow CTRL-C to be detected in a timely fashion,
# see: https://bugs.python.org/issue23057#msg246316
loop.create_task(windows_support_wakeup())
# TODO: We probably don't want to create a different executor if the
# TODO: loop was supplied. (User might have put stuff on that loop's
# TODO: executor).
if not executor:
logger.debug("Creating default executor")
executor = ThreadPoolExecutor(max_workers=executor_workers)
loop.set_default_executor(executor)
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
logger.info("Got KeyboardInterrupt")
if WINDOWS:
# Windows doesn't do any POSIX signal handling, and no
# abstraction layer for signals is currently implemented in
# asyncio. So we fall back to KeyboardInterrupt (triggered
# by the user/environment sending CTRL-C, or signal.CTRL_C_EVENT
shutdown_handler(loop)
logger.info("Entering shutdown phase.")
if shutdown_callback is not None:
logger.info("Executing provided shutdown_callback.")
try:
if inspect.iscoroutine(shutdown_callback):
loop.run_until_complete(shutdown_callback)
elif inspect.iscoroutinefunction(shutdown_callback):
loop.run_until_complete(shutdown_callback(loop))
elif callable(shutdown_callback):
shutdown_callback(loop)
else:
raise TypeError(
"The provided shutdown_callback must be either a function,"
"an awaitable, or a coroutine object, but it was " +
str(type(shutdown_callback)))
except BaseException as exc:
if pending_exception_to_raise:
logger.exception(
"The shutdown_callback() raised an error, but there is "
"already a different exception raised from the loop, so "
"this log message is all you're going to see about it.")
else:
pending_exception_to_raise = exc
def sep():
tasks = all_tasks(loop=loop)
do_not_cancel = set()
for t in tasks:
# TODO: we don't need access to the coro. We could simply
# TODO: store the task itself in the weakset.
if t._coro in _DO_NOT_CANCEL_COROS:
do_not_cancel.add(t)
tasks -= do_not_cancel
logger.info("Cancelling pending tasks.")
for t in tasks:
logger.debug("Cancelling task: %s", t)
t.cancel()
return tasks, do_not_cancel
tasks, do_not_cancel = sep()
async def wait_for_cancelled_tasks():
return await gather(*tasks, *do_not_cancel, return_exceptions=True)
if tasks or do_not_cancel:
logger.info("Running pending tasks till complete")
# TODO: obtain all the results, and log any results that are exceptions
# other than CancelledError. Will be useful for troubleshooting.
loop.run_until_complete(wait_for_cancelled_tasks())
logger.info("Waiting for executor shutdown.")
executor.shutdown(wait=True)
# If loop was supplied, it's up to the caller to close!
if not loop_was_supplied:
if sys.version_info >= (3, 6):
logger.info("Shutting down async generators")
loop.run_until_complete(loop.shutdown_asyncgens())
logger.info("Closing the loop.")
loop.close()
logger.info("Leaving. Bye!")
if pending_exception_to_raise:
logger.info("Reraising unhandled exception")
raise pending_exception_to_raise
def run_command(self, args: List[str]) -> None:
if len(args) == 0:
print(self.run_command_usage())
else:
configuration = None
log_level = logging.INFO
env_loop = str(os.getenv("TOMODACHI_LOOP", "")).lower() or None
if env_loop or "--loop" in args:
if "--loop" in args:
index = args.index("--loop")
args.pop(index)
value = args.pop(index).lower()
if env_loop and env_loop != value:
print(
"Invalid argument to --loop, '{}' differs from env TOMODACHI_LOOP"
.format(value))
sys.exit(2)
elif env_loop:
value = env_loop
else:
value = "auto"
if value in ("auto", "default"):
pass
elif value in ("asyncio", "aio", "async"):
asyncio.set_event_loop_policy(
asyncio.DefaultEventLoopPolicy())
pass
elif value in ("uvloop", "libuv", "uv"):
try:
import uvloop # noqa # isort:skip
except Exception: # pragma: no cover
print(
"The 'uvloop' package needs to be installed to use uvloop event loop"
)
sys.exit(2)
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
else:
print(
"Invalid argument to --loop, event loop '{}' not recognized"
.format(value))
sys.exit(2)
if "-c" in args or "--config" in args:
index = args.index("-c") if "-c" in args else args.index(
"--config")
args.pop(index)
config_files: List[str] = []
while len(args) > index and args[index][0] != "-":
value = args.pop(index)
if value not in config_files:
config_files.append(value)
if not len(config_files):
print("Missing config file on command line")
sys.exit(2)
try:
configuration = parse_config_files(config_files)
except FileNotFoundError as e:
print("Invalid config file: {}".format(str(e)))
sys.exit(2)
except ValueError as e:
print(
"Invalid config file, invalid JSON format: {}".format(
str(e)))
sys.exit(2)
env_production = str(os.getenv("TOMODACHI_PRODUCTION",
"")).lower() or None
if env_production and env_production in ("0", "no", "none",
"false"):
env_production = None
if env_production or "--production" in args:
if "--production" in args:
index = args.index("--production")
args.pop(index)
watcher = None
else:
cwd = os.path.realpath(os.getcwd())
root_directories = [cwd]
for arg in set(args):
if not arg.startswith("/") and not arg.startswith("~"):
root_directories.append(
os.path.realpath(
os.path.dirname(os.path.join(cwd, arg))))
else:
root_directories.append(
os.path.realpath(os.path.dirname(arg)))
for p in str(os.getenv("PYTHONPATH", "")).split(os.pathsep):
if not p:
continue
if not p.startswith("/") and not p.startswith("~"):
root_directories.append(
os.path.realpath(os.path.join(cwd, p)))
else:
root_directories.append(os.path.realpath(p))
from tomodachi.watcher import Watcher # noqa # isort:skip
watcher = Watcher(root=root_directories,
configuration=configuration)
if "-l" in args or "--log" in args or "--log-level" in args:
index = (args.index("-l")
if "-l" in args else args.index("--log")
if "--log" in args else args.index("--log-level"))
args.pop(index)
if len(args) > index:
log_level = getattr(logging,
args.pop(index).upper(),
None) or log_level
logging.basicConfig(format="%(asctime)s (%(name)s): %(message)s",
level=log_level)
logging.Formatter(fmt="%(asctime)s.%(msecs).03d",
datefmt="%Y-%m-%d %H:%M:%S")
ServiceLauncher.run_until_complete(set(args), configuration,
watcher)
sys.exit(tomodachi.SERVICE_EXIT_CODE)
from contextlib import suppress
from functools import partial
import logging
import socket
from xTool.utils.net import is_ipv6
UVLOOP_INSTALLED = False
try:
import uvloop # type: ignore # noqa
UVLOOP_INSTALLED = True
event_loop_policy = uvloop.EventLoopPolicy()
except ImportError:
event_loop_policy = asyncio.DefaultEventLoopPolicy()
except ImportError:
pass
log = logging.getLogger(__name__)
T = TypeVar("T")
OptionsType = Iterable[Tuple[int, int, int]]
F = TypeVar("F", bound=Callable[..., Any])
try:
import contextvars
def context_partial(func: F, *args: Any, **kwargs: Any) -> Any:
context = contextvars.copy_context()
def __init__(self):
self.inner_policy = asyncio.DefaultEventLoopPolicy()
