def run_loop(loop: asyncio.AbstractEventLoop, runners: List[Runner]) -> None:
for runner in runners:
print("Serving on {}:{}".format(runner.host, runner.port))
task = loop.create_task(_windows_ctrl_c_workaround())
foos = itertools.chain(*(
runner.create_servers(loop) for runner in runners
))
servers = loop.run_until_complete(asyncio.gather(*foos))
try:
loop.run_forever()
except KeyboardInterrupt:
# To prevent "Task exception was never retrieved"
if task.done():
task.exception()
raise
finally:
for server in servers:
server.close()
loop.run_until_complete(asyncio.gather(*(
server.wait_closed() for server in servers
)))
for runner in runners:
runner.teardown(loop)
server_temp_cleanup()
loop.close()
def run_server(loop: asyncio.AbstractEventLoop):
# https://stackoverflow.com/questions/51610074/how-to-run-an-aiohttp-server-in-a-thread
runner = web.AppRunner(app)
loop.run_until_complete(runner.setup())
site = web.TCPSite(runner, '127.0.0.1', port)
loop.run_until_complete(site.start())
loop.run_forever()
def __start(l: asyncio.AbstractEventLoop) -> None:
"""
Start the websocket-request-polling on a separate thread.
"""
asyncio.set_event_loop(l)
l.run_until_complete(websockets.serve(_srv, params.HOST, params.PORT))
l.run_forever()
def _loop(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
if not loop.is_running() or loop.is_closed():
loop.run_forever()
pending = asyncio.all_tasks(loop=loop)
if pending:
loop.run_until_complete(asyncio.gather(*pending))
def main(loop: asyncio.AbstractEventLoop) -> None:
telethon, pytonisadb, rabbitmq, pytonisa_files = loop.run_until_complete(
asyncio.gather(
start_telethon(),
start_pytonisadb(),
start_rabbitmq(loop),
start_pytonisa_file_storage(),
))
queuehandlers.telegram = telethon
queuehandlers.rabbitmq = rabbitmq
queuehandlers.pytonisadb = pytonisadb
queuehandlers.pytonisa_files = pytonisa_files
messagehandlers.rabbitmq = rabbitmq
messagehandlers.pytonisadb = pytonisadb
messagehandlers.pytonisa_files = pytonisa_files
log.info('Bot initiated')
try:
loop.run_forever()
except KeyboardInterrupt:
loop.run_until_complete(
asyncio.gather(
exit_telethon(telethon),
exit_rabbitmq(rabbitmq),
exit_pytonisadb(pytonisadb),
exit_pytonisa_file_storage(pytonisa_files),
))
def test_asyncio_no_root_task(asyncio_event_loop: asyncio.AbstractEventLoop) -> None:
"""
Regression test for #264.
Ensures that to_thread.run_sync() does not raise an error when there is no root task, but
instead tries to find the top most parent task by traversing the cancel scope tree, or failing
that, uses the current task to set up a shutdown callback.
"""
async def run_in_thread() -> None:
try:
await to_thread.run_sync(time.sleep, 0)
finally:
asyncio_event_loop.call_soon(asyncio_event_loop.stop)
task = asyncio_event_loop.create_task(run_in_thread())
asyncio_event_loop.run_forever()
task.result()
# Wait for worker threads to exit
for t in threading.enumerate():
if t.name == "AnyIO worker thread":
t.join(2)
assert not t.is_alive()
def loop_mgr(loop: asyncio.AbstractEventLoop):
"""
An asyncio loop manager, used by :py:class:`.AsyncioExecutor`
to run the loop forever in a thread and clean up after the
loop stops.
:param loop:
"""
try:
# loop manager will run this in it's own thread
loop.run_forever()
# the loop was stopped and concurrent.futures.Executor
# promises to complete tasks on shutdown.
while True:
tasks = asyncio.all_tasks(loop=loop)
pending = [t for t in tasks if not t.done()]
loop.run_until_complete(asyncio.gather(*pending))
# ensure the task collection is updated
# (this is _not_ redundant)
tasks = asyncio.all_tasks(loop=loop)
if all([t.done() for t in tasks]):
break
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
def graceful_shutdown(loop: asyncio.AbstractEventLoop = None):
tasks = asyncio.gather(*asyncio.Task.all_tasks(loop=loop), loop=loop, return_exceptions=True)
tasks.add_done_callback(lambda t: loop.stop())
tasks.cancel()
while not tasks.done() and not loop.is_closed():
loop.run_forever()
tasks.exception()
def start_background_loop(loop: asyncio.AbstractEventLoop, workers, model,
optimizer, image_dim, update_queue, results_box,
verbose) -> None:
asyncio.set_event_loop(loop)
asyncio.run_coroutine_threadsafe(
update_model(loop, workers, model, optimizer, image_dim, update_queue,
results_box, verbose), loop)
loop.run_forever()
def _loop_mgr(loop: asyncio.AbstractEventLoop):
asyncio.set_event_loop(loop)
loop.run_forever()
# If we reach here, the loop was stopped.
# We should gather any remaining tasks and finish them.
pending = asyncio.Task.all_tasks(loop=loop)
if pending:
loop.run_until_complete(asyncio.gather(*pending))
def run(self,
loop: asyncio.AbstractEventLoop = None
) -> None: # pragma: no cover
start_server = self.get_server_task(self.router)
if loop is None:
loop = asyncio.get_event_loop()
atexit.register(self.shutdown_handler)
loop.run_until_complete(start_server)
loop.run_forever()
def start_background_loop(loop: asyncio.AbstractEventLoop) -> None:
"""
Sets and starts the event loop
:param asyncio.AbstractEventLoop loop: The event loop to use
:return: None
"""
asyncio.set_event_loop(loop)
loop.run_forever()
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None:
"""Teardown and cleanup an event_loop created by setup_test_loop."""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def teardown_test_loop(loop: asyncio.AbstractEventLoop,
fast: bool = False) -> None:
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def start_background_loop(loop: asyncio.AbstractEventLoop) -> None:
"""
Sets and starts the event loop
Parameters
----------
loop : asyncio.AbstractEventLoop
The event loop to use
"""
asyncio.set_event_loop(loop)
loop.run_forever()
def test_server_restart(base_server: Server, http_client: HttpClient,
event_loop: asyncio.AbstractEventLoop):
result = {}
async def do_restart():
base_server.load_components()
await base_server.start_server()
ret = await http_client.post("/server/restart")
result.update(ret)
event_loop.create_task(do_restart())
event_loop.run_forever()
assert result["result"] == "ok" and base_server.exit_reason == "restart"
def run_forever(self, loop: asyncio.AbstractEventLoop = None):
"""
Creates an event loop and runs the server forever.
:param asyncio.AbstractEventLoop loop: the event loop to use
"""
loop = loop or asyncio.get_event_loop()
self.start_listening(loop)
def wakeup():
"""Needed for Windows to notice Ctrl-C and other signals"""
loop.call_later(0.1, wakeup)
loop.call_later(0.1, wakeup)
loop.run_forever()
def test_websocket_restart(base_server: Server,
websocket_client: WebsocketClient,
event_loop: asyncio.AbstractEventLoop):
result = {}
async def do_restart():
base_server.load_components()
await base_server.start_server()
await websocket_client.connect()
ret = await websocket_client.request("server.restart")
result["result"] = ret
event_loop.create_task(do_restart())
event_loop.run_forever()
assert result["result"] == "ok" and base_server.exit_reason == "restart"
def test_with_message(
self,
event_loop: asyncio.AbstractEventLoop,
log_records: LogRecordsType,
patched_shutdown: AsyncMockType,
) -> None:
context = {"message": "exception message"}
handle_exception(event_loop, context)
event_loop.stop()
event_loop.run_forever()
expected_record = [
rec for rec in log_records() if rec.levelno == logging.ERROR
and "exception message" in rec.message
]
assert len(expected_record)
assert patched_shutdown.called
def teardown_test_loop(loop: asyncio.AbstractEventLoop,
fast: bool=False) -> None:
"""Teardown and cleanup an event_loop created
by setup_test_loop.
"""
closed = loop.is_closed()
if not closed:
loop.call_soon(loop.stop)
loop.run_forever()
loop.close()
if not fast:
gc.collect()
asyncio.set_event_loop(None)
def run(self, event_loop: AbstractEventLoop = None):
# Configure the logging system
if isinstance(self.logging_config, dict):
logging.config.dictConfig(self.logging_config)
elif self.logging_config:
logging.basicConfig(level=logging.INFO)
# Assign a new default executor with the given max worker thread limit
event_loop = event_loop or asyncio.get_event_loop()
event_loop.set_default_executor(ThreadPoolExecutor(self.max_threads))
# Create the application context
context = self.create_context()
try:
# Start all the components and run the loop until they've finished
self.logger.info("Starting components")
coroutines = (component.start(context) for component in self.components)
coroutines = [coro for coro in coroutines if coro is not None]
event_loop.run_until_complete(asyncio.gather(*coroutines))
self.logger.info("All components started")
# Run the application's custom startup code
coro = self.start(context)
if coro is not None:
event_loop.run_until_complete(coro)
# Run all the application context's start callbacks
event_loop.run_until_complete(context.run_callbacks(ContextEventType.started))
self.logger.info("Application started")
except Exception as exc:
self.logger.exception("Error during application startup")
context.exception = exc
else:
# Finally, run the event loop until the process is terminated or Ctrl+C is pressed
try:
event_loop.run_forever()
except (KeyboardInterrupt, SystemExit):
pass
event_loop.run_until_complete(context.run_callbacks(ContextEventType.finished))
event_loop.close()
self.logger.info("Application stopped")
def run(loop: asyncio.AbstractEventLoop) -> None:
try:
loop.add_signal_handler(signal.SIGINT, loop.stop)
loop.add_signal_handler(signal.SIGTERM, loop.stop)
except (NotImplementedError, RuntimeError):
pass
asyncio.set_event_loop(loop)
try:
loop.run_forever()
except KeyboardInterrupt:
log.info('Received the signal to terminate the event loop.')
finally:
log.info('Cleaning up tasks.')
shutdown_loop(loop)
def run(self,
loop: asyncio.AbstractEventLoop = None,
host='0.0.0.0',
port=2775):
if loop is None:
loop = asyncio.get_event_loop()
server = self.create_server(loop=loop, host=host, port=port)
self.logger.info(f'Starting server on {host}:{port} ...')
try:
loop.run_forever()
finally:
self.logger.info('closing server...')
server.close()
loop.run_until_complete(server.wait_closed())
self.logger.info('closing event loop')
loop.close()
def listen_requests(
tcp_ip: str,
tcp_port: str,
loop: AbstractEventLoop,
shutdown_flag: threading.Event,
) -> None:
"""Run the request server until closing"""
coro = asyncio.start_server(
async_handle_request, tcp_ip, int(tcp_port), loop=loop,
)
server = loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
def _loop_mgr(loop: asyncio.AbstractEventLoop) -> None:
"""起一个线程执行事件循环的`run_forever`方法.
[Start up a thread for running the eventloop's `run_forever` method.]
当它被终止时会清理未完结的协程,但不会关闭事件循环
[When it shutdown, all the coroutines will be closed.but the eventloop will not close.]
Params:
loop (asyncio.AbstractEventLoop) : - 事件循环[the eventloop]
"""
if loop.is_closed():
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
loop.run_forever()
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
def test_with_exc_and_future(
self,
event_loop: asyncio.AbstractEventLoop,
log_records: LogRecordsType,
patched_shutdown: AsyncMockType,
) -> None:
context = {
"exception": ExceptionForTesting("Exception handler test"),
"future": asyncio.Future(),
}
handle_exception(event_loop, context)
event_loop.stop()
event_loop.run_forever()
expected_record = [
rec for rec in log_records() if rec.levelno == logging.ERROR and
"ExceptionForTesting" in rec.message and "unknown" in rec.message
and "Exception handler test" in rec.message
]
assert len(expected_record)
assert patched_shutdown.called
def test_with_exc_and_task(
self,
event_loop: asyncio.AbstractEventLoop,
log_records: LogRecordsType,
mocker: MockerFixture,
patched_shutdown: AsyncMockType,
) -> None:
context = {
"exception": ExceptionForTesting("Exception handler test"),
"future": mocker.Mock(**{"get_name.return_value": "test_task"}),
}
handle_exception(event_loop, context)
event_loop.stop()
event_loop.run_forever()
expected_record = [
rec for rec in log_records() if rec.levelno == logging.ERROR and
"ExceptionForTesting" in rec.message and "test_task" in rec.message
and "Exception handler test" in rec.message
]
assert len(expected_record)
assert patched_shutdown.called
def serve_main_app(config: Config, loop: asyncio.AbstractEventLoop = None):
setup_logging(config.verbose)
loop = loop or asyncio.get_event_loop()
if isinstance(config.app_factory, Application):
app = config.app_factory
else:
app = config.app_factory(loop=loop)
modify_main_app(app, config)
loop.run_until_complete(check_port_open(config.main_port, loop))
handler = app.make_handler(
logger=dft_logger,
access_log_format='%r %s %b',
loop=loop,
)
co = asyncio.gather(loop.create_server(handler,
HOST,
config.main_port,
backlog=128),
app.startup(),
loop=loop)
server, startup_res = loop.run_until_complete(co)
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
server.close()
loop.run_until_complete(server.wait_closed())
loop.run_until_complete(app.shutdown())
try:
loop.run_until_complete(handler.shutdown(0.1))
except asyncio.TimeoutError:
pass
loop.run_until_complete(app.cleanup())
loop.close()
def serve_main_app(config: Config,
tty_path: Optional[str],
loop: asyncio.AbstractEventLoop = None):
with set_tty(tty_path):
setup_logging(config.verbose)
# imports the factory. This gives users a chance to register alternative event loops
config.app_factory
loop = loop or asyncio.get_event_loop()
app = config.load_app(loop)
modify_main_app(app, config)
loop.run_until_complete(check_port_open(config.main_port, loop))
handler = app.make_handler(
logger=dft_logger,
access_log_format='%r %s %b',
loop=loop,
)
loop.run_until_complete(app.startup())
server = loop.run_until_complete(
loop.create_server(handler, HOST, config.main_port, backlog=128))
try:
loop.run_forever()
except KeyboardInterrupt: # pragma: no cover
pass
finally:
server.close()
loop.run_until_complete(server.wait_closed())
loop.run_until_complete(app.shutdown())
with contextlib.suppress(asyncio.TimeoutError, KeyboardInterrupt):
loop.run_until_complete(handler.shutdown(0.1))
with contextlib.suppress(KeyboardInterrupt):
loop.run_until_complete(app.cleanup())
with contextlib.suppress(KeyboardInterrupt):
loop.close()
def run_server(
args,
local_addr: str,
self_port: int = _SELF_PORT,
async_loop: aio.AbstractEventLoop = aio.get_event_loop()
) -> None:
protocol_factory = Server
try:
db = DnsDb()
transport, udp_server = _get_udp_server(async_loop,
protocol_factory,
local_addr=(local_addr,
self_port))
async_loop.run_until_complete(_server_init(db, args))
responder = Responder(udp_server, db)
tcp_server: aio.AbstractServer = _get_tcp_server(
async_loop, responder, local_addr, self_port)
async_loop.run_forever()
except KeyboardInterrupt:
raise
finally:
async_loop.close()
def startServer(self, loop: asyncio.AbstractEventLoop):
"""
Continuously process I/O events and handle the connections accordingly.
"""
server = asyncio.start_server(self.handleMessage,
self.host,
self.port,
loop=loop)
task = loop.run_until_complete(server)
print('Serving on:', repr(task.sockets[0].getsockname()))
# Continuously run the async loop
try:
loop.run_forever()
except KeyboardInterrupt:
print("Caught keyboard interrupt. Exiting...")
# Close the server
task.close()
loop.run_until_complete(task.wait_closed())
loop.close()
def close_event_loop(event_loop: asyncio.AbstractEventLoop) -> None:
# give event loop one chance to finish up
event_loop.stop()
event_loop.run_forever()
# wait for everything to finish, including tasks running in executors
# this assumes that all outstanding tasks finish in a reasonable time (i.e. no infinite loops).
all_tasks_fn = getattr(asyncio, "all_tasks", None)
if not all_tasks_fn:
all_tasks_fn = asyncio.Task.all_tasks
tasks = all_tasks_fn(loop=event_loop)
if tasks:
gather_future = asyncio.gather(*tasks, return_exceptions=True)
else:
# work around bad design in gather (always uses global event loop in Python 3.8)
gather_future = event_loop.create_future()
gather_future.set_result([])
event_loop.run_until_complete(gather_future)
# due to a bug in Python libraries, the default executor needs to be shutdown explicitly before the event loop
# see http://bugs.python.org/issue28464 . this bug manifests itself in at least one way: an intermittent failure
# in test_document_controller_releases_itself. reproduce by running the contents of that test in a loop of 100.
_default_executor = getattr(event_loop, "_default_executor", None)
if _default_executor:
_default_executor.shutdown()
event_loop.close()
