def test_catch_broken_queue_error(event_loop: AbstractEventLoop) -> None: """test for deleting log queue when read queue is awaiting streaming data from log queue""" logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) async def flow(read_streamer: LocalReadStreamer): id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) assert read_streamer._id_to_logqueue.get(id) assert read_streamer._key_to_readqueue.get(key) with pytest.raises(ValueError): # keep awaitting to read streamer get log until timeout await read_streamer.get(key) # successfully raise error and break all running asynchronous process async def post_flow(logstreamer): await asyncio.sleep(0.3) # force delete log queue logstreamer.delete("xxx") await asyncio.sleep(0.3) event_loop.create_task(flow(read_streamer)) event_loop.run_until_complete(post_flow(logstreamer)) event_loop.close()
def shutdown(loop: asyncio.AbstractEventLoop) -> None: """Cancel all pending tasks on `loop`, wait for them, and close the loop.""" try: if sys.version_info[:2] >= (3, 7): all_tasks = asyncio.all_tasks else: all_tasks = asyncio.Task.all_tasks # This part is borrowed from asyncio/runners.py in Python 3.7b2. to_cancel = [task for task in all_tasks(loop) if not task.done()] if not to_cancel: return for task in to_cancel: task.cancel() if sys.version_info >= (3, 7): loop.run_until_complete( asyncio.gather(*to_cancel, return_exceptions=True)) else: loop.run_until_complete( asyncio.gather(*to_cancel, loop=loop, return_exceptions=True)) finally: # `concurrent.futures.Future` objects cannot be cancelled once they # are already running. There might be some when the `shutdown()` happened. # Silence their logger's spew about the event loop being closed. cf_logger = logging.getLogger("concurrent.futures") cf_logger.setLevel(logging.CRITICAL) loop.close()
def run_loop(loop: asyncio.AbstractEventLoop, runners: List[Runner]) -> None: for runner in runners: print("Serving on {}:{}".format(runner.host, runner.port)) task = loop.create_task(_windows_ctrl_c_workaround()) foos = itertools.chain(*( runner.create_servers(loop) for runner in runners )) servers = loop.run_until_complete(asyncio.gather(*foos)) try: loop.run_forever() except KeyboardInterrupt: # To prevent "Task exception was never retrieved" if task.done(): task.exception() raise finally: for server in servers: server.close() loop.run_until_complete(asyncio.gather(*( server.wait_closed() for server in servers ))) for runner in runners: runner.teardown(loop) server_temp_cleanup() loop.close()
def _run_async(loop: asyncio.AbstractEventLoop, coro: Awaitable) -> Any: """Run a coroutine. Does some special signal handling to perform a graceful exit. Args: loop: Event loop to run coroutine in. coro: Coroutine to run Returns: Result of the coroutine. """ def graceful_exit() -> None: pending = asyncio.Task.all_tasks() for task in pending: task.cancel() loop.stop() try: loop.add_signal_handler(signal.SIGINT, graceful_exit) loop.add_signal_handler(signal.SIGTERM, graceful_exit) except NotImplementedError: pass try: return loop.run_until_complete(coro) except KeyboardInterrupt: print("Exiting") loop.close()
def _cleanup_loop(loop: asyncio.AbstractEventLoop) -> None: try: _cancel_tasks(loop) loop.run_until_complete(loop.shutdown_asyncgens()) finally: log.info('Closing the event loop.') loop.close()
def test_delete_internal_task(event_loop: AbstractEventLoop) -> None: """test for deleting log queue when read queue is awaiting streaming data from log queue""" logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) async def flow(read_streamer: LocalReadStreamer, logstreamer: ToyLogStreamer): id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) assert read_streamer._id_to_logqueue.get(id) assert read_streamer._key_to_readqueue.get(key) # running task for sync streamer log = read_streamer._id_to_logqueue.get(id) assert log assert not log.task.cancelled() await logstreamer.streaming() await logstreamer.streaming() assert not log.task.cancelled() # cancel sync await read_streamer.delete(key) await asyncio.sleep(0.3) assert log.task.cancelled() event_loop.run_until_complete(flow(read_streamer, logstreamer)) event_loop.close()
def test_localread_streamer(event_loop: AbstractEventLoop) -> None: async def flow(): logstreamer = ToyLogStreamer([log_streamer.QueueHandler()]) read_streamer = LocalReadStreamer(logstreamer) id = "xxx" key = "key-something" # add pipe between log streamer and application read streamer await read_streamer.add_client(id, key) assert read_streamer._id_to_logqueue.get(id), "log queue not found" assert read_streamer._key_to_readqueue.get(key), "read queue not found" assert logstreamer._handlers[0]._queues, "log streamer is empty" assert read_streamer._id_to_logqueue.get(id).task, "task is not running" # log streamer receive log and send it into queue for read streamer await logstreamer.streaming() await asyncio.sleep(0.01) # read streamer is collectting streaming data asynchronously # read streamer get log out = await read_streamer.get(key) assert out == "test\n" # finish streaming await read_streamer.delete(key) with pytest.raises(KeyError): # can't get data anymore await read_streamer.get(key) event_loop.run_until_complete(flow()) event_loop.close()
def run(loop: asyncio.AbstractEventLoop, arguments: dict): try: loop.run_until_complete(main(arguments)) loop.run_until_complete(asyncio.sleep(0.250)) loop.run_until_complete(loop.shutdown_asyncgens()) except Exception: log.exception("there has been an unrecoverable error") finally: loop.close()
def start_background_loop(stop_threads, loop: asyncio.AbstractEventLoop) -> None: async def wait_for_close(stop_threads): while True: await asyncio.sleep(1) if stop_threads(): break asyncio.set_event_loop(loop) loop.run_until_complete(wait_for_close(stop_threads)) loop.close()
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None: if loop.is_closed(): return try: loop.stop() cancel_all_tasks(loop) loop.run_until_complete(loop.shutdown_asyncgens()) finally: loop.close()
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None: closed = loop.is_closed() if not closed: loop.call_soon(loop.stop) loop.run_forever() loop.close() if not fast: gc.collect() asyncio.set_event_loop(None)
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool = False) -> None: """Teardown and cleanup an event_loop created by setup_test_loop.""" closed = loop.is_closed() if not closed: loop.call_soon(loop.stop) loop.run_forever() loop.close() if not fast: gc.collect() asyncio.set_event_loop(None)
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None: if loop.is_closed(): return try: cancel_all_tasks(loop) loop.run_until_complete(loop.shutdown_asyncgens()) except Exception as exc: log.warning(f"Error<{type(exc).__name__}> was raised. {exc}") finally: loop.close()
def start_async_callback( event_loop: asyncio.AbstractEventLoop, coroutine_func, shutdown_event: Event, ): asyncio.set_event_loop(event_loop) try: event_loop.run_until_complete(coroutine_func()) event_loop.run_until_complete(event_loop.shutdown_asyncgens()) event_loop.close() finally: shutdown_event.set()
def _loop_mgr(loop: asyncio.AbstractEventLoop): asyncio.set_event_loop(loop) if not loop.is_running(): loop.run_forever() # If we reach here, the loop was stopped. # We should gather any remaining tasks and finish them. pending = asyncio.all_tasks(loop) if pending: loop.run_until_complete(asyncio.gather(*pending)) if not loop.is_running(): loop.close()
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None: if loop.is_closed(): return try: loop.stop() cancel_all_tasks(loop) loop.run_until_complete(loop.shutdown_asyncgens()) except RuntimeError as error: log.warning("Error while shutting loop down: %s", str(error)) finally: loop.close()
def teardown_test_loop(loop: asyncio.AbstractEventLoop, fast: bool=False) -> None: """Teardown and cleanup an event_loop created by setup_test_loop. """ closed = loop.is_closed() if not closed: loop.call_soon(loop.stop) loop.run_forever() loop.close() if not fast: gc.collect() asyncio.set_event_loop(None)
def run(self, loop: asyncio.AbstractEventLoop = None, host='0.0.0.0', port=2775): if loop is None: loop = asyncio.get_event_loop() server = self.create_server(loop=loop, host=host, port=port) self.logger.info(f'Starting server on {host}:{port} ...') try: loop.run_forever() finally: self.logger.info('closing server...') server.close() loop.run_until_complete(server.wait_closed()) self.logger.info('closing event loop') loop.close()
def _destroy_loop(loop: asyncio.AbstractEventLoop) -> None: async def murder(future: asyncio.Future[typing.Any]) -> None: # These include _GatheringFuture which must be awaited if the children # throw an asyncio.CancelledError, otherwise it will spam logs with warnings # about exceptions not being retrieved before GC. try: _LOGGER.log(ux.TRACE, "killing %s", future) future.cancel() await future except asyncio.CancelledError: pass except Exception as ex: loop.call_exception_handler({ "message": "Future raised unexpected exception after requesting cancellation", "exception": ex, "future": future, }) remaining_tasks = [ t for t in asyncio.all_tasks(loop) if not t.cancelled() and not t.done() ] if remaining_tasks: _LOGGER.debug("terminating %s remaining tasks forcefully", len(remaining_tasks)) loop.run_until_complete( asyncio.gather(*(murder(task) for task in remaining_tasks))) else: _LOGGER.debug("No remaining tasks exist, good job!") if sys.version_info >= (3, 9): _LOGGER.debug("shutting down default executor") with contextlib.suppress(NotImplementedError): # This seems to raise a NotImplementedError when running with uvloop. loop.run_until_complete(loop.shutdown_default_executor()) _LOGGER.debug("shutting down asyncgens") loop.run_until_complete(loop.shutdown_asyncgens()) _LOGGER.debug("closing event loop") loop.close()
def run(self, event_loop: AbstractEventLoop = None): # Configure the logging system if isinstance(self.logging_config, dict): logging.config.dictConfig(self.logging_config) elif self.logging_config: logging.basicConfig(level=logging.INFO) # Assign a new default executor with the given max worker thread limit event_loop = event_loop or asyncio.get_event_loop() event_loop.set_default_executor(ThreadPoolExecutor(self.max_threads)) # Create the application context context = self.create_context() try: # Start all the components and run the loop until they've finished self.logger.info("Starting components") coroutines = (component.start(context) for component in self.components) coroutines = [coro for coro in coroutines if coro is not None] event_loop.run_until_complete(asyncio.gather(*coroutines)) self.logger.info("All components started") # Run the application's custom startup code coro = self.start(context) if coro is not None: event_loop.run_until_complete(coro) # Run all the application context's start callbacks event_loop.run_until_complete(context.run_callbacks(ContextEventType.started)) self.logger.info("Application started") except Exception as exc: self.logger.exception("Error during application startup") context.exception = exc else: # Finally, run the event loop until the process is terminated or Ctrl+C is pressed try: event_loop.run_forever() except (KeyboardInterrupt, SystemExit): pass event_loop.run_until_complete(context.run_callbacks(ContextEventType.finished)) event_loop.close() self.logger.info("Application stopped")
def listen_requests( tcp_ip: str, tcp_port: str, loop: AbstractEventLoop, shutdown_flag: threading.Event, ) -> None: """Run the request server until closing""" coro = asyncio.start_server( async_handle_request, tcp_ip, int(tcp_port), loop=loop, ) server = loop.run_until_complete(coro) try: loop.run_forever() except KeyboardInterrupt: pass server.close() loop.run_until_complete(server.wait_closed()) loop.close()
def serve_main_app(config: Config, loop: asyncio.AbstractEventLoop = None): setup_logging(config.verbose) loop = loop or asyncio.get_event_loop() if isinstance(config.app_factory, Application): app = config.app_factory else: app = config.app_factory(loop=loop) modify_main_app(app, config) loop.run_until_complete(check_port_open(config.main_port, loop)) handler = app.make_handler( logger=dft_logger, access_log_format='%r %s %b', loop=loop, ) co = asyncio.gather(loop.create_server(handler, HOST, config.main_port, backlog=128), app.startup(), loop=loop) server, startup_res = loop.run_until_complete(co) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: server.close() loop.run_until_complete(server.wait_closed()) loop.run_until_complete(app.shutdown()) try: loop.run_until_complete(handler.shutdown(0.1)) except asyncio.TimeoutError: pass loop.run_until_complete(app.cleanup()) loop.close()
def serve_main_app(config: Config, tty_path: Optional[str], loop: asyncio.AbstractEventLoop = None): with set_tty(tty_path): setup_logging(config.verbose) # imports the factory. This gives users a chance to register alternative event loops config.app_factory loop = loop or asyncio.get_event_loop() app = config.load_app(loop) modify_main_app(app, config) loop.run_until_complete(check_port_open(config.main_port, loop)) handler = app.make_handler( logger=dft_logger, access_log_format='%r %s %b', loop=loop, ) loop.run_until_complete(app.startup()) server = loop.run_until_complete( loop.create_server(handler, HOST, config.main_port, backlog=128)) try: loop.run_forever() except KeyboardInterrupt: # pragma: no cover pass finally: server.close() loop.run_until_complete(server.wait_closed()) loop.run_until_complete(app.shutdown()) with contextlib.suppress(asyncio.TimeoutError, KeyboardInterrupt): loop.run_until_complete(handler.shutdown(0.1)) with contextlib.suppress(KeyboardInterrupt): loop.run_until_complete(app.cleanup()) with contextlib.suppress(KeyboardInterrupt): loop.close()
def run_server( args, local_addr: str, self_port: int = _SELF_PORT, async_loop: aio.AbstractEventLoop = aio.get_event_loop() ) -> None: protocol_factory = Server try: db = DnsDb() transport, udp_server = _get_udp_server(async_loop, protocol_factory, local_addr=(local_addr, self_port)) async_loop.run_until_complete(_server_init(db, args)) responder = Responder(udp_server, db) tcp_server: aio.AbstractServer = _get_tcp_server( async_loop, responder, local_addr, self_port) async_loop.run_forever() except KeyboardInterrupt: raise finally: async_loop.close()
def _run_server(loop: AbstractEventLoop, stop: "Future[Any]") -> None: """Run the websocket server, stopping and closing the event loop when the server completes.""" # Websockets has this to say about closing connections: # # The close_timeout parameter defines a maximum wait time in seconds for # completing the closing handshake and terminating the TCP connection. # close() completes in at most 4 * close_timeout on the server side and # 5 * close_timeout on the client side. # # Since our configuration specifies a maximum time to wait (in total), we # need to divide by 4 to make sure that time isn't exceeded. # # See also: https://websockets.readthedocs.io/en/stable/api.html#module-websockets.protocol # https://websockets.readthedocs.io/en/stable/api.html#module-websockets.server host = config().server_host port = config().server_port close_timeout_sec = config().close_timeout_sec / 4 loop.run_until_complete(_websocket_server(stop=stop, host=host, port=port, close_timeout_sec=close_timeout_sec)) loop.stop() loop.close()
def startServer(self, loop: asyncio.AbstractEventLoop): """ Continuously process I/O events and handle the connections accordingly. """ server = asyncio.start_server(self.handleMessage, self.host, self.port, loop=loop) task = loop.run_until_complete(server) print('Serving on:', repr(task.sockets[0].getsockname())) # Continuously run the async loop try: loop.run_forever() except KeyboardInterrupt: print("Caught keyboard interrupt. Exiting...") # Close the server task.close() loop.run_until_complete(task.wait_closed()) loop.close()
def run_reboot_loop(loop: asyncio.AbstractEventLoop): """ Run the bot, and re-run it if it fails or disconnects. The bot will still stop if an error bubbles outside the event loop, in the case that KeyboardInterrupt is raised (Ctrl+C/SIGINT), or that sys.exit() is called. """ def reset_backoff(backoff: Backoff, sequence): if sequence == backoff.n: # don't do it if we had a retry in the meantime backoff.reset() logger.info("Welcome to KazTron v{}, booting up...".format(kaztron.__version__)) # noinspection PyBroadException try: bo_timer = Backoff(initial_time=3.0, base=1.58, max_attempts=12) wait_time = 0 while True: reset_task = loop.call_later(wait_time, reset_backoff, bo_timer, bo_timer.n) run(loop) logger.error("Bot halted unexpectedly.") reset_task.cancel() wait_time = bo_timer.next() logger.info("Restarting bot in {:.1f} seconds...".format(wait_time)) time.sleep(wait_time) logger.info("Restarting bot...") except StopIteration: logger.error("Too many failed attempts. Exiting.") sys.exit(ErrorCodes.RETRY_MAX_ATTEMPTS) except KeyboardInterrupt: # outside of runner.run logger.info("Interrupted by user. Exiting.") except Exception: logger.exception("Exception in reboot loop.") raise finally: logger.info("Exiting.") loop.close()
def close_event_loop(event_loop: asyncio.AbstractEventLoop) -> None: # give event loop one chance to finish up event_loop.stop() event_loop.run_forever() # wait for everything to finish, including tasks running in executors # this assumes that all outstanding tasks finish in a reasonable time (i.e. no infinite loops). all_tasks_fn = getattr(asyncio, "all_tasks", None) if not all_tasks_fn: all_tasks_fn = asyncio.Task.all_tasks tasks = all_tasks_fn(loop=event_loop) if tasks: gather_future = asyncio.gather(*tasks, return_exceptions=True) else: # work around bad design in gather (always uses global event loop in Python 3.8) gather_future = event_loop.create_future() gather_future.set_result([]) event_loop.run_until_complete(gather_future) # due to a bug in Python libraries, the default executor needs to be shutdown explicitly before the event loop # see http://bugs.python.org/issue28464 . this bug manifests itself in at least one way: an intermittent failure # in test_document_controller_releases_itself. reproduce by running the contents of that test in a loop of 100. _default_executor = getattr(event_loop, "_default_executor", None) if _default_executor: _default_executor.shutdown() event_loop.close()
def bruh(self, encoder, loop: asyncio.AbstractEventLoop): asyncio.set_event_loop(loop) loop.run_until_complete(encoder.encode()) loop.close()
def start_loop(loop: asyncio.AbstractEventLoop) -> None: asyncio.set_event_loop(loop) loop.run_forever() loop.close()
def async_query_handler(loop: asyncio.AbstractEventLoop): loop.run_until_complete(load_game_info()) loop.close()
def async_download_handler(url: str, save_path: str, size: int, button: tk.Button, loop: asyncio.AbstractEventLoop): button.configure(text="Downloading...") loop.run_until_complete(download_update(url, save_path, size, button)) loop.close()