def run_callback_threadsafe(loop: AbstractEventLoop, callback: Callable, *args: Any) -> concurrent.futures.Future: """Submit a callback object to a given event loop. Return a concurrent.futures.Future to access the result. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError('Cannot be called from within the event loop') future = concurrent.futures.Future() # type: concurrent.futures.Future def run_callback() -> None: """Run callback and store result.""" try: future.set_result(callback(*args)) # pylint: disable=broad-except except Exception as exc: if future.set_running_or_notify_cancel(): future.set_exception(exc) else: _LOGGER.warning("Exception on lost future: ", exc_info=True) loop.call_soon_threadsafe(run_callback) return future
def run_coroutine_threadsafe( coro: Union[Coroutine, Generator], loop: AbstractEventLoop) -> concurrent.futures.Future: """Submit a coroutine object to a given event loop. Return a concurrent.futures.Future to access the result. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError('Cannot be called from within the event loop') if not coroutines.iscoroutine(coro): raise TypeError('A coroutine object is required') future = concurrent.futures.Future() # type: concurrent.futures.Future def callback() -> None: """Handle the call to the coroutine.""" try: _chain_future(ensure_future(coro, loop=loop), future) except Exception as exc: # pylint: disable=broad-except if future.set_running_or_notify_cancel(): future.set_exception(exc) else: _LOGGER.warning("Exception on lost future: ", exc_info=True) loop.call_soon_threadsafe(callback) return future
def add_signal_handlers(loop: AbstractEventLoop): exit_func = functools.partial(asyncio.ensure_future, agent.stop()) try: loop.add_signal_handler(signal.SIGINT, exit_func) loop.add_signal_handler(signal.SIGTERM, exit_func) except NotImplementedError: pass # Ignore if not implemented. Means this program is running in windows.
def run_callback_threadsafe(loop: AbstractEventLoop, callback: Callable[..., T], *args: Any) -> "concurrent.futures.Future[T]": """Submit a callback object to a given event loop. Return a concurrent.futures.Future to access the result. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError("Cannot be called from within the event loop") future: concurrent.futures.Future = concurrent.futures.Future() def run_callback() -> None: """Run callback and store result.""" try: future.set_result(callback(*args)) except Exception as exc: # pylint: disable=broad-except if future.set_running_or_notify_cancel(): future.set_exception(exc) else: _LOGGER.warning("Exception on lost future: ", exc_info=True) loop.call_soon_threadsafe(run_callback) return future
def test_override_signal(event_loop: AbstractEventLoop) -> None: # hack signal handler as terminate will be ignored event_loop.add_signal_handler(SIGINT, lambda sig, frame:..., SIGINT, None) lazy_fire_terminate_signal(0.1) # asynchronous dummy heavy process will be terminated 0.1 sec later event_loop.run_until_complete(dummy_inf_loop(0.3))
def run_coroutine_threadsafe( coro: Union[Coroutine, Generator], loop: AbstractEventLoop) -> concurrent.futures.Future: """Submit a coroutine object to a given event loop. Return a concurrent.futures.Future to access the result. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError('Cannot be called from within the event loop') if not coroutines.iscoroutine(coro): raise TypeError('A coroutine object is required') future = concurrent.futures.Future() # type: concurrent.futures.Future def callback() -> None: """Handle the call to the coroutine.""" try: _chain_future(ensure_future(coro, loop=loop), future) except Exception as exc: # pylint: disable=broad-except if future.set_running_or_notify_cancel(): future.set_exception(exc) else: _LOGGER.warning("Exception on lost future: ", exc_info=True) loop.call_soon_threadsafe(callback) return future
async def shutdown(loop: AbstractEventLoop): tasks = [ task for task in asyncio.all_tasks(loop) if task is not asyncio.tasks.current_task(loop) ] list(map(lambda task: task.cancel(), tasks)) await asyncio.gather(*tasks, return_exceptions=True) loop.stop()
def run(self, loop: AbstractEventLoop) -> None: """Runs the asynchronous task. Args: loop: The event loop on which to schedule the task. """ self.logger.info("%s task running", self.purpose) loop.create_task(self.task(), name=self.purpose)
def error_handler(loop: AbstractEventLoop, context: Dict[str, Any]) -> None: if isinstance(context["exception"], AdapterException): try: loop.stop() loop.close() except Exception as exception: logging.debug(exception) else: logging.debug(context)
def start_and_run_server(loop: AbstractEventLoop, root: str) -> None: try: _start_server(root) run_server(loop, root) except Exception: # Run null server with warning to user that pyre server cannot be started. run_null_server(loop) Notifications.show_pyre_initialize_error(root) loop.run_forever()
def test_streaminglogger(event_loop: AbstractEventLoop) -> None: logger_ = logger.StreamingLogger() logger_.reset("xxx", "yyy") logger_.log("test") async def assert_logger(logger_: logger.StreamingLogger) -> None: log = await logger_.dequeue() assert log.log == "test" assert log.id == "xxx" event_loop.run_until_complete(assert_logger(logger_))
def add_socket_connection(loop: AbstractEventLoop, root: str) -> SocketConnection: local_root = find_local_root(original_directory=root) socket_connection = SocketConnection(str(log_directory(root, local_root)), "adapter.sock") socket_connection.connect() socket_connection.perform_handshake(_get_version(root)) # pyre-fixme[16]: `AbstractEventLoop` has no attribute `connect_accepted_socket`. socket_reader = loop.connect_accepted_socket(SocketProtocol, socket_connection.socket) loop.run_until_complete(socket_reader) return socket_connection
def test_handle_exit( event_loop: AbstractEventLoop, _base: Type[BaseWorker], ) -> None: # In "_run", if terminated (called handle_exit) once in a method, # loop will be soon broken before calling next method event_loop.run_until_complete(_base(freq=0.01)._run(event_loop)) # no loop if handle_exit has been already called base = _base(freq=0.01) base.handle_exit(SIGINT, None) base.handle_exit(SIGINT, None) event_loop.run_until_complete(base._run(event_loop))
async def test_tasklet_one_after_another(event_loop: AbstractEventLoop): statuses = [event_loop.create_future(), event_loop.create_future()] @tasklet async def set_ok(status): status.set_result("ok!") await set_ok(statuses[0]) await wait_for(statuses[0], timeout=1) await set_ok(statuses[1]) await wait_for(statuses[1], timeout=1) assert statuses[0].result() == statuses[1].result() == "ok!"
def run_callback_threadsafe(loop: AbstractEventLoop, callback: Callable[..., _T], *args: Any) -> concurrent.futures.Future[_T]: """Submit a callback object to a given event loop. Return a concurrent.futures.Future to access the result. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError("Cannot be called from within the event loop") future: concurrent.futures.Future[_T] = concurrent.futures.Future() def run_callback() -> None: """Run callback and store result.""" try: future.set_result(callback(*args)) except Exception as exc: # pylint: disable=broad-except if future.set_running_or_notify_cancel(): future.set_exception(exc) else: _LOGGER.warning("Exception on lost future: ", exc_info=True) loop.call_soon_threadsafe(run_callback) if hasattr(loop, _SHUTDOWN_RUN_CALLBACK_THREADSAFE): # # If the final `HomeAssistant.async_block_till_done` in # `HomeAssistant.async_stop` has already been called, the callback # will never run and, `future.result()` will block forever which # will prevent the thread running this code from shutting down which # will result in a deadlock when the main thread attempts to shutdown # the executor and `.join()` the thread running this code. # # To prevent this deadlock we do the following on shutdown: # # 1. Set the _SHUTDOWN_RUN_CALLBACK_THREADSAFE attr on this function # by calling `shutdown_run_callback_threadsafe` # 2. Call `hass.async_block_till_done` at least once after shutdown # to ensure all callbacks have run # 3. Raise an exception here to ensure `future.result()` can never be # called and hit the deadlock since once `shutdown_run_callback_threadsafe` # we cannot promise the callback will be executed. # raise RuntimeError( "The event loop is in the process of shutting down.") return future
def ensure_loop(loop: AbstractEventLoop = None) -> AbstractEventLoop: """ Use loop provided or create new if not provided or closed. Return loop passed if its provided,not closed and not running, otherwise returns new event loop. :param loop: optional event loop :return: asyncio event loop """ try: loop = loop or asyncio.new_event_loop() assert not loop.is_closed() assert not loop.is_running() except (RuntimeError, AssertionError): loop = asyncio.new_event_loop() return loop
async def test_tasklet_grouping(event_loop: AbstractEventLoop): no_tasks = 2 gates = [Event() for _ in range(no_tasks)] statuses = [event_loop.create_future() for _ in range(no_tasks)] def on_done(status: Future, task: Task) -> None: status.set_result("cancelled" if task.cancelled() else "done") def group_by(_, group: int) -> int: return group @tasklet async def suspend(e: Event, group: int): await e.wait() suspend.group_by = group_by (await suspend(gates[0], group=1)).add_done_callback(partial(on_done, statuses[0])) (await suspend(gates[1], group=2)).add_done_callback(partial(on_done, statuses[1])) for g in gates: g.set() await wait_for(gather(*statuses), timeout=1) assert statuses[0].result() == "done" assert statuses[1].result() == "done"
def iter_over_async(iterable: AsyncIterable[_T_co], loop: AbstractEventLoop) -> Generator[_T_co, None, None]: # https://stackoverflow.com/questions/63587660/ iterator = iterable.__aiter__() while True: try: yield loop.run_until_complete(iterator.__anext__()) except StopAsyncIteration: break
def _cancel_tasks(to_cancel: Set['asyncio.Task[Any]'], loop: AbstractEventLoop) -> None: if not to_cancel: return for task in to_cancel: task.cancel() loop.run_until_complete( gather(*to_cancel, loop=loop, return_exceptions=True)) for task in to_cancel: if task.cancelled(): continue if task.exception() is not None: loop.call_exception_handler({ 'message': 'unhandled exception during asyncio.run() shutdown', 'exception': task.exception(), 'task': task, })
def create_async_task(self, loop: AbstractEventLoop) -> TaskAwaitable: """Return asyncio Task for task run in asyncio loop.""" self._agent.runtime.set_loop(loop) if not isinstance(self._agent.runtime, AsyncRuntime): # pragma: nocover raise ValueError( "Agent runtime is not async compatible. Please use runtime_mode=async" ) return loop.create_task(self._agent.runtime.start_and_wait_completed())
def test_queue_handler(event_loop: AbstractEventLoop) -> None: handler = log_streamer.QueueHandler() event_loop.run_until_complete(workflow(handler)) queues = handler._queues assert queues.get("xxx").get_nowait() == "test-x\n" assert queues.get("yyy").get_nowait() == "test-y\n" assert not queues.get("zzz") async def delete(handler): # raise await handler.delete("aaa") # pop successfully await handler.delete("xxx") event_loop.run_until_complete(delete(handler)) assert not queues.get("xxx")
def run_server(loop: AbstractEventLoop, root: str) -> None: socket_connection = add_socket_connection(loop, root) stdin_pipe_reader = loop.connect_read_pipe( lambda: AdapterProtocol(socket_connection, root), sys.stdin) loop.run_until_complete(stdin_pipe_reader) loop.set_exception_handler(error_handler) loop.run_forever()
def fire_coroutine_threadsafe(coro: Coroutine, loop: AbstractEventLoop) -> None: """Submit a coroutine object to a given event loop. This method does not provide a way to retrieve the result and is intended for fire-and-forget use. This reduces the work involved to fire the function on the loop. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError("Cannot be called from within the event loop") if not coroutines.iscoroutine(coro): raise TypeError("A coroutine object is required: %s" % coro) def callback() -> None: """Handle the firing of a coroutine.""" ensure_future(coro, loop=loop) loop.call_soon_threadsafe(callback)
def test_queuefilehandler(event_loop: AbstractEventLoop) -> None: """NOTE: QueueFileHandler add automatically if log is sent""" with tempfile.TemporaryDirectory() as f: logdir = Path(f) / "log" handler = log_streamer.QueueFileHandler(str(logdir.resolve())) event_loop.run_until_complete(workflow(handler)) with (logdir / "xxx").open() as fx: assert fx.read() == "test-x\\n" with (logdir / "yyy").open() as fy: assert fy.read() == "test-y\\n" with (logdir / "zzz").open() as fz: assert fz.read() == "test-z\\n" async def delete(handler): # do nohing await handler.delete("xxx") event_loop.run_until_complete(delete(handler))
def fire_coroutine_threadsafe(coro: Coroutine, loop: AbstractEventLoop) -> None: """Submit a coroutine object to a given event loop. This method does not provide a way to retrieve the result and is intended for fire-and-forget use. This reduces the work involved to fire the function on the loop. """ ident = loop.__dict__.get("_thread_ident") if ident is not None and ident == threading.get_ident(): raise RuntimeError('Cannot be called from within the event loop') if not coroutines.iscoroutine(coro): raise TypeError('A coroutine object is required: %s' % coro) def callback() -> None: """Handle the firing of a coroutine.""" ensure_future(coro, loop=loop) loop.call_soon_threadsafe(callback)
async def test_tasklet_executes_callback(event_loop: AbstractEventLoop): status = event_loop.create_future() @tasklet async def set_code(code): status.set_result(code) await set_code("ok!") result = await wait_for(status, timeout=1) assert result == "ok!"
def __init__(self, loop: AbstractEventLoop, on_connect: Optional[Callable[[DatagramTransport], Awaitable]] = None, on_data: Optional[Callable[[str, MutableMapping[str, str]], Awaitable]] = None) \ -> None: """Initializer.""" self.loop = loop self.on_connect = on_connect self.on_data = on_data self.on_con_lost = loop.create_future() self.transport = None # type: Optional[DatagramTransport]
async def test_tasklet_cancellation(event_loop: AbstractEventLoop): gates = [Event(), Event()] statuses = [event_loop.create_future(), event_loop.create_future()] def on_done(status: Future, task: Future) -> None: status.set_result("cancelled" if task.cancelled() else "done") @tasklet async def suspend(e: Event): await e.wait() (await suspend(gates[0])).add_done_callback(partial(on_done, statuses[0])) # Next call should cancel the previously scheduled task (await suspend(gates[1])).add_done_callback(partial(on_done, statuses[1])) # Opening the gate should complete the newly scheduled task gates[1].set() await wait_for(gather(*statuses), timeout=1) assert statuses[0].result() == "cancelled" assert statuses[1].result() == "done"
def __init__(self, event_loop: AbstractEventLoop, sock: socket, address: Any, on_msg: Callable[[str, str], None], on_dc: Callable[[str], None]) -> None: self.id = uuid4().hex self.sock = sock self.event_loop = event_loop self.address = address self.on_msg = on_msg self.on_dc = on_dc welcome_msg = dedent(""" =============================================== Hello! You are now connected to the chat server. Your messages will be broadcast to other users. =============================================== """) event_loop.create_task(self.listen_for_msgs()) event_loop.create_task(self.send_msg(welcome_msg)) logger.info(f"[{self.id}] initialized.")
def create_async_task(self, loop: AbstractEventLoop) -> TaskAwaitable: """ Return asyncio Task for task run in asyncio loop. :param loop: abstract event loop :return: task to run runtime """ self._agent.runtime.set_loop(loop) if not isinstance(self._agent.runtime, AsyncRuntime): raise ValueError( "Agent runtime is not async compatible. Please use runtime_mode=async" ) return loop.create_task(self._agent.runtime.run_runtime())
def run_coroutine_threadsafe( coro, loop: AbstractEventLoop, ) -> concurrent.futures.Future: """Submit a coroutine object to a given event loop. Return a concurrent.futures.Future to access the result. """ if not asyncio.coroutines.iscoroutine(coro): raise TypeError('A coroutine object is required') future = concurrent.futures.Future() def callback(): try: if future.set_running_or_notify_cancel(): _chain_future(asyncio.tasks.ensure_future(coro, loop=loop), future) except Exception as exc: future.set_exception(exc) raise loop.call_soon_threadsafe(callback) return future
async def shutdown( # type: ignore[no-untyped-def] loop: AbstractEventLoop, logger: logging.Logger, teardown: AsyncFunction, signal=None # a named enum of ints ) -> None: '''Cancel active tasks for shutdown''' if signal: logger.info(f'Received exit signal {signal.name}') else: logger.info('Unexpeced shutdown initiated') await asyncio.sleep(5) # stall error loops if teardown: try: await teardown() except Exception: logger.exception('Error during teardown function') logger.error('Exiting uncleanly') sys.exit(1) tasks = [ t for t in asyncio.Task.all_tasks() if t is not asyncio.current_task() ] logger.info(f'Cancelling {len(tasks)} tasks') [task.cancel() for task in tasks] try: await asyncio.gather(*tasks, return_exceptions=True) except Exception: logger.exception('Error during loop task cancellation') logger.error('Exiting uncleanly') sys.exit(1) loop.stop()
def run_server(loop: AbstractEventLoop, root: str) -> None: logging.basicConfig( filename=_get_log_file(root), level=logging.DEBUG, format="%(asctime)s %(message)s", datefmt="%m/%d/%Y %I:%M:%S %p", filemode="w", ) logging.info("Starting adapter.") socket_connection = add_socket_connection(loop, root) stdin_pipe_reader = loop.connect_read_pipe( lambda: AdapterProtocol(socket_connection, root), sys.stdin) loop.run_until_complete(stdin_pipe_reader) loop.set_exception_handler(error_handler) loop.run_forever()
def sigterm_handler(logger: Logger, event_loop: AbstractEventLoop) -> None: if event_loop.is_running(): logger.info('Received SIGTERM') event_loop.stop()