def background_reader(stream, loop: asyncio.AbstractEventLoop, callback): """ Reads a stream and forwards each line to an async callback. """ for line in iter(stream.readline, b''): loop.call_soon_threadsafe(loop.create_task, callback(line))
def _do_waitpid( self, loop: asyncio.AbstractEventLoop, expected_pid: int, callback: _Callback, args: List[Any], ) -> None: assert expected_pid > 0 try: pid, status = os.waitpid(expected_pid, 0) except ChildProcessError: # The child process is already reaped # (may happen if waitpid() is called elsewhere). pid = expected_pid returncode = 255 logger.warning( "Unknown child process pid %d, will report returncode 255", pid) else: returncode = _compute_returncode(status) if loop.get_debug(): logger.debug("process %s exited with returncode %s", expected_pid, returncode) if loop.is_closed(): logger.warning("Loop %r that handles pid %r is closed", loop, pid) else: loop.call_soon_threadsafe(callback, pid, returncode, *args) self._threads.pop(expected_pid)
def call_async(loop: AbstractEventLoop, func: Callable, *args, **kwargs): """ Call the given callable in the event loop thread. If the call returns an awaitable, it is resolved before returning to the caller. If you need to pass keyword arguments named ``loop`` or ``func`` to the callable, use :func:`functools.partial` for that. :param func: a regular function or a coroutine function :param args: positional arguments to call with :param loop: the event loop in which to call the function :param kwargs: keyword arguments to call with :return: the return value of the function call """ async def callback(): try: retval = func(*args, **kwargs) if isawaitable(retval): retval = await retval except BaseException as e: f.set_exception(e) else: f.set_result(retval) f = concurrent.futures.Future() loop.call_soon_threadsafe(loop.create_task, callback()) return f.result()
def ws(event_loop: asyncio.AbstractEventLoop): t = Thread(target=event_loop.run_forever) t.start() async def _process_request(path, request_headers): if path == "/healthcheck": return HTTPStatus.OK, {}, b"" def _start_ws(host: str, port: int, handler, ssl=None, sock=None): kwargs = { "process_request": _process_request, "ssl": ssl, } if sock: kwargs["sock"] = sock else: kwargs["host"] = host kwargs["port"] = port event_loop.call_soon_threadsafe( asyncio.ensure_future, websockets.serve( handler, **kwargs, ), ) yield _start_ws event_loop.call_soon_threadsafe(event_loop.stop) t.join()
def execute_in_loop( loop: AbstractEventLoop, coro_fn: "Callable[[], Union[Awaitable[T_co], T_co]]", timeout: "Optional[TimeDeltaLike]" = 30.0, ) -> T_co: """ Run a coroutine in a target loop. Exceptions thrown by the coroutine are propagated to the caller. Must NOT be called from a coroutine on the same loop! :param loop: :param coro_fn: :param timeout: Seconds to wait for the termination of the coroutine. :return: The return value from the coroutine. """ from functools import wraps from queue import Queue q = Queue() # type: ignore def coro_fn_complete(fut): LOG.debug("coro_fn_complete: %s", fut) if fut.exception() is None: q.put_nowait(fut.result()) else: q.put_nowait(FailedInvocation(fut.exception())) @wraps(coro_fn) def run(): if iscoroutinefunction(coro_fn): fut = ensure_future(coro_fn()) fut.add_done_callback(lambda _: coro_fn_complete(fut)) elif isawaitable(coro_fn): fut = ensure_future(coro_fn) fut.add_done_callback(lambda _: coro_fn_complete(fut)) elif callable(coro_fn): result = coro_fn() if isawaitable(result): fut = ensure_future(result) fut.add_done_callback(lambda _: coro_fn_complete(fut)) else: q.put_nowait(result) else: raise ValueError("Received an unknown kind of callback") loop.call_soon_threadsafe(run) timeout_in_seconds = to_timedelta(timeout).total_seconds() result = q.get(timeout=timeout_in_seconds) if isinstance(result, FailedInvocation): raise result.ex return result
def _signal_wrapper(sig, frame, loop: asyncio.AbstractEventLoop, actual_handler): """This private function does nothing other than call the actual signal handler function in a way that is safe for asyncio. This function is called as the raw signal handler which means it is called pre-emptively, that's why we used ``call_soon_threadsafe`` below. The actual signal handler can interact with the loop in a safe way.""" # Disable the handlers so they won't be called again. _clear_signal_handlers() loop.call_soon_threadsafe(actual_handler, loop)
def call_asap(callback: Callable, *args: Any, context: Any = None, loop: asyncio.AbstractEventLoop = None) -> asyncio.Handle: assert loop if _is_unix_loop(loop): return _call_asap(loop, callback, *args, context=context) if context is not None: return loop.call_soon_threadsafe(callback, *args, context=context) return loop.call_soon_threadsafe(callback, *args)
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None: """Wait for pending tasks and stop an event loop.""" pending_tasks = set( asyncio.run_coroutine_threadsafe(_async_get_all_tasks(loop), loop).result(_GET_ALL_TASKS_TIMEOUT)) pending_tasks -= set(task for task in pending_tasks if task.done()) if pending_tasks: asyncio.run_coroutine_threadsafe( _wait_for_loop_tasks(pending_tasks), loop).result(_WAIT_FOR_LOOP_TASKS_TIMEOUT) loop.call_soon_threadsafe(loop.stop)
def call_asap(callback: Callable, *args: Any, context: Any = None, loop: asyncio.AbstractEventLoop = None) -> asyncio.Handle: """Call function asap by pushing at the front of the line.""" assert loop if _is_unix_loop(loop): return _call_asap(loop, callback, *args, context=context) if context is not None: return loop.call_soon_threadsafe(callback, *args, context=context) return loop.call_soon_threadsafe(callback, *args)
def __init__(self, loop: asyncio.AbstractEventLoop, app: ASGIApp, wait_time: float = None) -> None: self.loop = loop self.app = app self.wait_time = wait_time self.sync_event = SyncEvent() self.async_event = AsyncEvent(loop) loop.call_soon_threadsafe(self._init_asgi_lock)
def wrap_logging_handler(handler: logging.Handler, loop: asyncio.AbstractEventLoop = None, buffer_size: int = 1024, flush_interval: float = 0.1): loop = loop or asyncio.get_event_loop() buffered_handler = AsyncMemoryHandler(buffer_size, target=handler) periodic = PeriodicCallback(buffered_handler.flush_async) loop.call_soon_threadsafe(periodic.start, flush_interval, loop) return buffered_handler
def _in_thread(future: asyncio.Future, func: Callable, loop: asyncio.AbstractEventLoop): try: result = func() except Exception as e: if future.done(): return loop.call_soon_threadsafe(future.set_exception, e) else: if future.done(): return loop.call_soon_threadsafe(future.set_result, result)
def _readloop(self, loop: AbstractEventLoop): while not (self.fobj.closed or self._closed.is_set()): # Backpressure protocol. self._queue_open.wait(1) if not self._queue_open.is_set(): continue b = self._read_cancellable() loop.call_soon_threadsafe(self.feed, b) if not b: break if not self._closed.is_set(): loop.call_soon_threadsafe(self._closed.set)
def __init__( self, disp: Displayable, token: str, name: str, loop: asyncio.AbstractEventLoop, cache_size=CACHE_SIZE, debug: bool = DEBUG_SHOW, ): self.disp = disp self.token = token self.loop = loop self.tiles: cachetools.LRUCache[ Tuple[int, int, int], TileManager.TileRef] = cachetools.LRUCache(maxsize=cache_size) self.stats = ServerStats(name=name) if debug: self.debug_layer = ipyleaflet.GeoJSON( data=dict(type="FeatureCollection", features=[]), style_callback=lambda feature: dict( color="yellow" if feature["properties"]["speculative"] else "blue", fillColor="black" if feature["properties"]["cancelled"] else "green" if feature["properties"]["done"] else "orange", ), ) self._debugger_handle = loop.call_soon_threadsafe( loop.create_task, self._update_debug_layer()) else: self._debugger_handle = None
def execute_requests(outstanding_requests: asyncio.Queue, responses: asyncio.Queue, raw_results: asyncio.Queue, session: GsSession, loop: asyncio.AbstractEventLoop): with session: while True: requests_chunk = cls.drain_queue(outstanding_requests) if not requests_chunk: break try: responses_chunk = cls.calc_multi(requests_chunk) loop.call_soon_threadsafe(responses.put_nowait, list(responses_chunk.items())) except Exception as e: loop.call_soon_threadsafe(raw_results.put_nowait, [(r, e) for r in requests_chunk])
def _thread_function(self, handler: Media.ReceivedFramesHandler, loop: asyncio.AbstractEventLoop) -> None: def handler_wrapper( frs: typing.Sequence[typing.Tuple[Timestamp, Envelope]]) -> None: try: if not self._closed: # Don't call after closure to prevent race conditions and use-after-close. handler(frs) except Exception as exc: _logger.exception( "%s: Unhandled exception in the receive handler: %s; lost frames: %s", self, exc, frs) while not self._closed: try: ( read_ready, _, _, ) = select.select((self._sock, self._ctl_worker), (), (), _SELECT_TIMEOUT) ts_mono_ns = time.monotonic_ns() if self._sock in read_ready: frames: typing.List[typing.Tuple[Timestamp, Envelope]] = [] try: while True: frames.append(self._read_frame(ts_mono_ns)) except OSError as ex: if ex.errno != errno.EAGAIN: raise loop.call_soon_threadsafe(handler_wrapper, frames) if self._ctl_worker in read_ready: if self._ctl_worker.recv(1): # pragma: no branch break except Exception as ex: # pragma: no cover if self._sock.fileno() < 0 or self._ctl_worker.fileno( ) < 0 or self._ctl_main.fileno() < 0: self._closed = True _logger.exception("%s thread failure: %s", self, ex) time.sleep( 1) # Is this an adequate failure management strategy? self._closed = True _logger.debug("%s thread is about to exit", self)
def stop_event_loop_new_thread(loop: asyncio.AbstractEventLoop) -> None: """ Takes an event loop, stops it and once stopped closes it Parameters ---------- loop : asyncio.AbstractEventLoop The loop to stop and close """ thread_id = loop._thread_id loop.call_soon_threadsafe(loop.stop) threads = enumerate() match = [thread for thread in threads if thread.ident == thread_id] if len(match) == 1: match[0].join(timeout=1)
def wrap_logging_handler(handler: logging.Handler, loop: asyncio.AbstractEventLoop = None, buffer_size: int = 1024, flush_interval: float = 0.1): loop = loop or asyncio.get_event_loop() buffered_handler = _wrap_logging_handler(handler=handler, buffer_size=buffer_size) flusher = Thread(target=_thread_flusher, args=(buffered_handler, flush_interval, loop), name="Log flusher") flusher.daemon = True loop.call_soon_threadsafe(flusher.start) return buffered_handler
def _thread_function(self, loop: asyncio.AbstractEventLoop) -> None: while not self._closed: try: batch = self._read_batch() if batch: loop.call_soon_threadsafe(self._invoke_rx_handler, batch) except OSError as ex: if not self._closed: _logger.exception( "%s thread input/output error; stopping: %s", self, ex) break except Exception as ex: _logger.exception("%s thread failure: %s", self, ex) if not self._closed: time.sleep( 1) # Is this an adequate failure management strategy? self._closed = True _logger.info("%s thread is about to exit", self)
def run_callback_threadsafe(loop: asyncio.AbstractEventLoop, func: Callable, *args: Any) -> concurrent.futures.Future: """Send a callback to a given event loop. Returns a future. """ future = concurrent.futures.Future() def callback(): try: future.set_result(func(*args)) except Exception as ex: if future.set_running_or_notify_cancel(): future.set_exception(ex) else: _LOGGER.warning("Exception on future: ", exc_info=True) loop.call_soon_threadsafe(callback) return future
def _thread_entry_point(self, loop: asyncio.AbstractEventLoop) -> None: while self._sock.fileno() >= 0: try: read_ready, _, _ = select.select( [self._ctl_worker, self._sock], [], [], _READ_TIMEOUT) if self._sock in read_ready: # TODO: use socket timestamping when running on GNU/Linux (Windows does not support timestamping). ts = pyuavcan.transport.Timestamp.now() # Notice that we MUST create a new buffer for each received datagram to avoid race conditions. # Buffer memory cannot be shared because the rest of the stack is completely zero-copy; # meaning that the data we allocate here, at the very bottom of the protocol stack, # is likely to be carried all the way up to the application layer without being copied. data, endpoint = self._sock.recvfrom(_READ_SIZE) assert len( data ) < _READ_SIZE, "Datagram might have been truncated" source_ip = _parse_address(endpoint[0]) frame = UDPFrame.parse(memoryview(data)) _logger.debug( "%r: Received UDP packet of %d bytes from %s containing frame: %s", self, len(data), endpoint, frame, ) loop.call_soon_threadsafe(self._dispatch_frame, ts, source_ip, frame) if self._ctl_worker in read_ready: cmd = self._ctl_worker.recv(_READ_SIZE) if cmd: _logger.debug( "%r: Worker thread has received the stop signal: %r", self, cmd) break except Exception as ex: # pragma: no cover _logger.exception( "%r: Worker thread error: %s; will continue after a short nap", self, ex) time.sleep(1) _logger.debug("%r: Worker thread is exiting, bye bye", self)
def console_interface_function(client: MPDClient, loop: asyncio.AbstractEventLoop): """ Simple BLOCKING function with an infinite loop inside that read commands from stdin and pass them to the specified instance of MPDClient :param client: an instance of MPDClient :param loop: target EventLoop :return: None """ while True: data = input() # get another command from user print("Hey!!!", loop, client) # notify user that the command was read, for debugging if data == "exit": # exit from infinite loop on command == "exit" print("Input loop interrupted") break loop.call_soon_threadsafe(pass_command_to_loop, data, client, loop) loop.stop() # Stop event loop and exit from the program
def run_build(loop: asyncio.AbstractEventLoop, picklefile: str, batch: List[Builder]) -> List[Tuple[str, Any]]: queue: Any = multiprocessing.Queue() p = multiprocessing.Process(target=mp_build_batch, args=(queue, picklefile, batch)) p.start() msgs = [] while True: msg = queue.get() if msg is None: break if msg[0] == "LOGS": loop.call_soon_threadsafe(dispatch_log, msg[1]) elif msg[0] == "RESULT": msgs.append(msg) queue.close() queue.join_thread() p.join() return msgs
def _cancel_all_tasks(loop: asyncio.AbstractEventLoop, is_current: bool) -> None: to_cancel = asyncio.all_tasks(loop) if not to_cancel: return done = threading.Event() count = len(to_cancel) def one_task_finished(future): nonlocal count count -= 1 if count == 0: done.set() for task in to_cancel: loop.call_soon_threadsafe(task.cancel) task.add_done_callback(one_task_finished) if is_current: loop.run_until_complete( wait_for( asyncio.gather(*to_cancel, loop=loop, return_exceptions=True), TIMEOUT)) else: # user was responsible for cancelling all tasks if not done.wait(timeout=3): raise TimeoutError("Could not stop event loop in time") for task in to_cancel: if task.cancelled(): continue if task.exception() is not None: loop.call_exception_handler({ "message": "unhandled exception during event loop shutdown", "exception": task.exception(), "task": task, })
def thread_worker(loop: asyncio.AbstractEventLoop, ev: asyncio.Event): print("worker has started") time.sleep(2) print("setting alarm") loop.call_soon_threadsafe(ev.set)
def th_ensure_future(loop: asyncio.AbstractEventLoop, task): def f(): asyncio.ensure_future(task) loop.call_soon_threadsafe(f)