Exemplo n.º 1
0
def background_reader(stream, loop: asyncio.AbstractEventLoop, callback):
    """
    Reads a stream and forwards each line to an async callback.
    """

    for line in iter(stream.readline, b''):
        loop.call_soon_threadsafe(loop.create_task, callback(line))
        def _do_waitpid(
            self,
            loop: asyncio.AbstractEventLoop,
            expected_pid: int,
            callback: _Callback,
            args: List[Any],
        ) -> None:
            assert expected_pid > 0

            try:
                pid, status = os.waitpid(expected_pid, 0)
            except ChildProcessError:
                # The child process is already reaped
                # (may happen if waitpid() is called elsewhere).
                pid = expected_pid
                returncode = 255
                logger.warning(
                    "Unknown child process pid %d, will report returncode 255",
                    pid)
            else:
                returncode = _compute_returncode(status)
                if loop.get_debug():
                    logger.debug("process %s exited with returncode %s",
                                 expected_pid, returncode)

            if loop.is_closed():
                logger.warning("Loop %r that handles pid %r is closed", loop,
                               pid)
            else:
                loop.call_soon_threadsafe(callback, pid, returncode, *args)

            self._threads.pop(expected_pid)
Exemplo n.º 3
0
def call_async(loop: AbstractEventLoop, func: Callable, *args, **kwargs):
    """
    Call the given callable in the event loop thread.

    If the call returns an awaitable, it is resolved before returning to the caller.

    If you need to pass keyword arguments named ``loop`` or ``func`` to the callable, use
    :func:`functools.partial` for that.

    :param func: a regular function or a coroutine function
    :param args: positional arguments to call with
    :param loop: the event loop in which to call the function
    :param kwargs: keyword arguments to call with
    :return: the return value of the function call

    """
    async def callback():
        try:
            retval = func(*args, **kwargs)
            if isawaitable(retval):
                retval = await retval
        except BaseException as e:
            f.set_exception(e)
        else:
            f.set_result(retval)

    f = concurrent.futures.Future()
    loop.call_soon_threadsafe(loop.create_task, callback())
    return f.result()
Exemplo n.º 4
0
def ws(event_loop: asyncio.AbstractEventLoop):
    t = Thread(target=event_loop.run_forever)
    t.start()

    async def _process_request(path, request_headers):
        if path == "/healthcheck":
            return HTTPStatus.OK, {}, b""

    def _start_ws(host: str, port: int, handler, ssl=None, sock=None):
        kwargs = {
            "process_request": _process_request,
            "ssl": ssl,
        }
        if sock:
            kwargs["sock"] = sock
        else:
            kwargs["host"] = host
            kwargs["port"] = port
        event_loop.call_soon_threadsafe(
            asyncio.ensure_future,
            websockets.serve(
                handler,
                **kwargs,
            ),
        )

    yield _start_ws
    event_loop.call_soon_threadsafe(event_loop.stop)
    t.join()
Exemplo n.º 5
0
def execute_in_loop(
    loop: AbstractEventLoop,
    coro_fn: "Callable[[], Union[Awaitable[T_co], T_co]]",
    timeout: "Optional[TimeDeltaLike]" = 30.0,
) -> T_co:
    """
    Run a coroutine in a target loop. Exceptions thrown by the coroutine are
    propagated to the caller. Must NOT be called from a coroutine on the same
    loop!

    :param loop:
    :param coro_fn:
    :param timeout: Seconds to wait for the termination of the coroutine.
    :return: The return value from the coroutine.
    """
    from functools import wraps
    from queue import Queue

    q = Queue()  # type: ignore

    def coro_fn_complete(fut):
        LOG.debug("coro_fn_complete: %s", fut)
        if fut.exception() is None:
            q.put_nowait(fut.result())
        else:
            q.put_nowait(FailedInvocation(fut.exception()))

    @wraps(coro_fn)
    def run():
        if iscoroutinefunction(coro_fn):
            fut = ensure_future(coro_fn())
            fut.add_done_callback(lambda _: coro_fn_complete(fut))

        elif isawaitable(coro_fn):
            fut = ensure_future(coro_fn)
            fut.add_done_callback(lambda _: coro_fn_complete(fut))

        elif callable(coro_fn):
            result = coro_fn()
            if isawaitable(result):
                fut = ensure_future(result)
                fut.add_done_callback(lambda _: coro_fn_complete(fut))
            else:
                q.put_nowait(result)
        else:
            raise ValueError("Received an unknown kind of callback")

    loop.call_soon_threadsafe(run)

    timeout_in_seconds = to_timedelta(timeout).total_seconds()
    result = q.get(timeout=timeout_in_seconds)

    if isinstance(result, FailedInvocation):
        raise result.ex

    return result
Exemplo n.º 6
0
def _signal_wrapper(sig, frame, loop: asyncio.AbstractEventLoop,
                    actual_handler):
    """This private function does nothing other than call the actual signal
    handler function in a way that is safe for asyncio. This function is
    called as the raw signal handler which means it is called pre-emptively,
    that's why we used ``call_soon_threadsafe`` below. The actual signal
    handler can interact with the loop in a safe way."""
    # Disable the handlers so they won't be called again.
    _clear_signal_handlers()
    loop.call_soon_threadsafe(actual_handler, loop)
Exemplo n.º 7
0
def call_asap(callback: Callable,
              *args: Any,
              context: Any = None,
              loop: asyncio.AbstractEventLoop = None) -> asyncio.Handle:
    assert loop
    if _is_unix_loop(loop):
        return _call_asap(loop, callback, *args, context=context)
    if context is not None:
        return loop.call_soon_threadsafe(callback, *args, context=context)
    return loop.call_soon_threadsafe(callback, *args)
Exemplo n.º 8
0
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None:
    """Wait for pending tasks and stop an event loop."""
    pending_tasks = set(
        asyncio.run_coroutine_threadsafe(_async_get_all_tasks(loop),
                                         loop).result(_GET_ALL_TASKS_TIMEOUT))
    pending_tasks -= set(task for task in pending_tasks if task.done())
    if pending_tasks:
        asyncio.run_coroutine_threadsafe(
            _wait_for_loop_tasks(pending_tasks),
            loop).result(_WAIT_FOR_LOOP_TASKS_TIMEOUT)
    loop.call_soon_threadsafe(loop.stop)
Exemplo n.º 9
0
def call_asap(callback: Callable,
              *args: Any,
              context: Any = None,
              loop: asyncio.AbstractEventLoop = None) -> asyncio.Handle:
    """Call function asap by pushing at the front of the line."""
    assert loop
    if _is_unix_loop(loop):
        return _call_asap(loop, callback, *args, context=context)
    if context is not None:
        return loop.call_soon_threadsafe(callback, *args, context=context)
    return loop.call_soon_threadsafe(callback, *args)
Exemplo n.º 10
0
    def __init__(self,
                 loop: asyncio.AbstractEventLoop,
                 app: ASGIApp,
                 wait_time: float = None) -> None:
        self.loop = loop
        self.app = app
        self.wait_time = wait_time

        self.sync_event = SyncEvent()
        self.async_event = AsyncEvent(loop)

        loop.call_soon_threadsafe(self._init_asgi_lock)
Exemplo n.º 11
0
Arquivo: log.py Projeto: Pehat/aiomisc
def wrap_logging_handler(handler: logging.Handler,
                         loop: asyncio.AbstractEventLoop = None,
                         buffer_size: int = 1024,
                         flush_interval: float = 0.1):
    loop = loop or asyncio.get_event_loop()

    buffered_handler = AsyncMemoryHandler(buffer_size, target=handler)

    periodic = PeriodicCallback(buffered_handler.flush_async)
    loop.call_soon_threadsafe(periodic.start, flush_interval, loop)

    return buffered_handler
Exemplo n.º 12
0
    def _in_thread(future: asyncio.Future, func: Callable,
                   loop: asyncio.AbstractEventLoop):
        try:
            result = func()
        except Exception as e:
            if future.done():
                return

            loop.call_soon_threadsafe(future.set_exception, e)
        else:
            if future.done():
                return

            loop.call_soon_threadsafe(future.set_result, result)
Exemplo n.º 13
0
    def _readloop(self, loop: AbstractEventLoop):
        while not (self.fobj.closed or self._closed.is_set()):
            # Backpressure protocol.
            self._queue_open.wait(1)
            if not self._queue_open.is_set():
                continue

            b = self._read_cancellable()
            loop.call_soon_threadsafe(self.feed, b)
            if not b:
                break

        if not self._closed.is_set():
            loop.call_soon_threadsafe(self._closed.set)
Exemplo n.º 14
0
    def __init__(
        self,
        disp: Displayable,
        token: str,
        name: str,
        loop: asyncio.AbstractEventLoop,
        cache_size=CACHE_SIZE,
        debug: bool = DEBUG_SHOW,
    ):
        self.disp = disp
        self.token = token
        self.loop = loop
        self.tiles: cachetools.LRUCache[
            Tuple[int, int, int],
            TileManager.TileRef] = cachetools.LRUCache(maxsize=cache_size)
        self.stats = ServerStats(name=name)

        if debug:
            self.debug_layer = ipyleaflet.GeoJSON(
                data=dict(type="FeatureCollection", features=[]),
                style_callback=lambda feature: dict(
                    color="yellow"
                    if feature["properties"]["speculative"] else "blue",
                    fillColor="black"
                    if feature["properties"]["cancelled"] else "green"
                    if feature["properties"]["done"] else "orange",
                ),
            )
            self._debugger_handle = loop.call_soon_threadsafe(
                loop.create_task, self._update_debug_layer())
        else:
            self._debugger_handle = None
Exemplo n.º 15
0
        def execute_requests(outstanding_requests: asyncio.Queue,
                             responses: asyncio.Queue,
                             raw_results: asyncio.Queue,
                             session: GsSession,
                             loop: asyncio.AbstractEventLoop):
            with session:
                while True:
                    requests_chunk = cls.drain_queue(outstanding_requests)
                    if not requests_chunk:
                        break

                    try:
                        responses_chunk = cls.calc_multi(requests_chunk)
                        loop.call_soon_threadsafe(responses.put_nowait, list(responses_chunk.items()))
                    except Exception as e:
                        loop.call_soon_threadsafe(raw_results.put_nowait, [(r, e) for r in requests_chunk])
Exemplo n.º 16
0
    def _thread_function(self, handler: Media.ReceivedFramesHandler,
                         loop: asyncio.AbstractEventLoop) -> None:
        def handler_wrapper(
                frs: typing.Sequence[typing.Tuple[Timestamp,
                                                  Envelope]]) -> None:
            try:
                if not self._closed:  # Don't call after closure to prevent race conditions and use-after-close.
                    handler(frs)
            except Exception as exc:
                _logger.exception(
                    "%s: Unhandled exception in the receive handler: %s; lost frames: %s",
                    self, exc, frs)

        while not self._closed:
            try:
                (
                    read_ready,
                    _,
                    _,
                ) = select.select((self._sock, self._ctl_worker), (), (),
                                  _SELECT_TIMEOUT)
                ts_mono_ns = time.monotonic_ns()

                if self._sock in read_ready:
                    frames: typing.List[typing.Tuple[Timestamp, Envelope]] = []
                    try:
                        while True:
                            frames.append(self._read_frame(ts_mono_ns))
                    except OSError as ex:
                        if ex.errno != errno.EAGAIN:
                            raise
                    loop.call_soon_threadsafe(handler_wrapper, frames)

                if self._ctl_worker in read_ready:
                    if self._ctl_worker.recv(1):  # pragma: no branch
                        break
            except Exception as ex:  # pragma: no cover
                if self._sock.fileno() < 0 or self._ctl_worker.fileno(
                ) < 0 or self._ctl_main.fileno() < 0:
                    self._closed = True
                _logger.exception("%s thread failure: %s", self, ex)
                time.sleep(
                    1)  # Is this an adequate failure management strategy?

        self._closed = True
        _logger.debug("%s thread is about to exit", self)
Exemplo n.º 17
0
def stop_event_loop_new_thread(loop: asyncio.AbstractEventLoop) -> None:
    """
    Takes an event loop, stops it and once stopped closes it

    Parameters
    ----------
    loop : asyncio.AbstractEventLoop
        The loop to stop and close

    """

    thread_id = loop._thread_id
    loop.call_soon_threadsafe(loop.stop)
    threads = enumerate()
    match = [thread for thread in threads if thread.ident == thread_id]
    if len(match) == 1:
        match[0].join(timeout=1)
Exemplo n.º 18
0
def wrap_logging_handler(handler: logging.Handler,
                         loop: asyncio.AbstractEventLoop = None,
                         buffer_size: int = 1024,
                         flush_interval: float = 0.1):
    loop = loop or asyncio.get_event_loop()

    buffered_handler = _wrap_logging_handler(handler=handler,
                                             buffer_size=buffer_size)

    flusher = Thread(target=_thread_flusher,
                     args=(buffered_handler, flush_interval, loop),
                     name="Log flusher")

    flusher.daemon = True

    loop.call_soon_threadsafe(flusher.start)
    return buffered_handler
Exemplo n.º 19
0
    def _thread_function(self, loop: asyncio.AbstractEventLoop) -> None:
        while not self._closed:
            try:
                batch = self._read_batch()
                if batch:
                    loop.call_soon_threadsafe(self._invoke_rx_handler, batch)
            except OSError as ex:
                if not self._closed:
                    _logger.exception(
                        "%s thread input/output error; stopping: %s", self, ex)
                break
            except Exception as ex:
                _logger.exception("%s thread failure: %s", self, ex)
                if not self._closed:
                    time.sleep(
                        1)  # Is this an adequate failure management strategy?

        self._closed = True
        _logger.info("%s thread is about to exit", self)
Exemplo n.º 20
0
def run_callback_threadsafe(loop: asyncio.AbstractEventLoop, func: Callable,
                            *args: Any) -> concurrent.futures.Future:
    """Send a callback to a given event loop.

    Returns a future.
    """
    future = concurrent.futures.Future()

    def callback():
        try:
            future.set_result(func(*args))
        except Exception as ex:
            if future.set_running_or_notify_cancel():
                future.set_exception(ex)
            else:
                _LOGGER.warning("Exception on future: ", exc_info=True)

    loop.call_soon_threadsafe(callback)
    return future
Exemplo n.º 21
0
    def _thread_entry_point(self, loop: asyncio.AbstractEventLoop) -> None:
        while self._sock.fileno() >= 0:
            try:
                read_ready, _, _ = select.select(
                    [self._ctl_worker, self._sock], [], [], _READ_TIMEOUT)
                if self._sock in read_ready:
                    # TODO: use socket timestamping when running on GNU/Linux (Windows does not support timestamping).
                    ts = pyuavcan.transport.Timestamp.now()

                    # Notice that we MUST create a new buffer for each received datagram to avoid race conditions.
                    # Buffer memory cannot be shared because the rest of the stack is completely zero-copy;
                    # meaning that the data we allocate here, at the very bottom of the protocol stack,
                    # is likely to be carried all the way up to the application layer without being copied.
                    data, endpoint = self._sock.recvfrom(_READ_SIZE)
                    assert len(
                        data
                    ) < _READ_SIZE, "Datagram might have been truncated"
                    source_ip = _parse_address(endpoint[0])

                    frame = UDPFrame.parse(memoryview(data))
                    _logger.debug(
                        "%r: Received UDP packet of %d bytes from %s containing frame: %s",
                        self,
                        len(data),
                        endpoint,
                        frame,
                    )
                    loop.call_soon_threadsafe(self._dispatch_frame, ts,
                                              source_ip, frame)

                if self._ctl_worker in read_ready:
                    cmd = self._ctl_worker.recv(_READ_SIZE)
                    if cmd:
                        _logger.debug(
                            "%r: Worker thread has received the stop signal: %r",
                            self, cmd)
                        break
            except Exception as ex:  # pragma: no cover
                _logger.exception(
                    "%r: Worker thread error: %s; will continue after a short nap",
                    self, ex)
                time.sleep(1)
        _logger.debug("%r: Worker thread is exiting, bye bye", self)
Exemplo n.º 22
0
def console_interface_function(client: MPDClient, loop: asyncio.AbstractEventLoop):
    """
    Simple BLOCKING function with an infinite loop inside that read commands from stdin
    and pass them to the specified instance of MPDClient
    :param client: an instance of MPDClient
    :param loop: target EventLoop
    :return: None
    """
    while True:
        data = input()  # get another command from user

        print("Hey!!!", loop, client)  # notify user that the command was read, for debugging

        if data == "exit":  # exit from infinite loop on command == "exit"
            print("Input loop interrupted")
            break

        loop.call_soon_threadsafe(pass_command_to_loop, data, client, loop)

    loop.stop()  # Stop event loop and exit from the program
Exemplo n.º 23
0
def run_build(loop: asyncio.AbstractEventLoop, picklefile: str,
              batch: List[Builder]) -> List[Tuple[str, Any]]:
    queue: Any = multiprocessing.Queue()
    p = multiprocessing.Process(target=mp_build_batch,
                                args=(queue, picklefile, batch))
    p.start()
    msgs = []
    while True:
        msg = queue.get()
        if msg is None:
            break
        if msg[0] == "LOGS":
            loop.call_soon_threadsafe(dispatch_log, msg[1])

        elif msg[0] == "RESULT":
            msgs.append(msg)

    queue.close()
    queue.join_thread()

    p.join()
    return msgs
Exemplo n.º 24
0
def _cancel_all_tasks(loop: asyncio.AbstractEventLoop,
                      is_current: bool) -> None:
    to_cancel = asyncio.all_tasks(loop)
    if not to_cancel:
        return

    done = threading.Event()
    count = len(to_cancel)

    def one_task_finished(future):
        nonlocal count
        count -= 1
        if count == 0:
            done.set()

    for task in to_cancel:
        loop.call_soon_threadsafe(task.cancel)
        task.add_done_callback(one_task_finished)

    if is_current:
        loop.run_until_complete(
            wait_for(
                asyncio.gather(*to_cancel, loop=loop, return_exceptions=True),
                TIMEOUT))
    else:
        # user was responsible for cancelling all tasks
        if not done.wait(timeout=3):
            raise TimeoutError("Could not stop event loop in time")

    for task in to_cancel:
        if task.cancelled():
            continue
        if task.exception() is not None:
            loop.call_exception_handler({
                "message": "unhandled exception during event loop shutdown",
                "exception": task.exception(),
                "task": task,
            })
Exemplo n.º 25
0
def thread_worker(loop: asyncio.AbstractEventLoop, ev: asyncio.Event):
    print("worker has started")
    time.sleep(2)
    print("setting alarm")
    loop.call_soon_threadsafe(ev.set)
Exemplo n.º 26
0
def th_ensure_future(loop: asyncio.AbstractEventLoop, task):
    def f():
        asyncio.ensure_future(task)

    loop.call_soon_threadsafe(f)