def background_reader(stream, loop: asyncio.AbstractEventLoop, callback):
"""
Reads a stream and forwards each line to an async callback.
"""
for line in iter(stream.readline, b''):
loop.call_soon_threadsafe(loop.create_task, callback(line))
def _do_waitpid(
self,
loop: asyncio.AbstractEventLoop,
expected_pid: int,
callback: _Callback,
args: List[Any],
) -> None:
assert expected_pid > 0
try:
pid, status = os.waitpid(expected_pid, 0)
except ChildProcessError:
# The child process is already reaped
# (may happen if waitpid() is called elsewhere).
pid = expected_pid
returncode = 255
logger.warning(
"Unknown child process pid %d, will report returncode 255",
pid)
else:
returncode = _compute_returncode(status)
if loop.get_debug():
logger.debug("process %s exited with returncode %s",
expected_pid, returncode)
if loop.is_closed():
logger.warning("Loop %r that handles pid %r is closed", loop,
pid)
else:
loop.call_soon_threadsafe(callback, pid, returncode, *args)
self._threads.pop(expected_pid)
def call_async(loop: AbstractEventLoop, func: Callable, *args, **kwargs):
"""
Call the given callable in the event loop thread.
If the call returns an awaitable, it is resolved before returning to the caller.
If you need to pass keyword arguments named ``loop`` or ``func`` to the callable, use
:func:`functools.partial` for that.
:param func: a regular function or a coroutine function
:param args: positional arguments to call with
:param loop: the event loop in which to call the function
:param kwargs: keyword arguments to call with
:return: the return value of the function call
"""
async def callback():
try:
retval = func(*args, **kwargs)
if isawaitable(retval):
retval = await retval
except BaseException as e:
f.set_exception(e)
else:
f.set_result(retval)
f = concurrent.futures.Future()
loop.call_soon_threadsafe(loop.create_task, callback())
return f.result()
def ws(event_loop: asyncio.AbstractEventLoop):
t = Thread(target=event_loop.run_forever)
t.start()
async def _process_request(path, request_headers):
if path == "/healthcheck":
return HTTPStatus.OK, {}, b""
def _start_ws(host: str, port: int, handler, ssl=None, sock=None):
kwargs = {
"process_request": _process_request,
"ssl": ssl,
}
if sock:
kwargs["sock"] = sock
else:
kwargs["host"] = host
kwargs["port"] = port
event_loop.call_soon_threadsafe(
asyncio.ensure_future,
websockets.serve(
handler,
**kwargs,
),
)
yield _start_ws
event_loop.call_soon_threadsafe(event_loop.stop)
t.join()
def execute_in_loop(
loop: AbstractEventLoop,
coro_fn: "Callable[[], Union[Awaitable[T_co], T_co]]",
timeout: "Optional[TimeDeltaLike]" = 30.0,
) -> T_co:
"""
Run a coroutine in a target loop. Exceptions thrown by the coroutine are
propagated to the caller. Must NOT be called from a coroutine on the same
loop!
:param loop:
:param coro_fn:
:param timeout: Seconds to wait for the termination of the coroutine.
:return: The return value from the coroutine.
"""
from functools import wraps
from queue import Queue
q = Queue() # type: ignore
def coro_fn_complete(fut):
LOG.debug("coro_fn_complete: %s", fut)
if fut.exception() is None:
q.put_nowait(fut.result())
else:
q.put_nowait(FailedInvocation(fut.exception()))
@wraps(coro_fn)
def run():
if iscoroutinefunction(coro_fn):
fut = ensure_future(coro_fn())
fut.add_done_callback(lambda _: coro_fn_complete(fut))
elif isawaitable(coro_fn):
fut = ensure_future(coro_fn)
fut.add_done_callback(lambda _: coro_fn_complete(fut))
elif callable(coro_fn):
result = coro_fn()
if isawaitable(result):
fut = ensure_future(result)
fut.add_done_callback(lambda _: coro_fn_complete(fut))
else:
q.put_nowait(result)
else:
raise ValueError("Received an unknown kind of callback")
loop.call_soon_threadsafe(run)
timeout_in_seconds = to_timedelta(timeout).total_seconds()
result = q.get(timeout=timeout_in_seconds)
if isinstance(result, FailedInvocation):
raise result.ex
return result
def _signal_wrapper(sig, frame, loop: asyncio.AbstractEventLoop,
actual_handler):
"""This private function does nothing other than call the actual signal
handler function in a way that is safe for asyncio. This function is
called as the raw signal handler which means it is called pre-emptively,
that's why we used ``call_soon_threadsafe`` below. The actual signal
handler can interact with the loop in a safe way."""
# Disable the handlers so they won't be called again.
_clear_signal_handlers()
loop.call_soon_threadsafe(actual_handler, loop)
def call_asap(callback: Callable,
*args: Any,
context: Any = None,
loop: asyncio.AbstractEventLoop = None) -> asyncio.Handle:
assert loop
if _is_unix_loop(loop):
return _call_asap(loop, callback, *args, context=context)
if context is not None:
return loop.call_soon_threadsafe(callback, *args, context=context)
return loop.call_soon_threadsafe(callback, *args)
def shutdown_loop(loop: asyncio.AbstractEventLoop) -> None:
"""Wait for pending tasks and stop an event loop."""
pending_tasks = set(
asyncio.run_coroutine_threadsafe(_async_get_all_tasks(loop),
loop).result(_GET_ALL_TASKS_TIMEOUT))
pending_tasks -= set(task for task in pending_tasks if task.done())
if pending_tasks:
asyncio.run_coroutine_threadsafe(
_wait_for_loop_tasks(pending_tasks),
loop).result(_WAIT_FOR_LOOP_TASKS_TIMEOUT)
loop.call_soon_threadsafe(loop.stop)
def call_asap(callback: Callable,
*args: Any,
context: Any = None,
loop: asyncio.AbstractEventLoop = None) -> asyncio.Handle:
"""Call function asap by pushing at the front of the line."""
assert loop
if _is_unix_loop(loop):
return _call_asap(loop, callback, *args, context=context)
if context is not None:
return loop.call_soon_threadsafe(callback, *args, context=context)
return loop.call_soon_threadsafe(callback, *args)
def __init__(self,
loop: asyncio.AbstractEventLoop,
app: ASGIApp,
wait_time: float = None) -> None:
self.loop = loop
self.app = app
self.wait_time = wait_time
self.sync_event = SyncEvent()
self.async_event = AsyncEvent(loop)
loop.call_soon_threadsafe(self._init_asgi_lock)
def wrap_logging_handler(handler: logging.Handler,
loop: asyncio.AbstractEventLoop = None,
buffer_size: int = 1024,
flush_interval: float = 0.1):
loop = loop or asyncio.get_event_loop()
buffered_handler = AsyncMemoryHandler(buffer_size, target=handler)
periodic = PeriodicCallback(buffered_handler.flush_async)
loop.call_soon_threadsafe(periodic.start, flush_interval, loop)
return buffered_handler
def _in_thread(future: asyncio.Future, func: Callable,
loop: asyncio.AbstractEventLoop):
try:
result = func()
except Exception as e:
if future.done():
return
loop.call_soon_threadsafe(future.set_exception, e)
else:
if future.done():
return
loop.call_soon_threadsafe(future.set_result, result)
def _readloop(self, loop: AbstractEventLoop):
while not (self.fobj.closed or self._closed.is_set()):
# Backpressure protocol.
self._queue_open.wait(1)
if not self._queue_open.is_set():
continue
b = self._read_cancellable()
loop.call_soon_threadsafe(self.feed, b)
if not b:
break
if not self._closed.is_set():
loop.call_soon_threadsafe(self._closed.set)
def __init__(
self,
disp: Displayable,
token: str,
name: str,
loop: asyncio.AbstractEventLoop,
cache_size=CACHE_SIZE,
debug: bool = DEBUG_SHOW,
):
self.disp = disp
self.token = token
self.loop = loop
self.tiles: cachetools.LRUCache[
Tuple[int, int, int],
TileManager.TileRef] = cachetools.LRUCache(maxsize=cache_size)
self.stats = ServerStats(name=name)
if debug:
self.debug_layer = ipyleaflet.GeoJSON(
data=dict(type="FeatureCollection", features=[]),
style_callback=lambda feature: dict(
color="yellow"
if feature["properties"]["speculative"] else "blue",
fillColor="black"
if feature["properties"]["cancelled"] else "green"
if feature["properties"]["done"] else "orange",
),
)
self._debugger_handle = loop.call_soon_threadsafe(
loop.create_task, self._update_debug_layer())
else:
self._debugger_handle = None
def execute_requests(outstanding_requests: asyncio.Queue,
responses: asyncio.Queue,
raw_results: asyncio.Queue,
session: GsSession,
loop: asyncio.AbstractEventLoop):
with session:
while True:
requests_chunk = cls.drain_queue(outstanding_requests)
if not requests_chunk:
break
try:
responses_chunk = cls.calc_multi(requests_chunk)
loop.call_soon_threadsafe(responses.put_nowait, list(responses_chunk.items()))
except Exception as e:
loop.call_soon_threadsafe(raw_results.put_nowait, [(r, e) for r in requests_chunk])
def _thread_function(self, handler: Media.ReceivedFramesHandler,
loop: asyncio.AbstractEventLoop) -> None:
def handler_wrapper(
frs: typing.Sequence[typing.Tuple[Timestamp,
Envelope]]) -> None:
try:
if not self._closed: # Don't call after closure to prevent race conditions and use-after-close.
handler(frs)
except Exception as exc:
_logger.exception(
"%s: Unhandled exception in the receive handler: %s; lost frames: %s",
self, exc, frs)
while not self._closed:
try:
(
read_ready,
_,
_,
) = select.select((self._sock, self._ctl_worker), (), (),
_SELECT_TIMEOUT)
ts_mono_ns = time.monotonic_ns()
if self._sock in read_ready:
frames: typing.List[typing.Tuple[Timestamp, Envelope]] = []
try:
while True:
frames.append(self._read_frame(ts_mono_ns))
except OSError as ex:
if ex.errno != errno.EAGAIN:
raise
loop.call_soon_threadsafe(handler_wrapper, frames)
if self._ctl_worker in read_ready:
if self._ctl_worker.recv(1): # pragma: no branch
break
except Exception as ex: # pragma: no cover
if self._sock.fileno() < 0 or self._ctl_worker.fileno(
) < 0 or self._ctl_main.fileno() < 0:
self._closed = True
_logger.exception("%s thread failure: %s", self, ex)
time.sleep(
1) # Is this an adequate failure management strategy?
self._closed = True
_logger.debug("%s thread is about to exit", self)
def stop_event_loop_new_thread(loop: asyncio.AbstractEventLoop) -> None:
"""
Takes an event loop, stops it and once stopped closes it
Parameters
----------
loop : asyncio.AbstractEventLoop
The loop to stop and close
"""
thread_id = loop._thread_id
loop.call_soon_threadsafe(loop.stop)
threads = enumerate()
match = [thread for thread in threads if thread.ident == thread_id]
if len(match) == 1:
match[0].join(timeout=1)
def wrap_logging_handler(handler: logging.Handler,
loop: asyncio.AbstractEventLoop = None,
buffer_size: int = 1024,
flush_interval: float = 0.1):
loop = loop or asyncio.get_event_loop()
buffered_handler = _wrap_logging_handler(handler=handler,
buffer_size=buffer_size)
flusher = Thread(target=_thread_flusher,
args=(buffered_handler, flush_interval, loop),
name="Log flusher")
flusher.daemon = True
loop.call_soon_threadsafe(flusher.start)
return buffered_handler
def _thread_function(self, loop: asyncio.AbstractEventLoop) -> None:
while not self._closed:
try:
batch = self._read_batch()
if batch:
loop.call_soon_threadsafe(self._invoke_rx_handler, batch)
except OSError as ex:
if not self._closed:
_logger.exception(
"%s thread input/output error; stopping: %s", self, ex)
break
except Exception as ex:
_logger.exception("%s thread failure: %s", self, ex)
if not self._closed:
time.sleep(
1) # Is this an adequate failure management strategy?
self._closed = True
_logger.info("%s thread is about to exit", self)
def run_callback_threadsafe(loop: asyncio.AbstractEventLoop, func: Callable,
*args: Any) -> concurrent.futures.Future:
"""Send a callback to a given event loop.
Returns a future.
"""
future = concurrent.futures.Future()
def callback():
try:
future.set_result(func(*args))
except Exception as ex:
if future.set_running_or_notify_cancel():
future.set_exception(ex)
else:
_LOGGER.warning("Exception on future: ", exc_info=True)
loop.call_soon_threadsafe(callback)
return future
def _thread_entry_point(self, loop: asyncio.AbstractEventLoop) -> None:
while self._sock.fileno() >= 0:
try:
read_ready, _, _ = select.select(
[self._ctl_worker, self._sock], [], [], _READ_TIMEOUT)
if self._sock in read_ready:
# TODO: use socket timestamping when running on GNU/Linux (Windows does not support timestamping).
ts = pyuavcan.transport.Timestamp.now()
# Notice that we MUST create a new buffer for each received datagram to avoid race conditions.
# Buffer memory cannot be shared because the rest of the stack is completely zero-copy;
# meaning that the data we allocate here, at the very bottom of the protocol stack,
# is likely to be carried all the way up to the application layer without being copied.
data, endpoint = self._sock.recvfrom(_READ_SIZE)
assert len(
data
) < _READ_SIZE, "Datagram might have been truncated"
source_ip = _parse_address(endpoint[0])
frame = UDPFrame.parse(memoryview(data))
_logger.debug(
"%r: Received UDP packet of %d bytes from %s containing frame: %s",
self,
len(data),
endpoint,
frame,
)
loop.call_soon_threadsafe(self._dispatch_frame, ts,
source_ip, frame)
if self._ctl_worker in read_ready:
cmd = self._ctl_worker.recv(_READ_SIZE)
if cmd:
_logger.debug(
"%r: Worker thread has received the stop signal: %r",
self, cmd)
break
except Exception as ex: # pragma: no cover
_logger.exception(
"%r: Worker thread error: %s; will continue after a short nap",
self, ex)
time.sleep(1)
_logger.debug("%r: Worker thread is exiting, bye bye", self)
def console_interface_function(client: MPDClient, loop: asyncio.AbstractEventLoop):
"""
Simple BLOCKING function with an infinite loop inside that read commands from stdin
and pass them to the specified instance of MPDClient
:param client: an instance of MPDClient
:param loop: target EventLoop
:return: None
"""
while True:
data = input() # get another command from user
print("Hey!!!", loop, client) # notify user that the command was read, for debugging
if data == "exit": # exit from infinite loop on command == "exit"
print("Input loop interrupted")
break
loop.call_soon_threadsafe(pass_command_to_loop, data, client, loop)
loop.stop() # Stop event loop and exit from the program
def run_build(loop: asyncio.AbstractEventLoop, picklefile: str,
batch: List[Builder]) -> List[Tuple[str, Any]]:
queue: Any = multiprocessing.Queue()
p = multiprocessing.Process(target=mp_build_batch,
args=(queue, picklefile, batch))
p.start()
msgs = []
while True:
msg = queue.get()
if msg is None:
break
if msg[0] == "LOGS":
loop.call_soon_threadsafe(dispatch_log, msg[1])
elif msg[0] == "RESULT":
msgs.append(msg)
queue.close()
queue.join_thread()
p.join()
return msgs
def _cancel_all_tasks(loop: asyncio.AbstractEventLoop,
is_current: bool) -> None:
to_cancel = asyncio.all_tasks(loop)
if not to_cancel:
return
done = threading.Event()
count = len(to_cancel)
def one_task_finished(future):
nonlocal count
count -= 1
if count == 0:
done.set()
for task in to_cancel:
loop.call_soon_threadsafe(task.cancel)
task.add_done_callback(one_task_finished)
if is_current:
loop.run_until_complete(
wait_for(
asyncio.gather(*to_cancel, loop=loop, return_exceptions=True),
TIMEOUT))
else:
# user was responsible for cancelling all tasks
if not done.wait(timeout=3):
raise TimeoutError("Could not stop event loop in time")
for task in to_cancel:
if task.cancelled():
continue
if task.exception() is not None:
loop.call_exception_handler({
"message": "unhandled exception during event loop shutdown",
"exception": task.exception(),
"task": task,
})
def thread_worker(loop: asyncio.AbstractEventLoop, ev: asyncio.Event):
print("worker has started")
time.sleep(2)
print("setting alarm")
loop.call_soon_threadsafe(ev.set)
def th_ensure_future(loop: asyncio.AbstractEventLoop, task):
def f():
asyncio.ensure_future(task)
loop.call_soon_threadsafe(f)