def run_callback_threadsafe(loop: AbstractEventLoop, callback: Callable,
*args: Any) -> concurrent.futures.Future:
"""Submit a callback object to a given event loop.
Return a concurrent.futures.Future to access the result.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError('Cannot be called from within the event loop')
future = concurrent.futures.Future() # type: concurrent.futures.Future
def run_callback() -> None:
"""Run callback and store result."""
try:
future.set_result(callback(*args))
# pylint: disable=broad-except
except Exception as exc:
if future.set_running_or_notify_cancel():
future.set_exception(exc)
else:
_LOGGER.warning("Exception on lost future: ", exc_info=True)
loop.call_soon_threadsafe(run_callback)
return future
def run_coroutine_threadsafe(
coro: Union[Coroutine, Generator],
loop: AbstractEventLoop) -> concurrent.futures.Future:
"""Submit a coroutine object to a given event loop.
Return a concurrent.futures.Future to access the result.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError('Cannot be called from within the event loop')
if not coroutines.iscoroutine(coro):
raise TypeError('A coroutine object is required')
future = concurrent.futures.Future() # type: concurrent.futures.Future
def callback() -> None:
"""Handle the call to the coroutine."""
try:
_chain_future(ensure_future(coro, loop=loop), future)
except Exception as exc: # pylint: disable=broad-except
if future.set_running_or_notify_cancel():
future.set_exception(exc)
else:
_LOGGER.warning("Exception on lost future: ", exc_info=True)
loop.call_soon_threadsafe(callback)
return future
def add_signal_handlers(loop: AbstractEventLoop):
exit_func = functools.partial(asyncio.ensure_future, agent.stop())
try:
loop.add_signal_handler(signal.SIGINT, exit_func)
loop.add_signal_handler(signal.SIGTERM, exit_func)
except NotImplementedError:
pass # Ignore if not implemented. Means this program is running in windows.
def run_callback_threadsafe(loop: AbstractEventLoop, callback: Callable[...,
T],
*args: Any) -> "concurrent.futures.Future[T]":
"""Submit a callback object to a given event loop.
Return a concurrent.futures.Future to access the result.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError("Cannot be called from within the event loop")
future: concurrent.futures.Future = concurrent.futures.Future()
def run_callback() -> None:
"""Run callback and store result."""
try:
future.set_result(callback(*args))
except Exception as exc: # pylint: disable=broad-except
if future.set_running_or_notify_cancel():
future.set_exception(exc)
else:
_LOGGER.warning("Exception on lost future: ", exc_info=True)
loop.call_soon_threadsafe(run_callback)
return future
def test_override_signal(event_loop: AbstractEventLoop) -> None:
# hack signal handler as terminate will be ignored
event_loop.add_signal_handler(SIGINT, lambda sig, frame:..., SIGINT, None)
lazy_fire_terminate_signal(0.1)
# asynchronous dummy heavy process will be terminated 0.1 sec later
event_loop.run_until_complete(dummy_inf_loop(0.3))
def run_coroutine_threadsafe(
coro: Union[Coroutine, Generator],
loop: AbstractEventLoop) -> concurrent.futures.Future:
"""Submit a coroutine object to a given event loop.
Return a concurrent.futures.Future to access the result.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError('Cannot be called from within the event loop')
if not coroutines.iscoroutine(coro):
raise TypeError('A coroutine object is required')
future = concurrent.futures.Future() # type: concurrent.futures.Future
def callback() -> None:
"""Handle the call to the coroutine."""
try:
_chain_future(ensure_future(coro, loop=loop), future)
except Exception as exc: # pylint: disable=broad-except
if future.set_running_or_notify_cancel():
future.set_exception(exc)
else:
_LOGGER.warning("Exception on lost future: ", exc_info=True)
loop.call_soon_threadsafe(callback)
return future
async def shutdown(loop: AbstractEventLoop):
tasks = [
task for task in asyncio.all_tasks(loop)
if task is not asyncio.tasks.current_task(loop)
]
list(map(lambda task: task.cancel(), tasks))
await asyncio.gather(*tasks, return_exceptions=True)
loop.stop()
def run(self, loop: AbstractEventLoop) -> None:
"""Runs the asynchronous task.
Args:
loop: The event loop on which to schedule the task.
"""
self.logger.info("%s task running", self.purpose)
loop.create_task(self.task(), name=self.purpose)
def error_handler(loop: AbstractEventLoop, context: Dict[str, Any]) -> None:
if isinstance(context["exception"], AdapterException):
try:
loop.stop()
loop.close()
except Exception as exception:
logging.debug(exception)
else:
logging.debug(context)
def start_and_run_server(loop: AbstractEventLoop, root: str) -> None:
try:
_start_server(root)
run_server(loop, root)
except Exception:
# Run null server with warning to user that pyre server cannot be started.
run_null_server(loop)
Notifications.show_pyre_initialize_error(root)
loop.run_forever()
def test_streaminglogger(event_loop: AbstractEventLoop) -> None:
logger_ = logger.StreamingLogger()
logger_.reset("xxx", "yyy")
logger_.log("test")
async def assert_logger(logger_: logger.StreamingLogger) -> None:
log = await logger_.dequeue()
assert log.log == "test"
assert log.id == "xxx"
event_loop.run_until_complete(assert_logger(logger_))
def add_socket_connection(loop: AbstractEventLoop,
root: str) -> SocketConnection:
local_root = find_local_root(original_directory=root)
socket_connection = SocketConnection(str(log_directory(root, local_root)),
"adapter.sock")
socket_connection.connect()
socket_connection.perform_handshake(_get_version(root))
# pyre-fixme[16]: `AbstractEventLoop` has no attribute `connect_accepted_socket`.
socket_reader = loop.connect_accepted_socket(SocketProtocol,
socket_connection.socket)
loop.run_until_complete(socket_reader)
return socket_connection
def test_handle_exit(
event_loop: AbstractEventLoop,
_base: Type[BaseWorker],
) -> None:
# In "_run", if terminated (called handle_exit) once in a method,
# loop will be soon broken before calling next method
event_loop.run_until_complete(_base(freq=0.01)._run(event_loop))
# no loop if handle_exit has been already called
base = _base(freq=0.01)
base.handle_exit(SIGINT, None)
base.handle_exit(SIGINT, None)
event_loop.run_until_complete(base._run(event_loop))
async def test_tasklet_one_after_another(event_loop: AbstractEventLoop):
statuses = [event_loop.create_future(), event_loop.create_future()]
@tasklet
async def set_ok(status):
status.set_result("ok!")
await set_ok(statuses[0])
await wait_for(statuses[0], timeout=1)
await set_ok(statuses[1])
await wait_for(statuses[1], timeout=1)
assert statuses[0].result() == statuses[1].result() == "ok!"
def run_callback_threadsafe(loop: AbstractEventLoop, callback: Callable[...,
_T],
*args: Any) -> concurrent.futures.Future[_T]:
"""Submit a callback object to a given event loop.
Return a concurrent.futures.Future to access the result.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError("Cannot be called from within the event loop")
future: concurrent.futures.Future[_T] = concurrent.futures.Future()
def run_callback() -> None:
"""Run callback and store result."""
try:
future.set_result(callback(*args))
except Exception as exc: # pylint: disable=broad-except
if future.set_running_or_notify_cancel():
future.set_exception(exc)
else:
_LOGGER.warning("Exception on lost future: ", exc_info=True)
loop.call_soon_threadsafe(run_callback)
if hasattr(loop, _SHUTDOWN_RUN_CALLBACK_THREADSAFE):
#
# If the final `HomeAssistant.async_block_till_done` in
# `HomeAssistant.async_stop` has already been called, the callback
# will never run and, `future.result()` will block forever which
# will prevent the thread running this code from shutting down which
# will result in a deadlock when the main thread attempts to shutdown
# the executor and `.join()` the thread running this code.
#
# To prevent this deadlock we do the following on shutdown:
#
# 1. Set the _SHUTDOWN_RUN_CALLBACK_THREADSAFE attr on this function
# by calling `shutdown_run_callback_threadsafe`
# 2. Call `hass.async_block_till_done` at least once after shutdown
# to ensure all callbacks have run
# 3. Raise an exception here to ensure `future.result()` can never be
# called and hit the deadlock since once `shutdown_run_callback_threadsafe`
# we cannot promise the callback will be executed.
#
raise RuntimeError(
"The event loop is in the process of shutting down.")
return future
def ensure_loop(loop: AbstractEventLoop = None) -> AbstractEventLoop:
"""
Use loop provided or create new if not provided or closed.
Return loop passed if its provided,not closed and not running, otherwise returns new event loop.
:param loop: optional event loop
:return: asyncio event loop
"""
try:
loop = loop or asyncio.new_event_loop()
assert not loop.is_closed()
assert not loop.is_running()
except (RuntimeError, AssertionError):
loop = asyncio.new_event_loop()
return loop
async def test_tasklet_grouping(event_loop: AbstractEventLoop):
no_tasks = 2
gates = [Event() for _ in range(no_tasks)]
statuses = [event_loop.create_future() for _ in range(no_tasks)]
def on_done(status: Future, task: Task) -> None:
status.set_result("cancelled" if task.cancelled() else "done")
def group_by(_, group: int) -> int:
return group
@tasklet
async def suspend(e: Event, group: int):
await e.wait()
suspend.group_by = group_by
(await suspend(gates[0],
group=1)).add_done_callback(partial(on_done, statuses[0]))
(await suspend(gates[1],
group=2)).add_done_callback(partial(on_done, statuses[1]))
for g in gates:
g.set()
await wait_for(gather(*statuses), timeout=1)
assert statuses[0].result() == "done"
assert statuses[1].result() == "done"
def iter_over_async(iterable: AsyncIterable[_T_co], loop: AbstractEventLoop) -> Generator[_T_co, None, None]:
# https://stackoverflow.com/questions/63587660/
iterator = iterable.__aiter__()
while True:
try:
yield loop.run_until_complete(iterator.__anext__())
except StopAsyncIteration:
break
def _cancel_tasks(to_cancel: Set['asyncio.Task[Any]'],
loop: AbstractEventLoop) -> None:
if not to_cancel:
return
for task in to_cancel:
task.cancel()
loop.run_until_complete(
gather(*to_cancel, loop=loop, return_exceptions=True))
for task in to_cancel:
if task.cancelled():
continue
if task.exception() is not None:
loop.call_exception_handler({
'message': 'unhandled exception during asyncio.run() shutdown',
'exception': task.exception(),
'task': task,
})
def create_async_task(self, loop: AbstractEventLoop) -> TaskAwaitable:
"""Return asyncio Task for task run in asyncio loop."""
self._agent.runtime.set_loop(loop)
if not isinstance(self._agent.runtime, AsyncRuntime): # pragma: nocover
raise ValueError(
"Agent runtime is not async compatible. Please use runtime_mode=async"
)
return loop.create_task(self._agent.runtime.start_and_wait_completed())
def test_queue_handler(event_loop: AbstractEventLoop) -> None:
handler = log_streamer.QueueHandler()
event_loop.run_until_complete(workflow(handler))
queues = handler._queues
assert queues.get("xxx").get_nowait() == "test-x\n"
assert queues.get("yyy").get_nowait() == "test-y\n"
assert not queues.get("zzz")
async def delete(handler):
# raise
await handler.delete("aaa")
# pop successfully
await handler.delete("xxx")
event_loop.run_until_complete(delete(handler))
assert not queues.get("xxx")
def run_server(loop: AbstractEventLoop, root: str) -> None:
socket_connection = add_socket_connection(loop, root)
stdin_pipe_reader = loop.connect_read_pipe(
lambda: AdapterProtocol(socket_connection, root), sys.stdin)
loop.run_until_complete(stdin_pipe_reader)
loop.set_exception_handler(error_handler)
loop.run_forever()
def fire_coroutine_threadsafe(coro: Coroutine, loop: AbstractEventLoop) -> None:
"""Submit a coroutine object to a given event loop.
This method does not provide a way to retrieve the result and
is intended for fire-and-forget use. This reduces the
work involved to fire the function on the loop.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError("Cannot be called from within the event loop")
if not coroutines.iscoroutine(coro):
raise TypeError("A coroutine object is required: %s" % coro)
def callback() -> None:
"""Handle the firing of a coroutine."""
ensure_future(coro, loop=loop)
loop.call_soon_threadsafe(callback)
def test_queuefilehandler(event_loop: AbstractEventLoop) -> None:
"""NOTE: QueueFileHandler add automatically if log is sent"""
with tempfile.TemporaryDirectory() as f:
logdir = Path(f) / "log"
handler = log_streamer.QueueFileHandler(str(logdir.resolve()))
event_loop.run_until_complete(workflow(handler))
with (logdir / "xxx").open() as fx:
assert fx.read() == "test-x\\n"
with (logdir / "yyy").open() as fy:
assert fy.read() == "test-y\\n"
with (logdir / "zzz").open() as fz:
assert fz.read() == "test-z\\n"
async def delete(handler):
# do nohing
await handler.delete("xxx")
event_loop.run_until_complete(delete(handler))
def fire_coroutine_threadsafe(coro: Coroutine,
loop: AbstractEventLoop) -> None:
"""Submit a coroutine object to a given event loop.
This method does not provide a way to retrieve the result and
is intended for fire-and-forget use. This reduces the
work involved to fire the function on the loop.
"""
ident = loop.__dict__.get("_thread_ident")
if ident is not None and ident == threading.get_ident():
raise RuntimeError('Cannot be called from within the event loop')
if not coroutines.iscoroutine(coro):
raise TypeError('A coroutine object is required: %s' % coro)
def callback() -> None:
"""Handle the firing of a coroutine."""
ensure_future(coro, loop=loop)
loop.call_soon_threadsafe(callback)
async def test_tasklet_executes_callback(event_loop: AbstractEventLoop):
status = event_loop.create_future()
@tasklet
async def set_code(code):
status.set_result(code)
await set_code("ok!")
result = await wait_for(status, timeout=1)
assert result == "ok!"
def __init__(self,
loop: AbstractEventLoop,
on_connect: Optional[Callable[[DatagramTransport], Awaitable]] = None,
on_data: Optional[Callable[[str, MutableMapping[str, str]], Awaitable]] = None) \
-> None:
"""Initializer."""
self.loop = loop
self.on_connect = on_connect
self.on_data = on_data
self.on_con_lost = loop.create_future()
self.transport = None # type: Optional[DatagramTransport]
async def test_tasklet_cancellation(event_loop: AbstractEventLoop):
gates = [Event(), Event()]
statuses = [event_loop.create_future(), event_loop.create_future()]
def on_done(status: Future, task: Future) -> None:
status.set_result("cancelled" if task.cancelled() else "done")
@tasklet
async def suspend(e: Event):
await e.wait()
(await suspend(gates[0])).add_done_callback(partial(on_done, statuses[0]))
# Next call should cancel the previously scheduled task
(await suspend(gates[1])).add_done_callback(partial(on_done, statuses[1]))
# Opening the gate should complete the newly scheduled task
gates[1].set()
await wait_for(gather(*statuses), timeout=1)
assert statuses[0].result() == "cancelled"
assert statuses[1].result() == "done"
def __init__(self, event_loop: AbstractEventLoop, sock: socket,
address: Any, on_msg: Callable[[str, str], None],
on_dc: Callable[[str], None]) -> None:
self.id = uuid4().hex
self.sock = sock
self.event_loop = event_loop
self.address = address
self.on_msg = on_msg
self.on_dc = on_dc
welcome_msg = dedent("""
===============================================
Hello!
You are now connected to the chat server.
Your messages will be broadcast to other users.
===============================================
""")
event_loop.create_task(self.listen_for_msgs())
event_loop.create_task(self.send_msg(welcome_msg))
logger.info(f"[{self.id}] initialized.")
def create_async_task(self, loop: AbstractEventLoop) -> TaskAwaitable:
"""
Return asyncio Task for task run in asyncio loop.
:param loop: abstract event loop
:return: task to run runtime
"""
self._agent.runtime.set_loop(loop)
if not isinstance(self._agent.runtime, AsyncRuntime):
raise ValueError(
"Agent runtime is not async compatible. Please use runtime_mode=async"
)
return loop.create_task(self._agent.runtime.run_runtime())
def run_coroutine_threadsafe(
coro,
loop: AbstractEventLoop,
) -> concurrent.futures.Future:
"""Submit a coroutine object to a given event loop.
Return a concurrent.futures.Future to access the result.
"""
if not asyncio.coroutines.iscoroutine(coro):
raise TypeError('A coroutine object is required')
future = concurrent.futures.Future()
def callback():
try:
if future.set_running_or_notify_cancel():
_chain_future(asyncio.tasks.ensure_future(coro, loop=loop),
future)
except Exception as exc:
future.set_exception(exc)
raise
loop.call_soon_threadsafe(callback)
return future
async def shutdown( # type: ignore[no-untyped-def]
loop: AbstractEventLoop,
logger: logging.Logger,
teardown: AsyncFunction,
signal=None # a named enum of ints
) -> None:
'''Cancel active tasks for shutdown'''
if signal:
logger.info(f'Received exit signal {signal.name}')
else:
logger.info('Unexpeced shutdown initiated')
await asyncio.sleep(5) # stall error loops
if teardown:
try:
await teardown()
except Exception:
logger.exception('Error during teardown function')
logger.error('Exiting uncleanly')
sys.exit(1)
tasks = [
t for t in asyncio.Task.all_tasks() if t is not asyncio.current_task()
]
logger.info(f'Cancelling {len(tasks)} tasks')
[task.cancel() for task in tasks]
try:
await asyncio.gather(*tasks, return_exceptions=True)
except Exception:
logger.exception('Error during loop task cancellation')
logger.error('Exiting uncleanly')
sys.exit(1)
loop.stop()
def run_server(loop: AbstractEventLoop, root: str) -> None:
logging.basicConfig(
filename=_get_log_file(root),
level=logging.DEBUG,
format="%(asctime)s %(message)s",
datefmt="%m/%d/%Y %I:%M:%S %p",
filemode="w",
)
logging.info("Starting adapter.")
socket_connection = add_socket_connection(loop, root)
stdin_pipe_reader = loop.connect_read_pipe(
lambda: AdapterProtocol(socket_connection, root), sys.stdin)
loop.run_until_complete(stdin_pipe_reader)
loop.set_exception_handler(error_handler)
loop.run_forever()
def sigterm_handler(logger: Logger, event_loop: AbstractEventLoop) -> None:
if event_loop.is_running():
logger.info('Received SIGTERM')
event_loop.stop()