async def get(self, key: datastore.Key) -> RT: """Return the object named by key. Checks each datastore in order.""" value: typing.Optional[RT] = None exceptions: typing.List[KeyError] = [] # Take snapshot of store list so that the list will remain consistent # over the full execution of this method, even as other tasks may run # during `await`/`async with` stores: typing.List[DS] = self._stores.copy() for store in stores: try: value_: RT = await store.get(key) # type: ignore[assigment] # noqa: F821 except KeyError as exc: exceptions.append(exc) else: value = value_ break if value is None: raise trio.MultiError(exceptions) # Add model to lower stores only if isinstance(self._stores[0], datastore.abc.BinaryDatastore): result_stream = datastore.core.util.stream.TeeingReceiveStream(value) else: result_stream = datastore.core.util.stream.TeeingReceiveChannel(value) for store2 in stores: if store is store2: break result_stream.start_task_soon(run_put_task, store2, key) return result_stream
async def __aexit__(self, etype, value, tb):
"""Wait on all subactor's main routines to complete.
"""
if etype is not None:
try:
# XXX: hypothetically an error could be raised and then
# a cancel signal shows up slightly after in which case
# the `else:` block here might not complete?
# For now, shield both.
with trio.CancelScope(shield=True):
if etype in (trio.Cancelled, KeyboardInterrupt):
log.warning(f"Nursery for {current_actor().uid} was "
f"cancelled with {etype}")
else:
log.exception(f"Nursery for {current_actor().uid} "
f"errored with {etype}, ")
await self.cancel()
except trio.MultiError as merr:
if value not in merr.exceptions:
raise trio.MultiError(merr.exceptions + [value])
raise
else:
# XXX: this is effectively the (for now) lone
# cancellation/supervisor strategy which exactly
# mimicks trio's behaviour
log.debug(f"Waiting on subactors {self._children} to complete")
try:
await self.wait()
except (Exception, trio.MultiError) as err:
log.warning(f"Nursery caught {err}, cancelling")
await self.cancel()
raise
log.debug(f"Nursery teardown complete")
async def _bootstrap_fixtures_and_run_test(**kwargs):
__tracebackhide__ = True
test_ctx = TrioTestContext()
test = TrioFixture("<test {!r}>".format(testfunc.__name__),
testfunc,
kwargs,
is_test=True)
contextvars_ctx = contextvars.copy_context()
contextvars_ctx.run(canary.set, "in correct context")
async with trio.open_nursery() as nursery:
for fixture in test.register_and_collect_dependencies():
nursery.start_soon(fixture.run,
test_ctx,
contextvars_ctx,
name=fixture.name)
silent_cancellers = (test_ctx.fixtures_with_cancel -
test_ctx.fixtures_with_errors)
if silent_cancellers:
for fixture in silent_cancellers:
test_ctx.error_list.append(
RuntimeError("{} cancelled the test but didn't "
"raise an error".format(fixture.name)))
if test_ctx.error_list:
raise trio.MultiError(test_ctx.error_list)
def add_exc(exc, to_add):
if exc is None:
existing = ()
elif isinstance(exc, trio.MultiError):
existing = e.exceptions
else:
existing = (exc, )
return trio.MultiError([*existing, to_add])
def raise_unraisables():
unraisables = []
try:
orig_unraisablehook, sys.unraisablehook = sys.unraisablehook, unraisables.append
yield
finally:
sys.unraisablehook = orig_unraisablehook
if unraisables:
raise trio.MultiError([unr.exc_value for unr in unraisables])
async def test_run_trio_task_errors(monkeypatch):
async with trio_asyncio.open_loop() as loop:
# Test never getting to start the task
handle = loop.run_trio_task(trio.sleep_forever)
handle.cancel()
# Test cancelling the task
handle = loop.run_trio_task(trio.sleep_forever)
await trio.testing.wait_all_tasks_blocked()
handle.cancel()
# Helper for the rest of this test, which covers cases where
# the Trio task raises an exception
async def raise_in_aio_loop(exc):
async def raise_it():
raise exc
async with trio_asyncio.open_loop() as loop:
loop.run_trio_task(raise_it)
# We temporarily modify the default exception handler to collect
# the exceptions instead of logging or raising them
exceptions = []
def collect_exceptions(loop, context):
if context.get("exception"):
exceptions.append(context["exception"])
else:
exceptions.append(RuntimeError(context.get("message") or "unknown"))
monkeypatch.setattr(
trio_asyncio.TrioEventLoop, "default_exception_handler", collect_exceptions
)
expected = [
ValueError("hi"), ValueError("lo"), KeyError(), IndexError()
]
await raise_in_aio_loop(expected[0])
with pytest.raises(SystemExit):
await raise_in_aio_loop(SystemExit(0))
with pytest.raises(SystemExit):
await raise_in_aio_loop(trio.MultiError([expected[1], SystemExit()]))
await raise_in_aio_loop(trio.MultiError(expected[2:]))
assert exceptions == expected
def remove_exc(exc, to_remove):
if exc is None:
existing = ()
elif isinstance(exc, trio.MultiError):
existing = exc.exceptions
else:
existing = (exc, )
if to_remove not in existing:
raise ValueError("{!r} not contained in {!r}".format(to_remove, exc))
new = [e for e in existing if e is not to_remove]
if new:
return trio.MultiError(new)
def transform_multi_error(exc):
# Cancelled errors must be treated separetely
cancelled_errors, other_exceptions = split_multi_error(exc)
# No transformation needed
if not other_exceptions or not isinstance(other_exceptions,
trio.MultiError):
return exc
# Collapse non-cancelled exceptions
collapsed_errors = collapse_multi_error(other_exceptions)
if not cancelled_errors:
return collapsed_errors
return trio.MultiError([cancelled_errors, collapsed_errors])
async def run(self) -> None:
if self._run_lock.locked():
raise LifecycleError(
"Cannot run a service with the run lock already engaged. Already started?"
)
elif self.is_started:
raise LifecycleError(
"Cannot run a service which is already started.")
try:
async with self._run_lock:
async with trio.open_nursery() as system_nursery:
system_nursery.start_soon(self._handle_cancelled)
try:
async with trio.open_nursery() as task_nursery:
self._task_nursery = task_nursery
self._started.set()
self.run_task(self._service.run, name="run")
# This is hack to get the task stats correct. We don't want to
# count the `Service.run` method as a task. This is still
# imperfect as it will still count as a completed task when it
# finishes.
self._total_task_count = 0
# ***BLOCKING HERE***
# The code flow will block here until the background tasks have
# completed or cancellation occurs.
except Exception:
# Exceptions from any tasks spawned by our service will be caught by trio
# and raised here, so we store them to report together with any others we
# have already captured.
self._errors.append(cast(EXC_INFO, sys.exc_info()))
finally:
system_nursery.cancel_scope.cancel()
finally:
# We need this inside a finally because a trio.Cancelled exception may be raised
# here and it wouldn't be swalled by the 'except Exception' above.
self._finished.set()
self.logger.debug("%s: finished", self)
# This is outside of the finally block above because we don't want to suppress
# trio.Cancelled or trio.MultiError exceptions coming directly from trio.
if self.did_error:
raise trio.MultiError(
tuple(
exc_value.with_traceback(exc_tb)
for _, exc_value, exc_tb in self._errors))
async def receiver():
async with child_recv_chan:
async for i in child_recv_chan:
# Just consume all results from the channel until exhausted
pass
# And then wrap up the result and push it to the parent channel
errors = [e.error for e in result_list if isinstance(e, outcome.Error)]
if len(errors) > 0:
result = outcome.Error(trio.MultiError(errors))
else:
result = outcome.Value([o.unwrap() for o in result_list])
async with parent_send_chan:
await parent_send_chan.send(result)
def close_all():
sockets_to_close = set()
try:
yield sockets_to_close
finally:
errs = []
for sock in sockets_to_close:
try:
sock.close()
except BaseException as exc:
errs.append(exc)
if errs:
raise trio.MultiError(errs)
async def aclose(self) -> None:
"""Closes and removes all mounted datastores"""
errors: typing.List[Exception] = []
for store in self.unmount_all():
try:
await store.aclose()
except trio.Cancelled:
pass # We check for cancellation later on
except Exception as error:
errors.append(error)
# Ensure error propagation
if errors:
raise trio.MultiError(errors)
# Ensure cancellation is propagated
await trio.sleep(0)
async def test_with_setup() -> None:
async with trio.open_nursery() as nursery:
self.nursery = nursery
await self.asyncSetUp()
try:
await test(self)
except BaseException as exn:
try:
await self.asyncTearDown()
except BaseException as teardown_exn:
# have to merge the exceptions if they both throw;
# might as well do this with trio.MultiError since we have it
raise trio.MultiError([exn, teardown_exn])
else:
raise
else:
await self.asyncTearDown()
nursery.cancel_scope.cancel()
async def run(self) -> None:
if self._run_lock.locked():
raise LifecycleError(
"Cannot run a service with the run lock already engaged. Already started?"
)
elif self.is_started:
raise LifecycleError(
"Cannot run a service which is already started.")
async with self._run_lock:
async with trio.open_nursery() as system_nursery:
try:
async with trio.open_nursery() as task_nursery:
self._task_nursery = task_nursery
system_nursery.start_soon(
self._handle_cancelled,
task_nursery,
)
system_nursery.start_soon(
self._handle_stopped,
system_nursery,
)
task_nursery.start_soon(self._handle_run)
self._started.set()
# ***BLOCKING HERE***
# The code flow will block here until the background tasks have
# completed or cancellation occurs.
finally:
# Mark as having stopped
self._stopped.set()
self.logger.debug('%s stopped', self)
# If an error occured, re-raise it here
if self.did_error:
raise trio.MultiError(
tuple(
exc_value.with_traceback(exc_tb)
for _, exc_value, exc_tb in self._errors))
async def _stores_cleanup(self) -> None:
"""Closes and removes all added datastores"""
errors: typing.List[Exception] = []
while len(self._stores):
store = self._stores.pop()
try:
await store.aclose()
except trio.Cancelled:
pass # We check for cancellation later on
except Exception as error:
errors.append(error)
# Ensure error propagation
if errors:
raise trio.MultiError(errors)
# Ensure cancellation is propagated
await trio.sleep(0)
async def _bootstrap_fixtures_and_run_test(**kwargs):
__tracebackhide__ = True
test_ctx = TrioTestContext()
test = TrioFixture("<test {!r}>".format(testfunc.__name__),
testfunc,
kwargs,
is_test=True)
contextvars_ctx = contextvars.copy_context()
contextvars_ctx.run(canary.set, "in correct context")
async with trio.open_nursery() as nursery:
for fixture in test.register_and_collect_dependencies():
nursery.start_soon(fixture.run,
test_ctx,
contextvars_ctx,
name=fixture.name)
if test_ctx.error_list:
raise trio.MultiError(test_ctx.error_list)
async def stat(self, key: datastore.Key) -> MD:
"""Returns the metadata of the object named by key. Checks each
datastore in order."""
metadata: typing.Optional[MD] = None
exceptions: typing.List[KeyError] = []
# Take snapshot of store list so that the list will remain consistent
# during the following iteration, even as other tasks may run
# during `await`/`async with`
stores: typing.List[DS] = self._stores.copy()
for store in stores:
try:
metadata_: MD = await store.stat(key
) # type: ignore[assignment]
except KeyError as exc:
exceptions.append(exc)
else:
metadata = metadata_
break
if metadata is None:
raise trio.MultiError(exceptions)
return metadata
import _common
import sys
def custom_excepthook(*args):
print("custom running!")
return sys.__excepthook__(*args)
sys.excepthook = custom_excepthook
# Should warn that we'll get kinda-broken tracebacks
import trio
# The custom excepthook should run, because trio was polite and didn't
# override it
raise trio.MultiError([ValueError(), KeyError()])
async def open_tcp_listeners(port, *, host=None, backlog=None):
"""Create :class:`SocketListener` objects to listen for TCP connections.
Args:
port (int): The port to listen on.
If you use 0 as your port, then the kernel will automatically pick
an arbitrary open port. But be careful: if you use this feature when
binding to multiple IP addresses, then each IP address will get its
own random port, and the returned listeners will probably be
listening on different ports. In particular, this will happen if you
use ``host=None`` – which is the default – because in this case
:func:`open_tcp_listeners` will bind to both the IPv4 wildcard
address (``0.0.0.0``) and also the IPv6 wildcard address (``::``).
host (str, bytes-like, or None): The local interface to bind to. This is
passed to :func:`~socket.getaddrinfo` with the ``AI_PASSIVE`` flag
set.
If you want to bind to the wildcard address on both IPv4 and IPv6,
in order to accept connections on all available interfaces, then
pass ``None``. This is the default.
If you have a specific interface you want to bind to, pass its IP
address or hostname here. If a hostname resolves to multiple IP
addresses, this function will open one listener on each of them.
If you want to use only IPv4, or only IPv6, but want to accept on
all interfaces, pass the family-specific wildcard address:
``"0.0.0.0"`` for IPv4-only and ``"::"`` for IPv6-only.
backlog (int or None): The listen backlog to use. If you leave this as
``None`` then Trio will pick a good default. (Currently: whatever
your system has configured as the maximum backlog.)
Returns:
list of :class:`SocketListener`
"""
# getaddrinfo sometimes allows port=None, sometimes not (depending on
# whether host=None). And on some systems it treats "" as 0, others it
# doesn't:
# http://klickverbot.at/blog/2012/01/getaddrinfo-edge-case-behavior-on-windows-linux-and-osx/
if not isinstance(port, int):
raise TypeError("port must be an int not {!r}".format(port))
backlog = _compute_backlog(backlog)
addresses = await tsocket.getaddrinfo(host,
port,
type=tsocket.SOCK_STREAM,
flags=tsocket.AI_PASSIVE)
listeners = []
unsupported_address_families = []
try:
for family, type, proto, _, sockaddr in addresses:
try:
sock = tsocket.socket(family, type, proto)
except OSError as ex:
if ex.errno == errno.EAFNOSUPPORT:
# If a system only supports IPv4, or only IPv6, it
# is still likely that getaddrinfo will return
# both an IPv4 and an IPv6 address. As long as at
# least one of the returned addresses can be
# turned into a socket, we won't complain about a
# failure to create the other.
unsupported_address_families.append(ex)
continue
else:
raise
try:
# See https://github.com/python-trio/trio/issues/39
if sys.platform != "win32":
sock.setsockopt(tsocket.SOL_SOCKET, tsocket.SO_REUSEADDR,
1)
if family == tsocket.AF_INET6:
sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY,
1)
await sock.bind(sockaddr)
sock.listen(backlog)
listeners.append(trio.SocketListener(sock))
except:
sock.close()
raise
except:
for listener in listeners:
listener.socket.close()
raise
if unsupported_address_families and not listeners:
raise OSError(
errno.EAFNOSUPPORT,
"This system doesn't support any of the kinds of "
"socket that that address could use",
) from trio.MultiError(unsupported_address_families)
return listeners
async def open_tcp_stream(
host,
port,
*,
# No trailing comma b/c bpo-9232 (fixed in py36)
happy_eyeballs_delay=DEFAULT_DELAY):
"""Connect to the given host and port over TCP.
If the given ``host`` has multiple IP addresses associated with it, then
we have a problem: which one do we use?
One approach would be to attempt to connect to the first one, and then if
that fails, attempt to connect to the second one ... until we've tried all
of them. But the problem with this is that if the first IP address is
unreachable (for example, because it's an IPv6 address and our network
discards IPv6 packets), then we might end up waiting tens of seconds for
the first connection attempt to timeout before we try the second address.
Another approach would be to attempt to connect to all of the addresses at
the same time, in parallel, and then use whichever connection succeeds
first, abandoning the others. This would be fast, but create a lot of
unnecessary load on the network and the remote server.
This function strikes a balance between these two extremes: it works its
way through the available addresses one at a time, like the first
approach; but, if ``happy_eyeballs_delay`` seconds have passed and it's
still waiting for an attempt to succeed or fail, then it gets impatient
and starts the next connection attempt in parallel. As soon as any one
connection attempt succeeds, all the other attempts are cancelled. This
avoids unnecessary load because most connections will succeed after just
one or two attempts, but if one of the addresses is unreachable then it
doesn't slow us down too much.
This is known as a "happy eyeballs" algorithm, and our particular variant
is modelled after how Chrome connects to webservers; see `RFC 6555
<https://tools.ietf.org/html/rfc6555>`__ for more details.
Args:
host (str or bytes): The host to connect to. Can be an IPv4 address,
IPv6 address, or a hostname.
port (int): The port to connect to.
happy_eyeballs_delay (float): How many seconds to wait for each
connection attempt to succeed or fail before getting impatient and
starting another one in parallel. Set to :obj:`math.inf` if you want
to limit to only one connection attempt at a time (like
:func:`socket.create_connection`). Default: 0.3 (300 ms).
Returns:
SocketStream: a :class:`~trio.abc.Stream` connected to the given server.
Raises:
OSError: if the connection fails.
See also:
open_ssl_over_tcp_stream
"""
# To keep our public API surface smaller, rule out some cases that
# getaddrinfo will accept in some circumstances, but that act weird or
# have non-portable behavior or are just plain not useful.
# No type check on host though b/c we want to allow bytes-likes.
if host is None:
raise ValueError("host cannot be None")
if not isinstance(port, int):
raise TypeError("port must be int, not {!r}".format(port))
if happy_eyeballs_delay is None:
happy_eyeballs_delay = DEFAULT_DELAY
targets = await getaddrinfo(host, port, type=SOCK_STREAM)
# I don't think this can actually happen -- if there are no results,
# getaddrinfo should have raised OSError instead of returning an empty
# list. But let's be paranoid and handle it anyway:
if not targets:
msg = "no results found for hostname lookup: {}".format(
format_host_port(host, port))
raise OSError(msg)
reorder_for_rfc_6555_section_5_4(targets)
targets_iter = iter(targets)
# This list records all the connection failures that we ignored.
oserrors = []
# It's possible for multiple connection attempts to succeed at the ~same
# time; this list records all successful connections.
winning_sockets = []
# Sleep for the given amount of time, then kick off the next task and
# start a connection attempt. On failure, expedite the next task; on
# success, kill everything. Possible outcomes:
# - records a failure in oserrors, returns None
# - records a connected socket in winning_sockets, returns None
# - crash (raises an unexpected exception)
async def attempt_connect(nursery, previous_attempt_failed):
# Wait until either the previous attempt failed, or the timeout
# expires (unless this is the first invocation, in which case we just
# go ahead).
if previous_attempt_failed is not None:
with trio.move_on_after(happy_eyeballs_delay):
await previous_attempt_failed.wait()
# Claim our target.
try:
*socket_args, _, target_sockaddr = next(targets_iter)
except StopIteration:
return
# Then kick off the next attempt.
this_attempt_failed = trio.Event()
nursery.start_soon(attempt_connect, nursery, this_attempt_failed)
# Then make this invocation's attempt
try:
with close_on_error(socket(*socket_args)) as sock:
await sock.connect(target_sockaddr)
except OSError as exc:
# This connection attempt failed, but the next one might
# succeed. Save the error for later so we can report it if
# everything fails, and tell the next attempt that it should go
# ahead (if it hasn't already).
oserrors.append(exc)
this_attempt_failed.set()
else:
# Success! Save the winning socket and cancel all outstanding
# connection attempts.
winning_sockets.append(sock)
nursery.cancel_scope.cancel()
# Kick off the chain of connection attempts.
async with trio.open_nursery() as nursery:
nursery.start_soon(attempt_connect, nursery, None)
# All connection attempts complete, and no unexpected errors escaped. So
# at this point the oserrors and winning_sockets lists are filled in.
if winning_sockets:
first_prize = winning_sockets.pop(0)
for sock in winning_sockets:
sock.close()
return trio.SocketStream(first_prize)
else:
assert len(oserrors) == len(targets)
msg = "all attempts to connect to {} failed".format(
format_host_port(host, port))
raise OSError(msg) from trio.MultiError(oserrors)
async def _open_and_supervise_one_cancels_all_nursery(
actor: Actor, ) -> typing.AsyncGenerator[ActorNursery, None]:
# the collection of errors retreived from spawned sub-actors
errors: Dict[Tuple[str, str], Exception] = {}
# This is the outermost level "deamon actor" nursery. It is awaited
# **after** the below inner "run in actor nursery". This allows for
# handling errors that are generated by the inner nursery in
# a supervisor strategy **before** blocking indefinitely to wait for
# actors spawned in "daemon mode" (aka started using
# ``ActorNursery.start_actor()``).
# errors from this daemon actor nursery bubble up to caller
async with trio.open_nursery() as da_nursery:
try:
# This is the inner level "run in actor" nursery. It is
# awaited first since actors spawned in this way (using
# ``ActorNusery.run_in_actor()``) are expected to only
# return a single result and then complete (i.e. be canclled
# gracefully). Errors collected from these actors are
# immediately raised for handling by a supervisor strategy.
# As such if the strategy propagates any error(s) upwards
# the above "daemon actor" nursery will be notified.
async with trio.open_nursery() as ria_nursery:
anursery = ActorNursery(actor, ria_nursery, da_nursery, errors)
try:
# spawning of actors happens in the caller's scope
# after we yield upwards
yield anursery
log.runtime(f"Waiting on subactors {anursery._children} "
"to complete")
# Last bit before first nursery block ends in the case
# where we didn't error in the caller's scope
# signal all process monitor tasks to conduct
# hard join phase.
anursery._join_procs.set()
except BaseException as err:
# If we error in the root but the debugger is
# engaged we don't want to prematurely kill (and
# thus clobber access to) the local tty since it
# will make the pdb repl unusable.
# Instead try to wait for pdb to be released before
# tearing down.
await maybe_wait_for_debugger(
child_in_debug=anursery._at_least_one_child_in_debug)
# if the caller's scope errored then we activate our
# one-cancels-all supervisor strategy (don't
# worry more are coming).
anursery._join_procs.set()
try:
# XXX: hypothetically an error could be
# raised and then a cancel signal shows up
# slightly after in which case the `else:`
# block here might not complete? For now,
# shield both.
with trio.CancelScope(shield=True):
etype = type(err)
if etype in (trio.Cancelled, KeyboardInterrupt
) or (is_multi_cancelled(err)):
log.cancel(
f"Nursery for {current_actor().uid} "
f"was cancelled with {etype}")
else:
log.exception(
f"Nursery for {current_actor().uid} "
f"errored with {err}, ")
# cancel all subactors
await anursery.cancel()
except trio.MultiError as merr:
# If we receive additional errors while waiting on
# remaining subactors that were cancelled,
# aggregate those errors with the original error
# that triggered this teardown.
if err not in merr.exceptions:
raise trio.MultiError(merr.exceptions + [err])
else:
raise
# ria_nursery scope end
# XXX: do we need a `trio.Cancelled` catch here as well?
# this is the catch around the ``.run_in_actor()`` nursery
except (Exception, trio.MultiError, trio.Cancelled) as err:
# XXX: yet another guard before allowing the cancel
# sequence in case a (single) child is in debug.
await maybe_wait_for_debugger(
child_in_debug=anursery._at_least_one_child_in_debug)
# If actor-local error was raised while waiting on
# ".run_in_actor()" actors then we also want to cancel all
# remaining sub-actors (due to our lone strategy:
# one-cancels-all).
log.cancel(f"Nursery cancelling due to {err}")
if anursery._children:
with trio.CancelScope(shield=True):
await anursery.cancel()
raise
finally:
# No errors were raised while awaiting ".run_in_actor()"
# actors but those actors may have returned remote errors as
# results (meaning they errored remotely and have relayed
# those errors back to this parent actor). The errors are
# collected in ``errors`` so cancel all actors, summarize
# all errors and re-raise.
if errors:
if anursery._children:
with trio.CancelScope(shield=True):
await anursery.cancel()
# use `MultiError` as needed
if len(errors) > 1:
raise trio.MultiError(tuple(errors.values()))
else:
raise list(errors.values())[0]
def foo():
raise trio.MultiError([_cancelled(), ValueError()])
def _flush_stats_sync(self,
write_restore_file: bool = False,
expect_file: bool = True) -> None:
"""Update the filesystem stats on the disk in a way that is safe for
concurrent access
This function performs lots of synchronous I/O, call it from an I/O
thread only and ensure you hold :attr:`_stats_lock` while doing so!
The general proceedure being a follows:
1. Write the new/current stats to a temporary file
2. Atomically exchange ``diskUsage.cache`` and the temporary file
* *Fallback if atomic exchange is not available on the host OS*:
Move ``diskUsage.cache`` to new location and non-replacingly
rename temporary file to ``diskUsage.cache`` (set a flag if the
target file already exists by the time we try to move our's in)
* The primary issue with this variant is that the stats file will
be gone (moved or unlinked) for a splitsecond before the new one
has been moved in. If the program crashes between these two
actions, we will loose some stats.
3. Compare mtime of the moved/previous ``diskUsage.cache`` file to the
file's value on startup
* If they do not match update the expected value and mtime from the
moved file and retry from step 1
* In effect this constitutes a 3-way merge with our expected value
as the base, the read value as the “remote” (cause somebody else
must have written it) and our new value as “local”
4. Check flag set by the exchange fallback code and continue as in
step 3 if it is set (using the current ``diskUsage.cache`` file
as the “remote”)
5. Success
"""
assert self._stats is not None
assert self._stats_prev is not None
assert self._stats_lock.locked()
unlink_paths: typing.List[pathlib.Path] = []
try:
path = pathlib.Path(self.object_path(self.stats_key))
path_restore = pathlib.Path(str(path) + "-restore")
path_dir = path.parent
path_prefix = f".{path.name}.tmp-"
while True:
# Read restore file if it exists (works around non-cooperating
# implementations just overwriting everything blindly)
try:
path_restore_tmp = move_to_tempfile_sync(
path_restore, dir=path_dir, prefix=path_prefix)
except FileNotFoundError:
pass # No restore file currently written
else:
unlink_paths.append(path_restore_tmp)
restore_data = json.loads(path_restore_tmp.read_bytes())
restore_orig = Stats.from_json(restore_data.get(
"orig", {}))
restore_local = Stats.from_json(
restore_data.get("local", {}))
try:
with path.open() as remote_file:
restore_remote = Stats.from_json(
json.loads(remote_file.read()))
restore_remote.mtime_ns = os.fstat(
remote_file.fileno()).st_mtime_ns
# Apply delta from current file to restore data
if not restore_remote.can_merge:
restore_local.merge(restore_orig, restore_remote)
self._stats_orig = restore_remote
else:
self._stats_orig = restore_orig
except FileNotFoundError:
pass
new_stats = restore_local.copy()
# Apply delta from our value to restore data
new_stats.merge(self._stats_prev, self._stats)
self._stats = new_stats
self._stats_prev = restore_local
with tempfile.NamedTemporaryFile(mode="w",
encoding="utf-8",
dir=path_dir,
prefix=path_prefix,
delete=False) as new_file:
# Remember the file's path so that we may unlink it when
# the time comes
new_path = pathlib.Path(self.root_path / new_file.name)
unlink_paths.append(new_path)
# Write data
new_file.write(json.dumps(self._stats.to_json()))
# Also remember the mtime after writing for later collision detection
new_path_mtime_ns: int = new_path.stat().st_mtime_ns
need_move_retry: bool = False
old_path: typing.Optional[pathlib.Path] = None
try:
# Atomically exchange temporary and production file
exchange.exchange(path, new_path)
# – The temporary file now contains the previous contents
old_path = new_path
# – The target file has now been updated
self._stats.mtime_ns = new_path_mtime_ns
except (FileNotFoundError, AttributeError,
NotImplementedError) as exc:
if not isinstance(exc, FileNotFoundError):
# Fallback code in case atomic exchange is not available
#
# The primary issue with this variant is that the stats file
# will be gone (moved or unlinked) for a splitsecond before
# the new one has been moved in. If the program crashes
# between these two actions, we loose the stats.
# Move target file to temporary location
try:
old_path = move_to_tempfile_sync(
path,
wait_for_src=expect_file,
dir=path_dir,
prefix=path_prefix)
except FileNotFoundError:
old_path = None
# This code applies both to the case of atomic exchange not being
# available and the target file being non-existent
try:
# Move created temporary file with our data to
# production file location
rename_noreplace.rename_noreplace(new_path, path)
self._stats.mtime_ns = new_path_mtime_ns
except FileExistsError:
# Somebody else beat us to it, we'll need to incorporate
# their changes and retry
expect_file = True
need_move_retry = True
if old_path is not None:
# Compare timestamp of swapped out file to the expected value
old_path_mtime_ns = old_path.stat().st_mtime_ns
if old_path_mtime_ns != self._stats_prev.mtime_ns:
old_stats = Stats.from_json(
json.loads(old_path.read_bytes()))
old_stats.mtime_ns = old_path_mtime_ns
cur_stats = self._stats.copy()
# Apply delta to current stats (aka perform a “merge”)
if not old_stats.can_merge:
if self._stats_orig is not None:
self._stats.merge(self._stats_orig, old_stats)
self._stats_orig = old_stats
else:
self._stats.merge(self._stats_prev, old_stats)
# Expect the stats data we just swapped in from now on
# and request a retry soon
self._stats_prev = cur_stats
self._stats_prev.mtime_ns = new_path_mtime_ns
continue
if need_move_retry:
old_stats = Stats.from_json(json.loads(path.read_bytes()))
cur_stats = self._stats
# Apply delta to current stats (aka perform a “merge”)
self._stats.merge(self._stats_prev, old_stats)
# Expect the stats data we just swapped in from now on and
# request a retry soon
self._stats_prev = cur_stats
continue
if write_restore_file:
assert self._stats_prev is not None
assert self._stats_orig is not None
with tempfile.NamedTemporaryFile(
mode="w",
encoding="utf-8",
dir=path_dir,
prefix=path_prefix,
delete=False) as new_restore_file:
# Remember the file's path so that we may unlink it when
# the time comes
new_restore_path = pathlib.Path(self.root_path /
new_restore_file.name)
unlink_paths.append(new_restore_path)
# Write data
new_restore_file.write(
json.dumps({
"local":
self._stats.to_json(mtime=True),
"prev":
self._stats_prev.to_json(mtime=True),
"orig":
self._stats_orig.to_json(mtime=True),
}))
try:
# Move created temporary file with our data to
# production file location
rename_noreplace.rename_noreplace(
new_restore_path, path_restore)
except FileExistsError:
# Somebody else beat us to it, we'll need to incorporate
# their changes and retry
continue
# Remember current snapshot as new base and return
self._stats_prev = self._stats.copy()
if self._stats_orig is None:
self._stats_orig = self._stats.copy()
return
finally:
# Clean up all temporary files created during the above
errors: typing.List[Exception] = []
for unlink_path in unlink_paths:
try:
unlink_path.unlink()
except FileNotFoundError:
pass
except Exception as exc:
errors.append(exc)
if errors:
if len(unlink_paths) == 1:
raise errors[0]
raise trio.MultiError(errors)
async def open_nursery() -> typing.AsyncGenerator[ActorNursery, None]:
"""Create and yield a new ``ActorNursery`` to be used for spawning
structured concurrent subactors.
When an actor is spawned a new trio task is started which
invokes one of the process spawning backends to create and start
a new subprocess. These tasks are started by one of two nurseries
detailed below. The reason for spawning processes from within
a new task is because ``trio_run_in_process`` itself creates a new
internal nursery and the same task that opens a nursery **must**
close it. It turns out this approach is probably more correct
anyway since it is more clear from the following nested nurseries
which cancellation scopes correspond to each spawned subactor set.
"""
actor = current_actor()
if not actor:
raise RuntimeError("No actor instance has been defined yet?")
# the collection of errors retreived from spawned sub-actors
errors: Dict[Tuple[str, str], Exception] = {}
# This is the outermost level "deamon actor" nursery. It is awaited
# **after** the below inner "run in actor nursery". This allows for
# handling errors that are generated by the inner nursery in
# a supervisor strategy **before** blocking indefinitely to wait for
# actors spawned in "daemon mode" (aka started using
# ``ActorNursery.start_actor()``).
async with trio.open_nursery() as da_nursery:
try:
# This is the inner level "run in actor" nursery. It is
# awaited first since actors spawned in this way (using
# ``ActorNusery.run_in_actor()``) are expected to only
# return a single result and then complete (i.e. be canclled
# gracefully). Errors collected from these actors are
# immediately raised for handling by a supervisor strategy.
# As such if the strategy propagates any error(s) upwards
# the above "daemon actor" nursery will be notified.
async with trio.open_nursery() as ria_nursery:
anursery = ActorNursery(actor, ria_nursery, da_nursery, errors)
try:
# spawning of actors happens in the caller's scope
# after we yield upwards
yield anursery
log.debug(f"Waiting on subactors {anursery._children}"
"to complete")
except (BaseException, Exception) as err:
# if the caller's scope errored then we activate our
# one-cancels-all supervisor strategy (don't
# worry more are coming).
anursery._join_procs.set()
try:
# XXX: hypothetically an error could be raised and then
# a cancel signal shows up slightly after in which case
# the `else:` block here might not complete?
# For now, shield both.
with trio.CancelScope(shield=True):
if err in (trio.Cancelled, KeyboardInterrupt):
log.warning(
f"Nursery for {current_actor().uid} was "
f"cancelled with {err}")
else:
log.exception(
f"Nursery for {current_actor().uid} "
f"errored with {err}, ")
# cancel all subactors
await anursery.cancel()
except trio.MultiError as merr:
# If we receive additional errors while waiting on
# remaining subactors that were cancelled,
# aggregate those errors with the original error
# that triggered this teardown.
if err not in merr.exceptions:
raise trio.MultiError(merr.exceptions + [err])
else:
raise
# Last bit before first nursery block ends in the case
# where we didn't error in the caller's scope
log.debug(f"Waiting on all subactors to complete")
anursery._join_procs.set()
# ria_nursery scope end
except (Exception, trio.MultiError) as err:
# If actor-local error was raised while waiting on
# ".run_in_actor()" actors then we also want to cancel all
# remaining sub-actors (due to our lone strategy:
# one-cancels-all).
log.warning(f"Nursery cancelling due to {err}")
if anursery._children:
with trio.CancelScope(shield=True):
await anursery.cancel()
raise
finally:
# No errors were raised while awaiting ".run_in_actor()"
# actors but those actors may have returned remote errors as
# results (meaning they errored remotely and have relayed
# those errors back to this parent actor). The errors are
# collected in ``errors`` so cancel all actors, summarize
# all errors and re-raise.
if errors:
if anursery._children:
with trio.CancelScope(shield=True):
await anursery.cancel()
if len(errors) > 1:
raise trio.MultiError(tuple(errors.values()))
else:
raise list(errors.values())[0]
# ria_nursery scope end
log.debug(f"Nursery teardown complete")
import _common
import trio
def exc1_fn():
try:
raise ValueError
except Exception as exc:
return exc
def exc2_fn():
try:
raise KeyError
except Exception as exc:
return exc
# This should be printed nicely, because trio overrode sys.excepthook
raise trio.MultiError([exc1_fn(), exc2_fn()])
class MyExceptionBase(Exception):
pass
class MyException(MyExceptionBase):
pass
@pytest.mark.parametrize(
"context, to_raise, expected_exception",
[
# simple exception
(defer_to_cancelled(ValueError), ValueError, ValueError),
# MultiError gets deferred
(defer_to_cancelled(ValueError),
trio.MultiError([_cancelled(), ValueError()]), trio.Cancelled),
# multiple exception types
(defer_to_cancelled(ValueError, KeyError),
trio.MultiError([_cancelled(), ValueError(),
KeyError()]), trio.Cancelled),
# nested MultiError
(defer_to_cancelled(ValueError),
trio.MultiError(
[ValueError(),
trio.MultiError([_cancelled(), _cancelled()])]), trio.Cancelled),
# non-matching exception
(defer_to_cancelled(ValueError),
trio.MultiError([_cancelled(), KeyError()]), trio.MultiError),
])
async def test_defer_to_cancelled(context, to_raise, expected_exception):
with pytest.raises(expected_exception):
async def open_tcp_stream(
host,
port,
*,
# No trailing comma b/c bpo-9232 (fixed in py36)
happy_eyeballs_delay=DEFAULT_DELAY,
):
"""Connect to the given host and port over TCP.
If the given ``host`` has multiple IP addresses associated with it, then
we have a problem: which one do we use?
One approach would be to attempt to connect to the first one, and then if
that fails, attempt to connect to the second one ... until we've tried all
of them. But the problem with this is that if the first IP address is
unreachable (for example, because it's an IPv6 address and our network
discards IPv6 packets), then we might end up waiting tens of seconds for
the first connection attempt to timeout before we try the second address.
Another approach would be to attempt to connect to all of the addresses at
the same time, in parallel, and then use whichever connection succeeds
first, abandoning the others. This would be fast, but create a lot of
unnecessary load on the network and the remote server.
This function strikes a balance between these two extremes: it works its
way through the available addresses one at a time, like the first
approach; but, if ``happy_eyeballs_delay`` seconds have passed and it's
still waiting for an attempt to succeed or fail, then it gets impatient
and starts the next connection attempt in parallel. As soon as any one
connection attempt succeeds, all the other attempts are cancelled. This
avoids unnecessary load because most connections will succeed after just
one or two attempts, but if one of the addresses is unreachable then it
doesn't slow us down too much.
This is known as a "happy eyeballs" algorithm, and our particular variant
is modelled after how Chrome connects to webservers; see `RFC 6555
<https://tools.ietf.org/html/rfc6555>`__ for more details.
Args:
host (str or bytes): The host to connect to. Can be an IPv4 address,
IPv6 address, or a hostname.
port (int): The port to connect to.
happy_eyeballs_delay (float): How many seconds to wait for each
connection attempt to succeed or fail before getting impatient and
starting another one in parallel. Set to `math.inf` if you want
to limit to only one connection attempt at a time (like
:func:`socket.create_connection`). Default: 0.25 (250 ms).
Returns:
SocketStream: a :class:`~trio.abc.Stream` connected to the given server.
Raises:
OSError: if the connection fails.
See also:
open_ssl_over_tcp_stream
"""
# To keep our public API surface smaller, rule out some cases that
# getaddrinfo will accept in some circumstances, but that act weird or
# have non-portable behavior or are just plain not useful.
# No type check on host though b/c we want to allow bytes-likes.
if host is None:
raise ValueError("host cannot be None")
if not isinstance(port, int):
raise TypeError("port must be int, not {!r}".format(port))
if happy_eyeballs_delay is None:
happy_eyeballs_delay = DEFAULT_DELAY
targets = await getaddrinfo(host, port, type=SOCK_STREAM)
# I don't think this can actually happen -- if there are no results,
# getaddrinfo should have raised OSError instead of returning an empty
# list. But let's be paranoid and handle it anyway:
if not targets:
msg = "no results found for hostname lookup: {}".format(
format_host_port(host, port))
raise OSError(msg)
reorder_for_rfc_6555_section_5_4(targets)
# This list records all the connection failures that we ignored.
oserrors = []
# Keeps track of the socket that we're going to complete with,
# need to make sure this isn't automatically closed
winning_socket = None
# Try connecting to the specified address. Possible outcomes:
# - success: record connected socket in winning_socket and cancel
# concurrent attempts
# - failure: record exception in oserrors, set attempt_failed allowing
# the next connection attempt to start early
# code needs to ensure sockets can be closed appropriately in the
# face of crash or cancellation
async def attempt_connect(socket_args, sockaddr, attempt_failed):
nonlocal winning_socket
try:
sock = socket(*socket_args)
open_sockets.add(sock)
await sock.connect(sockaddr)
# Success! Save the winning socket and cancel all outstanding
# connection attempts.
winning_socket = sock
nursery.cancel_scope.cancel()
except OSError as exc:
# This connection attempt failed, but the next one might
# succeed. Save the error for later so we can report it if
# everything fails, and tell the next attempt that it should go
# ahead (if it hasn't already).
oserrors.append(exc)
attempt_failed.set()
with close_all() as open_sockets:
# nursery spawns a task for each connection attempt, will be
# cancelled by the task that gets a successful connection
async with trio.open_nursery() as nursery:
for *sa, _, addr in targets:
# create an event to indicate connection failure,
# allowing the next target to be tried early
attempt_failed = trio.Event()
nursery.start_soon(attempt_connect, sa, addr, attempt_failed)
# give this attempt at most this time before moving on
with trio.move_on_after(happy_eyeballs_delay):
await attempt_failed.wait()
# nothing succeeded
if winning_socket is None:
assert len(oserrors) == len(targets)
msg = "all attempts to connect to {} failed".format(
format_host_port(host, port))
raise OSError(msg) from trio.MultiError(oserrors)
else:
stream = trio.SocketStream(winning_socket)
open_sockets.remove(winning_socket)
return stream
async def foo():
raise trio.MultiError([trio.Cancelled._create(), ValueError()])
async def open_tcp_stream(
host,
port,
*,
happy_eyeballs_delay=DEFAULT_DELAY,
local_address=None,
):
"""Connect to the given host and port over TCP.
If the given ``host`` has multiple IP addresses associated with it, then
we have a problem: which one do we use?
One approach would be to attempt to connect to the first one, and then if
that fails, attempt to connect to the second one ... until we've tried all
of them. But the problem with this is that if the first IP address is
unreachable (for example, because it's an IPv6 address and our network
discards IPv6 packets), then we might end up waiting tens of seconds for
the first connection attempt to timeout before we try the second address.
Another approach would be to attempt to connect to all of the addresses at
the same time, in parallel, and then use whichever connection succeeds
first, abandoning the others. This would be fast, but create a lot of
unnecessary load on the network and the remote server.
This function strikes a balance between these two extremes: it works its
way through the available addresses one at a time, like the first
approach; but, if ``happy_eyeballs_delay`` seconds have passed and it's
still waiting for an attempt to succeed or fail, then it gets impatient
and starts the next connection attempt in parallel. As soon as any one
connection attempt succeeds, all the other attempts are cancelled. This
avoids unnecessary load because most connections will succeed after just
one or two attempts, but if one of the addresses is unreachable then it
doesn't slow us down too much.
This is known as a "happy eyeballs" algorithm, and our particular variant
is modelled after how Chrome connects to webservers; see `RFC 6555
<https://tools.ietf.org/html/rfc6555>`__ for more details.
Args:
host (str or bytes): The host to connect to. Can be an IPv4 address,
IPv6 address, or a hostname.
port (int): The port to connect to.
happy_eyeballs_delay (float): How many seconds to wait for each
connection attempt to succeed or fail before getting impatient and
starting another one in parallel. Set to `math.inf` if you want
to limit to only one connection attempt at a time (like
:func:`socket.create_connection`). Default: 0.25 (250 ms).
local_address (None or str): The local IP address or hostname to use as
the source for outgoing connections. If ``None``, we let the OS pick
the source IP.
This is useful in some exotic networking configurations where your
host has multiple IP addresses, and you want to force the use of a
specific one.
Note that if you pass an IPv4 ``local_address``, then you won't be
able to connect to IPv6 hosts, and vice-versa. If you want to take
advantage of this to force the use of IPv4 or IPv6 without
specifying an exact source address, you can use the IPv4 wildcard
address ``local_address="0.0.0.0"``, or the IPv6 wildcard address
``local_address="::"``.
Returns:
SocketStream: a :class:`~trio.abc.Stream` connected to the given server.
Raises:
OSError: if the connection fails.
See also:
open_ssl_over_tcp_stream
"""
# To keep our public API surface smaller, rule out some cases that
# getaddrinfo will accept in some circumstances, but that act weird or
# have non-portable behavior or are just plain not useful.
# No type check on host though b/c we want to allow bytes-likes.
if host is None:
raise ValueError("host cannot be None")
if not isinstance(port, int):
raise TypeError("port must be int, not {!r}".format(port))
if happy_eyeballs_delay is None:
happy_eyeballs_delay = DEFAULT_DELAY
targets = await getaddrinfo(host, port, type=SOCK_STREAM)
# I don't think this can actually happen -- if there are no results,
# getaddrinfo should have raised OSError instead of returning an empty
# list. But let's be paranoid and handle it anyway:
if not targets:
msg = "no results found for hostname lookup: {}".format(
format_host_port(host, port))
raise OSError(msg)
reorder_for_rfc_6555_section_5_4(targets)
# This list records all the connection failures that we ignored.
oserrors = []
# Keeps track of the socket that we're going to complete with,
# need to make sure this isn't automatically closed
winning_socket = None
# Try connecting to the specified address. Possible outcomes:
# - success: record connected socket in winning_socket and cancel
# concurrent attempts
# - failure: record exception in oserrors, set attempt_failed allowing
# the next connection attempt to start early
# code needs to ensure sockets can be closed appropriately in the
# face of crash or cancellation
async def attempt_connect(socket_args, sockaddr, attempt_failed):
nonlocal winning_socket
try:
sock = socket(*socket_args)
open_sockets.add(sock)
if local_address is not None:
# TCP connections are identified by a 4-tuple:
#
# (local IP, local port, remote IP, remote port)
#
# So if a single local IP wants to make multiple connections
# to the same (remote IP, remote port) pair, then those
# connections have to use different local ports, or else TCP
# won't be able to tell them apart. OTOH, if you have multiple
# connections to different remote IP/ports, then those
# connections can share a local port.
#
# Normally, when you call bind(), the kernel will immediately
# assign a specific local port to your socket. At this point
# the kernel doesn't know which (remote IP, remote port)
# you're going to use, so it has to pick a local port that
# *no* other connection is using. That's the only way to
# guarantee that this local port will be usable later when we
# call connect(). (Alternatively, you can set SO_REUSEADDR to
# allow multiple nascent connections to share the same port,
# but then connect() might fail with EADDRNOTAVAIL if we get
# unlucky and our TCP 4-tuple ends up colliding with another
# unrelated connection.)
#
# So calling bind() before connect() works, but it disables
# sharing of local ports. This is inefficient: it makes you
# more likely to run out of local ports.
#
# But on some versions of Linux, we can re-enable sharing of
# local ports by setting a special flag. This flag tells
# bind() to only bind the IP, and not the port. That way,
# connect() is allowed to pick the the port, and it can do a
# better job of it because it knows the remote IP/port.
try:
sock.setsockopt(trio.socket.IPPROTO_IP,
trio.socket.IP_BIND_ADDRESS_NO_PORT, 1)
except (OSError, AttributeError):
pass
try:
await sock.bind((local_address, 0))
except OSError:
raise OSError(
f"local_address={local_address!r} is incompatible "
f"with remote address {sockaddr}")
await sock.connect(sockaddr)
# Success! Save the winning socket and cancel all outstanding
# connection attempts.
winning_socket = sock
nursery.cancel_scope.cancel()
except OSError as exc:
# This connection attempt failed, but the next one might
# succeed. Save the error for later so we can report it if
# everything fails, and tell the next attempt that it should go
# ahead (if it hasn't already).
oserrors.append(exc)
attempt_failed.set()
with close_all() as open_sockets:
# nursery spawns a task for each connection attempt, will be
# cancelled by the task that gets a successful connection
async with trio.open_nursery() as nursery:
for *sa, _, addr in targets:
# create an event to indicate connection failure,
# allowing the next target to be tried early
attempt_failed = trio.Event()
nursery.start_soon(attempt_connect, sa, addr, attempt_failed)
# give this attempt at most this time before moving on
with trio.move_on_after(happy_eyeballs_delay):
await attempt_failed.wait()
# nothing succeeded
if winning_socket is None:
assert len(oserrors) == len(targets)
msg = "all attempts to connect to {} failed".format(
format_host_port(host, port))
raise OSError(msg) from trio.MultiError(oserrors)
else:
stream = trio.SocketStream(winning_socket)
open_sockets.remove(winning_socket)
return stream
async def wait(self) -> None:
"""Wait for all subactors to complete.
This is probably the most complicated (and confusing, sorry)
function that does all the clever crap to deal with cancellation,
error propagation, and graceful subprocess tear down.
"""
async def exhaust_portal(portal, actor):
"""Pull final result from portal (assuming it has one).
If the main task is an async generator do our best to consume
what's left of it.
"""
try:
log.debug(f"Waiting on final result from {actor.uid}")
final = res = await portal.result()
# if it's an async-gen then alert that we're cancelling it
if inspect.isasyncgen(res):
final = []
log.warning(
f"Blindly consuming asyncgen for {actor.uid}")
with trio.fail_after(1):
async with aclosing(res) as agen:
async for item in agen:
log.debug(f"Consuming item {item}")
final.append(item)
except (Exception, trio.MultiError) as err:
# we reraise in the parent task via a ``trio.MultiError``
return err
else:
return final
async def cancel_on_completion(
portal: Portal,
actor: Actor,
task_status=trio.TASK_STATUS_IGNORED,
) -> None:
"""Cancel actor gracefully once it's "main" portal's
result arrives.
Should only be called for actors spawned with `run_in_actor()`.
"""
with trio.CancelScope() as cs:
task_status.started(cs)
# if this call errors we store the exception for later
# in ``errors`` which will be reraised inside
# a MultiError and we still send out a cancel request
result = await exhaust_portal(portal, actor)
if isinstance(result, Exception):
errors.append(result)
log.warning(
f"Cancelling {portal.channel.uid} after error {result}"
)
else:
log.info(f"Cancelling {portal.channel.uid} gracefully")
# cancel the process now that we have a final result
await portal.cancel_actor()
# XXX: lol, this will never get run without a shield above..
# if cs.cancelled_caught:
# log.warning(
# "Result waiter was cancelled, process may have died")
async def wait_for_proc(
proc: mp.Process,
actor: Actor,
cancel_scope: Optional[trio.CancelScope] = None,
) -> None:
# TODO: timeout block here?
if proc.is_alive():
await proc_waiter(proc)
# please god don't hang
proc.join()
log.debug(f"Joined {proc}")
# indicate we are no longer managing this subactor
self._children.pop(actor.uid)
# proc terminated, cancel result waiter that may have
# been spawned in tandem if not done already
if cancel_scope:
log.warning(
f"Cancelling existing result waiter task for {actor.uid}")
cancel_scope.cancel()
log.debug(f"Waiting on all subactors to complete")
# since we pop each child subactor on termination,
# iterate a copy
children = self._children.copy()
errors: List[Exception] = []
# wait on run_in_actor() tasks, unblocks when all complete
async with trio.open_nursery() as nursery:
for subactor, proc, portal in children.values():
cs = None
# portal from ``run_in_actor()``
if portal in self._cancel_after_result_on_exit:
assert portal
cs = await nursery.start(
cancel_on_completion, portal, subactor)
# TODO: how do we handle remote host spawned actors?
nursery.start_soon(
wait_for_proc, proc, subactor, cs)
if errors:
if not self.cancelled:
# bubble up error(s) here and expect to be called again
# once the nursery has been cancelled externally (ex.
# from within __aexit__() if an error is caught around
# ``self.wait()`` then, ``self.cancel()`` is called
# immediately, in the default supervisor strat, after
# which in turn ``self.wait()`` is called again.)
raise trio.MultiError(errors)
# wait on all `start_actor()` subactors to complete
# if errors were captured above and we have not been cancelled
# then these ``start_actor()`` spawned actors will block until
# cancelled externally
children = self._children.copy()
async with trio.open_nursery() as nursery:
for subactor, proc, portal in children.values():
# TODO: how do we handle remote host spawned actors?
assert portal
nursery.start_soon(wait_for_proc, proc, subactor, cs)
log.debug(f"All subactors for {self} have terminated")
if errors:
# always raise any error if we're also cancelled
raise trio.MultiError(errors)
