async def get(self, key: datastore.Key) -> RT: """Return the object named by key. Checks each datastore in order.""" value: typing.Optional[RT] = None exceptions: typing.List[KeyError] = [] # Take snapshot of store list so that the list will remain consistent # over the full execution of this method, even as other tasks may run # during `await`/`async with` stores: typing.List[DS] = self._stores.copy() for store in stores: try: value_: RT = await store.get(key) # type: ignore[assigment] # noqa: F821 except KeyError as exc: exceptions.append(exc) else: value = value_ break if value is None: raise trio.MultiError(exceptions) # Add model to lower stores only if isinstance(self._stores[0], datastore.abc.BinaryDatastore): result_stream = datastore.core.util.stream.TeeingReceiveStream(value) else: result_stream = datastore.core.util.stream.TeeingReceiveChannel(value) for store2 in stores: if store is store2: break result_stream.start_task_soon(run_put_task, store2, key) return result_stream
async def __aexit__(self, etype, value, tb): """Wait on all subactor's main routines to complete. """ if etype is not None: try: # XXX: hypothetically an error could be raised and then # a cancel signal shows up slightly after in which case # the `else:` block here might not complete? # For now, shield both. with trio.CancelScope(shield=True): if etype in (trio.Cancelled, KeyboardInterrupt): log.warning(f"Nursery for {current_actor().uid} was " f"cancelled with {etype}") else: log.exception(f"Nursery for {current_actor().uid} " f"errored with {etype}, ") await self.cancel() except trio.MultiError as merr: if value not in merr.exceptions: raise trio.MultiError(merr.exceptions + [value]) raise else: # XXX: this is effectively the (for now) lone # cancellation/supervisor strategy which exactly # mimicks trio's behaviour log.debug(f"Waiting on subactors {self._children} to complete") try: await self.wait() except (Exception, trio.MultiError) as err: log.warning(f"Nursery caught {err}, cancelling") await self.cancel() raise log.debug(f"Nursery teardown complete")
async def _bootstrap_fixtures_and_run_test(**kwargs): __tracebackhide__ = True test_ctx = TrioTestContext() test = TrioFixture("<test {!r}>".format(testfunc.__name__), testfunc, kwargs, is_test=True) contextvars_ctx = contextvars.copy_context() contextvars_ctx.run(canary.set, "in correct context") async with trio.open_nursery() as nursery: for fixture in test.register_and_collect_dependencies(): nursery.start_soon(fixture.run, test_ctx, contextvars_ctx, name=fixture.name) silent_cancellers = (test_ctx.fixtures_with_cancel - test_ctx.fixtures_with_errors) if silent_cancellers: for fixture in silent_cancellers: test_ctx.error_list.append( RuntimeError("{} cancelled the test but didn't " "raise an error".format(fixture.name))) if test_ctx.error_list: raise trio.MultiError(test_ctx.error_list)
def add_exc(exc, to_add): if exc is None: existing = () elif isinstance(exc, trio.MultiError): existing = e.exceptions else: existing = (exc, ) return trio.MultiError([*existing, to_add])
def raise_unraisables(): unraisables = [] try: orig_unraisablehook, sys.unraisablehook = sys.unraisablehook, unraisables.append yield finally: sys.unraisablehook = orig_unraisablehook if unraisables: raise trio.MultiError([unr.exc_value for unr in unraisables])
async def test_run_trio_task_errors(monkeypatch): async with trio_asyncio.open_loop() as loop: # Test never getting to start the task handle = loop.run_trio_task(trio.sleep_forever) handle.cancel() # Test cancelling the task handle = loop.run_trio_task(trio.sleep_forever) await trio.testing.wait_all_tasks_blocked() handle.cancel() # Helper for the rest of this test, which covers cases where # the Trio task raises an exception async def raise_in_aio_loop(exc): async def raise_it(): raise exc async with trio_asyncio.open_loop() as loop: loop.run_trio_task(raise_it) # We temporarily modify the default exception handler to collect # the exceptions instead of logging or raising them exceptions = [] def collect_exceptions(loop, context): if context.get("exception"): exceptions.append(context["exception"]) else: exceptions.append(RuntimeError(context.get("message") or "unknown")) monkeypatch.setattr( trio_asyncio.TrioEventLoop, "default_exception_handler", collect_exceptions ) expected = [ ValueError("hi"), ValueError("lo"), KeyError(), IndexError() ] await raise_in_aio_loop(expected[0]) with pytest.raises(SystemExit): await raise_in_aio_loop(SystemExit(0)) with pytest.raises(SystemExit): await raise_in_aio_loop(trio.MultiError([expected[1], SystemExit()])) await raise_in_aio_loop(trio.MultiError(expected[2:])) assert exceptions == expected
def remove_exc(exc, to_remove): if exc is None: existing = () elif isinstance(exc, trio.MultiError): existing = exc.exceptions else: existing = (exc, ) if to_remove not in existing: raise ValueError("{!r} not contained in {!r}".format(to_remove, exc)) new = [e for e in existing if e is not to_remove] if new: return trio.MultiError(new)
def transform_multi_error(exc): # Cancelled errors must be treated separetely cancelled_errors, other_exceptions = split_multi_error(exc) # No transformation needed if not other_exceptions or not isinstance(other_exceptions, trio.MultiError): return exc # Collapse non-cancelled exceptions collapsed_errors = collapse_multi_error(other_exceptions) if not cancelled_errors: return collapsed_errors return trio.MultiError([cancelled_errors, collapsed_errors])
async def run(self) -> None: if self._run_lock.locked(): raise LifecycleError( "Cannot run a service with the run lock already engaged. Already started?" ) elif self.is_started: raise LifecycleError( "Cannot run a service which is already started.") try: async with self._run_lock: async with trio.open_nursery() as system_nursery: system_nursery.start_soon(self._handle_cancelled) try: async with trio.open_nursery() as task_nursery: self._task_nursery = task_nursery self._started.set() self.run_task(self._service.run, name="run") # This is hack to get the task stats correct. We don't want to # count the `Service.run` method as a task. This is still # imperfect as it will still count as a completed task when it # finishes. self._total_task_count = 0 # ***BLOCKING HERE*** # The code flow will block here until the background tasks have # completed or cancellation occurs. except Exception: # Exceptions from any tasks spawned by our service will be caught by trio # and raised here, so we store them to report together with any others we # have already captured. self._errors.append(cast(EXC_INFO, sys.exc_info())) finally: system_nursery.cancel_scope.cancel() finally: # We need this inside a finally because a trio.Cancelled exception may be raised # here and it wouldn't be swalled by the 'except Exception' above. self._finished.set() self.logger.debug("%s: finished", self) # This is outside of the finally block above because we don't want to suppress # trio.Cancelled or trio.MultiError exceptions coming directly from trio. if self.did_error: raise trio.MultiError( tuple( exc_value.with_traceback(exc_tb) for _, exc_value, exc_tb in self._errors))
async def receiver(): async with child_recv_chan: async for i in child_recv_chan: # Just consume all results from the channel until exhausted pass # And then wrap up the result and push it to the parent channel errors = [e.error for e in result_list if isinstance(e, outcome.Error)] if len(errors) > 0: result = outcome.Error(trio.MultiError(errors)) else: result = outcome.Value([o.unwrap() for o in result_list]) async with parent_send_chan: await parent_send_chan.send(result)
def close_all(): sockets_to_close = set() try: yield sockets_to_close finally: errs = [] for sock in sockets_to_close: try: sock.close() except BaseException as exc: errs.append(exc) if errs: raise trio.MultiError(errs)
async def aclose(self) -> None: """Closes and removes all mounted datastores""" errors: typing.List[Exception] = [] for store in self.unmount_all(): try: await store.aclose() except trio.Cancelled: pass # We check for cancellation later on except Exception as error: errors.append(error) # Ensure error propagation if errors: raise trio.MultiError(errors) # Ensure cancellation is propagated await trio.sleep(0)
async def test_with_setup() -> None: async with trio.open_nursery() as nursery: self.nursery = nursery await self.asyncSetUp() try: await test(self) except BaseException as exn: try: await self.asyncTearDown() except BaseException as teardown_exn: # have to merge the exceptions if they both throw; # might as well do this with trio.MultiError since we have it raise trio.MultiError([exn, teardown_exn]) else: raise else: await self.asyncTearDown() nursery.cancel_scope.cancel()
async def run(self) -> None: if self._run_lock.locked(): raise LifecycleError( "Cannot run a service with the run lock already engaged. Already started?" ) elif self.is_started: raise LifecycleError( "Cannot run a service which is already started.") async with self._run_lock: async with trio.open_nursery() as system_nursery: try: async with trio.open_nursery() as task_nursery: self._task_nursery = task_nursery system_nursery.start_soon( self._handle_cancelled, task_nursery, ) system_nursery.start_soon( self._handle_stopped, system_nursery, ) task_nursery.start_soon(self._handle_run) self._started.set() # ***BLOCKING HERE*** # The code flow will block here until the background tasks have # completed or cancellation occurs. finally: # Mark as having stopped self._stopped.set() self.logger.debug('%s stopped', self) # If an error occured, re-raise it here if self.did_error: raise trio.MultiError( tuple( exc_value.with_traceback(exc_tb) for _, exc_value, exc_tb in self._errors))
async def _stores_cleanup(self) -> None: """Closes and removes all added datastores""" errors: typing.List[Exception] = [] while len(self._stores): store = self._stores.pop() try: await store.aclose() except trio.Cancelled: pass # We check for cancellation later on except Exception as error: errors.append(error) # Ensure error propagation if errors: raise trio.MultiError(errors) # Ensure cancellation is propagated await trio.sleep(0)
async def _bootstrap_fixtures_and_run_test(**kwargs): __tracebackhide__ = True test_ctx = TrioTestContext() test = TrioFixture("<test {!r}>".format(testfunc.__name__), testfunc, kwargs, is_test=True) contextvars_ctx = contextvars.copy_context() contextvars_ctx.run(canary.set, "in correct context") async with trio.open_nursery() as nursery: for fixture in test.register_and_collect_dependencies(): nursery.start_soon(fixture.run, test_ctx, contextvars_ctx, name=fixture.name) if test_ctx.error_list: raise trio.MultiError(test_ctx.error_list)
async def stat(self, key: datastore.Key) -> MD: """Returns the metadata of the object named by key. Checks each datastore in order.""" metadata: typing.Optional[MD] = None exceptions: typing.List[KeyError] = [] # Take snapshot of store list so that the list will remain consistent # during the following iteration, even as other tasks may run # during `await`/`async with` stores: typing.List[DS] = self._stores.copy() for store in stores: try: metadata_: MD = await store.stat(key ) # type: ignore[assignment] except KeyError as exc: exceptions.append(exc) else: metadata = metadata_ break if metadata is None: raise trio.MultiError(exceptions) return metadata
import _common import sys def custom_excepthook(*args): print("custom running!") return sys.__excepthook__(*args) sys.excepthook = custom_excepthook # Should warn that we'll get kinda-broken tracebacks import trio # The custom excepthook should run, because trio was polite and didn't # override it raise trio.MultiError([ValueError(), KeyError()])
async def open_tcp_listeners(port, *, host=None, backlog=None): """Create :class:`SocketListener` objects to listen for TCP connections. Args: port (int): The port to listen on. If you use 0 as your port, then the kernel will automatically pick an arbitrary open port. But be careful: if you use this feature when binding to multiple IP addresses, then each IP address will get its own random port, and the returned listeners will probably be listening on different ports. In particular, this will happen if you use ``host=None`` – which is the default – because in this case :func:`open_tcp_listeners` will bind to both the IPv4 wildcard address (``0.0.0.0``) and also the IPv6 wildcard address (``::``). host (str, bytes-like, or None): The local interface to bind to. This is passed to :func:`~socket.getaddrinfo` with the ``AI_PASSIVE`` flag set. If you want to bind to the wildcard address on both IPv4 and IPv6, in order to accept connections on all available interfaces, then pass ``None``. This is the default. If you have a specific interface you want to bind to, pass its IP address or hostname here. If a hostname resolves to multiple IP addresses, this function will open one listener on each of them. If you want to use only IPv4, or only IPv6, but want to accept on all interfaces, pass the family-specific wildcard address: ``"0.0.0.0"`` for IPv4-only and ``"::"`` for IPv6-only. backlog (int or None): The listen backlog to use. If you leave this as ``None`` then Trio will pick a good default. (Currently: whatever your system has configured as the maximum backlog.) Returns: list of :class:`SocketListener` """ # getaddrinfo sometimes allows port=None, sometimes not (depending on # whether host=None). And on some systems it treats "" as 0, others it # doesn't: # http://klickverbot.at/blog/2012/01/getaddrinfo-edge-case-behavior-on-windows-linux-and-osx/ if not isinstance(port, int): raise TypeError("port must be an int not {!r}".format(port)) backlog = _compute_backlog(backlog) addresses = await tsocket.getaddrinfo(host, port, type=tsocket.SOCK_STREAM, flags=tsocket.AI_PASSIVE) listeners = [] unsupported_address_families = [] try: for family, type, proto, _, sockaddr in addresses: try: sock = tsocket.socket(family, type, proto) except OSError as ex: if ex.errno == errno.EAFNOSUPPORT: # If a system only supports IPv4, or only IPv6, it # is still likely that getaddrinfo will return # both an IPv4 and an IPv6 address. As long as at # least one of the returned addresses can be # turned into a socket, we won't complain about a # failure to create the other. unsupported_address_families.append(ex) continue else: raise try: # See https://github.com/python-trio/trio/issues/39 if sys.platform != "win32": sock.setsockopt(tsocket.SOL_SOCKET, tsocket.SO_REUSEADDR, 1) if family == tsocket.AF_INET6: sock.setsockopt(tsocket.IPPROTO_IPV6, tsocket.IPV6_V6ONLY, 1) await sock.bind(sockaddr) sock.listen(backlog) listeners.append(trio.SocketListener(sock)) except: sock.close() raise except: for listener in listeners: listener.socket.close() raise if unsupported_address_families and not listeners: raise OSError( errno.EAFNOSUPPORT, "This system doesn't support any of the kinds of " "socket that that address could use", ) from trio.MultiError(unsupported_address_families) return listeners
async def open_tcp_stream( host, port, *, # No trailing comma b/c bpo-9232 (fixed in py36) happy_eyeballs_delay=DEFAULT_DELAY): """Connect to the given host and port over TCP. If the given ``host`` has multiple IP addresses associated with it, then we have a problem: which one do we use? One approach would be to attempt to connect to the first one, and then if that fails, attempt to connect to the second one ... until we've tried all of them. But the problem with this is that if the first IP address is unreachable (for example, because it's an IPv6 address and our network discards IPv6 packets), then we might end up waiting tens of seconds for the first connection attempt to timeout before we try the second address. Another approach would be to attempt to connect to all of the addresses at the same time, in parallel, and then use whichever connection succeeds first, abandoning the others. This would be fast, but create a lot of unnecessary load on the network and the remote server. This function strikes a balance between these two extremes: it works its way through the available addresses one at a time, like the first approach; but, if ``happy_eyeballs_delay`` seconds have passed and it's still waiting for an attempt to succeed or fail, then it gets impatient and starts the next connection attempt in parallel. As soon as any one connection attempt succeeds, all the other attempts are cancelled. This avoids unnecessary load because most connections will succeed after just one or two attempts, but if one of the addresses is unreachable then it doesn't slow us down too much. This is known as a "happy eyeballs" algorithm, and our particular variant is modelled after how Chrome connects to webservers; see `RFC 6555 <https://tools.ietf.org/html/rfc6555>`__ for more details. Args: host (str or bytes): The host to connect to. Can be an IPv4 address, IPv6 address, or a hostname. port (int): The port to connect to. happy_eyeballs_delay (float): How many seconds to wait for each connection attempt to succeed or fail before getting impatient and starting another one in parallel. Set to :obj:`math.inf` if you want to limit to only one connection attempt at a time (like :func:`socket.create_connection`). Default: 0.3 (300 ms). Returns: SocketStream: a :class:`~trio.abc.Stream` connected to the given server. Raises: OSError: if the connection fails. See also: open_ssl_over_tcp_stream """ # To keep our public API surface smaller, rule out some cases that # getaddrinfo will accept in some circumstances, but that act weird or # have non-portable behavior or are just plain not useful. # No type check on host though b/c we want to allow bytes-likes. if host is None: raise ValueError("host cannot be None") if not isinstance(port, int): raise TypeError("port must be int, not {!r}".format(port)) if happy_eyeballs_delay is None: happy_eyeballs_delay = DEFAULT_DELAY targets = await getaddrinfo(host, port, type=SOCK_STREAM) # I don't think this can actually happen -- if there are no results, # getaddrinfo should have raised OSError instead of returning an empty # list. But let's be paranoid and handle it anyway: if not targets: msg = "no results found for hostname lookup: {}".format( format_host_port(host, port)) raise OSError(msg) reorder_for_rfc_6555_section_5_4(targets) targets_iter = iter(targets) # This list records all the connection failures that we ignored. oserrors = [] # It's possible for multiple connection attempts to succeed at the ~same # time; this list records all successful connections. winning_sockets = [] # Sleep for the given amount of time, then kick off the next task and # start a connection attempt. On failure, expedite the next task; on # success, kill everything. Possible outcomes: # - records a failure in oserrors, returns None # - records a connected socket in winning_sockets, returns None # - crash (raises an unexpected exception) async def attempt_connect(nursery, previous_attempt_failed): # Wait until either the previous attempt failed, or the timeout # expires (unless this is the first invocation, in which case we just # go ahead). if previous_attempt_failed is not None: with trio.move_on_after(happy_eyeballs_delay): await previous_attempt_failed.wait() # Claim our target. try: *socket_args, _, target_sockaddr = next(targets_iter) except StopIteration: return # Then kick off the next attempt. this_attempt_failed = trio.Event() nursery.start_soon(attempt_connect, nursery, this_attempt_failed) # Then make this invocation's attempt try: with close_on_error(socket(*socket_args)) as sock: await sock.connect(target_sockaddr) except OSError as exc: # This connection attempt failed, but the next one might # succeed. Save the error for later so we can report it if # everything fails, and tell the next attempt that it should go # ahead (if it hasn't already). oserrors.append(exc) this_attempt_failed.set() else: # Success! Save the winning socket and cancel all outstanding # connection attempts. winning_sockets.append(sock) nursery.cancel_scope.cancel() # Kick off the chain of connection attempts. async with trio.open_nursery() as nursery: nursery.start_soon(attempt_connect, nursery, None) # All connection attempts complete, and no unexpected errors escaped. So # at this point the oserrors and winning_sockets lists are filled in. if winning_sockets: first_prize = winning_sockets.pop(0) for sock in winning_sockets: sock.close() return trio.SocketStream(first_prize) else: assert len(oserrors) == len(targets) msg = "all attempts to connect to {} failed".format( format_host_port(host, port)) raise OSError(msg) from trio.MultiError(oserrors)
async def _open_and_supervise_one_cancels_all_nursery( actor: Actor, ) -> typing.AsyncGenerator[ActorNursery, None]: # the collection of errors retreived from spawned sub-actors errors: Dict[Tuple[str, str], Exception] = {} # This is the outermost level "deamon actor" nursery. It is awaited # **after** the below inner "run in actor nursery". This allows for # handling errors that are generated by the inner nursery in # a supervisor strategy **before** blocking indefinitely to wait for # actors spawned in "daemon mode" (aka started using # ``ActorNursery.start_actor()``). # errors from this daemon actor nursery bubble up to caller async with trio.open_nursery() as da_nursery: try: # This is the inner level "run in actor" nursery. It is # awaited first since actors spawned in this way (using # ``ActorNusery.run_in_actor()``) are expected to only # return a single result and then complete (i.e. be canclled # gracefully). Errors collected from these actors are # immediately raised for handling by a supervisor strategy. # As such if the strategy propagates any error(s) upwards # the above "daemon actor" nursery will be notified. async with trio.open_nursery() as ria_nursery: anursery = ActorNursery(actor, ria_nursery, da_nursery, errors) try: # spawning of actors happens in the caller's scope # after we yield upwards yield anursery log.runtime(f"Waiting on subactors {anursery._children} " "to complete") # Last bit before first nursery block ends in the case # where we didn't error in the caller's scope # signal all process monitor tasks to conduct # hard join phase. anursery._join_procs.set() except BaseException as err: # If we error in the root but the debugger is # engaged we don't want to prematurely kill (and # thus clobber access to) the local tty since it # will make the pdb repl unusable. # Instead try to wait for pdb to be released before # tearing down. await maybe_wait_for_debugger( child_in_debug=anursery._at_least_one_child_in_debug) # if the caller's scope errored then we activate our # one-cancels-all supervisor strategy (don't # worry more are coming). anursery._join_procs.set() try: # XXX: hypothetically an error could be # raised and then a cancel signal shows up # slightly after in which case the `else:` # block here might not complete? For now, # shield both. with trio.CancelScope(shield=True): etype = type(err) if etype in (trio.Cancelled, KeyboardInterrupt ) or (is_multi_cancelled(err)): log.cancel( f"Nursery for {current_actor().uid} " f"was cancelled with {etype}") else: log.exception( f"Nursery for {current_actor().uid} " f"errored with {err}, ") # cancel all subactors await anursery.cancel() except trio.MultiError as merr: # If we receive additional errors while waiting on # remaining subactors that were cancelled, # aggregate those errors with the original error # that triggered this teardown. if err not in merr.exceptions: raise trio.MultiError(merr.exceptions + [err]) else: raise # ria_nursery scope end # XXX: do we need a `trio.Cancelled` catch here as well? # this is the catch around the ``.run_in_actor()`` nursery except (Exception, trio.MultiError, trio.Cancelled) as err: # XXX: yet another guard before allowing the cancel # sequence in case a (single) child is in debug. await maybe_wait_for_debugger( child_in_debug=anursery._at_least_one_child_in_debug) # If actor-local error was raised while waiting on # ".run_in_actor()" actors then we also want to cancel all # remaining sub-actors (due to our lone strategy: # one-cancels-all). log.cancel(f"Nursery cancelling due to {err}") if anursery._children: with trio.CancelScope(shield=True): await anursery.cancel() raise finally: # No errors were raised while awaiting ".run_in_actor()" # actors but those actors may have returned remote errors as # results (meaning they errored remotely and have relayed # those errors back to this parent actor). The errors are # collected in ``errors`` so cancel all actors, summarize # all errors and re-raise. if errors: if anursery._children: with trio.CancelScope(shield=True): await anursery.cancel() # use `MultiError` as needed if len(errors) > 1: raise trio.MultiError(tuple(errors.values())) else: raise list(errors.values())[0]
def foo(): raise trio.MultiError([_cancelled(), ValueError()])
def _flush_stats_sync(self, write_restore_file: bool = False, expect_file: bool = True) -> None: """Update the filesystem stats on the disk in a way that is safe for concurrent access This function performs lots of synchronous I/O, call it from an I/O thread only and ensure you hold :attr:`_stats_lock` while doing so! The general proceedure being a follows: 1. Write the new/current stats to a temporary file 2. Atomically exchange ``diskUsage.cache`` and the temporary file * *Fallback if atomic exchange is not available on the host OS*: Move ``diskUsage.cache`` to new location and non-replacingly rename temporary file to ``diskUsage.cache`` (set a flag if the target file already exists by the time we try to move our's in) * The primary issue with this variant is that the stats file will be gone (moved or unlinked) for a splitsecond before the new one has been moved in. If the program crashes between these two actions, we will loose some stats. 3. Compare mtime of the moved/previous ``diskUsage.cache`` file to the file's value on startup * If they do not match update the expected value and mtime from the moved file and retry from step 1 * In effect this constitutes a 3-way merge with our expected value as the base, the read value as the “remote” (cause somebody else must have written it) and our new value as “local” 4. Check flag set by the exchange fallback code and continue as in step 3 if it is set (using the current ``diskUsage.cache`` file as the “remote”) 5. Success """ assert self._stats is not None assert self._stats_prev is not None assert self._stats_lock.locked() unlink_paths: typing.List[pathlib.Path] = [] try: path = pathlib.Path(self.object_path(self.stats_key)) path_restore = pathlib.Path(str(path) + "-restore") path_dir = path.parent path_prefix = f".{path.name}.tmp-" while True: # Read restore file if it exists (works around non-cooperating # implementations just overwriting everything blindly) try: path_restore_tmp = move_to_tempfile_sync( path_restore, dir=path_dir, prefix=path_prefix) except FileNotFoundError: pass # No restore file currently written else: unlink_paths.append(path_restore_tmp) restore_data = json.loads(path_restore_tmp.read_bytes()) restore_orig = Stats.from_json(restore_data.get( "orig", {})) restore_local = Stats.from_json( restore_data.get("local", {})) try: with path.open() as remote_file: restore_remote = Stats.from_json( json.loads(remote_file.read())) restore_remote.mtime_ns = os.fstat( remote_file.fileno()).st_mtime_ns # Apply delta from current file to restore data if not restore_remote.can_merge: restore_local.merge(restore_orig, restore_remote) self._stats_orig = restore_remote else: self._stats_orig = restore_orig except FileNotFoundError: pass new_stats = restore_local.copy() # Apply delta from our value to restore data new_stats.merge(self._stats_prev, self._stats) self._stats = new_stats self._stats_prev = restore_local with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path_dir, prefix=path_prefix, delete=False) as new_file: # Remember the file's path so that we may unlink it when # the time comes new_path = pathlib.Path(self.root_path / new_file.name) unlink_paths.append(new_path) # Write data new_file.write(json.dumps(self._stats.to_json())) # Also remember the mtime after writing for later collision detection new_path_mtime_ns: int = new_path.stat().st_mtime_ns need_move_retry: bool = False old_path: typing.Optional[pathlib.Path] = None try: # Atomically exchange temporary and production file exchange.exchange(path, new_path) # – The temporary file now contains the previous contents old_path = new_path # – The target file has now been updated self._stats.mtime_ns = new_path_mtime_ns except (FileNotFoundError, AttributeError, NotImplementedError) as exc: if not isinstance(exc, FileNotFoundError): # Fallback code in case atomic exchange is not available # # The primary issue with this variant is that the stats file # will be gone (moved or unlinked) for a splitsecond before # the new one has been moved in. If the program crashes # between these two actions, we loose the stats. # Move target file to temporary location try: old_path = move_to_tempfile_sync( path, wait_for_src=expect_file, dir=path_dir, prefix=path_prefix) except FileNotFoundError: old_path = None # This code applies both to the case of atomic exchange not being # available and the target file being non-existent try: # Move created temporary file with our data to # production file location rename_noreplace.rename_noreplace(new_path, path) self._stats.mtime_ns = new_path_mtime_ns except FileExistsError: # Somebody else beat us to it, we'll need to incorporate # their changes and retry expect_file = True need_move_retry = True if old_path is not None: # Compare timestamp of swapped out file to the expected value old_path_mtime_ns = old_path.stat().st_mtime_ns if old_path_mtime_ns != self._stats_prev.mtime_ns: old_stats = Stats.from_json( json.loads(old_path.read_bytes())) old_stats.mtime_ns = old_path_mtime_ns cur_stats = self._stats.copy() # Apply delta to current stats (aka perform a “merge”) if not old_stats.can_merge: if self._stats_orig is not None: self._stats.merge(self._stats_orig, old_stats) self._stats_orig = old_stats else: self._stats.merge(self._stats_prev, old_stats) # Expect the stats data we just swapped in from now on # and request a retry soon self._stats_prev = cur_stats self._stats_prev.mtime_ns = new_path_mtime_ns continue if need_move_retry: old_stats = Stats.from_json(json.loads(path.read_bytes())) cur_stats = self._stats # Apply delta to current stats (aka perform a “merge”) self._stats.merge(self._stats_prev, old_stats) # Expect the stats data we just swapped in from now on and # request a retry soon self._stats_prev = cur_stats continue if write_restore_file: assert self._stats_prev is not None assert self._stats_orig is not None with tempfile.NamedTemporaryFile( mode="w", encoding="utf-8", dir=path_dir, prefix=path_prefix, delete=False) as new_restore_file: # Remember the file's path so that we may unlink it when # the time comes new_restore_path = pathlib.Path(self.root_path / new_restore_file.name) unlink_paths.append(new_restore_path) # Write data new_restore_file.write( json.dumps({ "local": self._stats.to_json(mtime=True), "prev": self._stats_prev.to_json(mtime=True), "orig": self._stats_orig.to_json(mtime=True), })) try: # Move created temporary file with our data to # production file location rename_noreplace.rename_noreplace( new_restore_path, path_restore) except FileExistsError: # Somebody else beat us to it, we'll need to incorporate # their changes and retry continue # Remember current snapshot as new base and return self._stats_prev = self._stats.copy() if self._stats_orig is None: self._stats_orig = self._stats.copy() return finally: # Clean up all temporary files created during the above errors: typing.List[Exception] = [] for unlink_path in unlink_paths: try: unlink_path.unlink() except FileNotFoundError: pass except Exception as exc: errors.append(exc) if errors: if len(unlink_paths) == 1: raise errors[0] raise trio.MultiError(errors)
async def open_nursery() -> typing.AsyncGenerator[ActorNursery, None]: """Create and yield a new ``ActorNursery`` to be used for spawning structured concurrent subactors. When an actor is spawned a new trio task is started which invokes one of the process spawning backends to create and start a new subprocess. These tasks are started by one of two nurseries detailed below. The reason for spawning processes from within a new task is because ``trio_run_in_process`` itself creates a new internal nursery and the same task that opens a nursery **must** close it. It turns out this approach is probably more correct anyway since it is more clear from the following nested nurseries which cancellation scopes correspond to each spawned subactor set. """ actor = current_actor() if not actor: raise RuntimeError("No actor instance has been defined yet?") # the collection of errors retreived from spawned sub-actors errors: Dict[Tuple[str, str], Exception] = {} # This is the outermost level "deamon actor" nursery. It is awaited # **after** the below inner "run in actor nursery". This allows for # handling errors that are generated by the inner nursery in # a supervisor strategy **before** blocking indefinitely to wait for # actors spawned in "daemon mode" (aka started using # ``ActorNursery.start_actor()``). async with trio.open_nursery() as da_nursery: try: # This is the inner level "run in actor" nursery. It is # awaited first since actors spawned in this way (using # ``ActorNusery.run_in_actor()``) are expected to only # return a single result and then complete (i.e. be canclled # gracefully). Errors collected from these actors are # immediately raised for handling by a supervisor strategy. # As such if the strategy propagates any error(s) upwards # the above "daemon actor" nursery will be notified. async with trio.open_nursery() as ria_nursery: anursery = ActorNursery(actor, ria_nursery, da_nursery, errors) try: # spawning of actors happens in the caller's scope # after we yield upwards yield anursery log.debug(f"Waiting on subactors {anursery._children}" "to complete") except (BaseException, Exception) as err: # if the caller's scope errored then we activate our # one-cancels-all supervisor strategy (don't # worry more are coming). anursery._join_procs.set() try: # XXX: hypothetically an error could be raised and then # a cancel signal shows up slightly after in which case # the `else:` block here might not complete? # For now, shield both. with trio.CancelScope(shield=True): if err in (trio.Cancelled, KeyboardInterrupt): log.warning( f"Nursery for {current_actor().uid} was " f"cancelled with {err}") else: log.exception( f"Nursery for {current_actor().uid} " f"errored with {err}, ") # cancel all subactors await anursery.cancel() except trio.MultiError as merr: # If we receive additional errors while waiting on # remaining subactors that were cancelled, # aggregate those errors with the original error # that triggered this teardown. if err not in merr.exceptions: raise trio.MultiError(merr.exceptions + [err]) else: raise # Last bit before first nursery block ends in the case # where we didn't error in the caller's scope log.debug(f"Waiting on all subactors to complete") anursery._join_procs.set() # ria_nursery scope end except (Exception, trio.MultiError) as err: # If actor-local error was raised while waiting on # ".run_in_actor()" actors then we also want to cancel all # remaining sub-actors (due to our lone strategy: # one-cancels-all). log.warning(f"Nursery cancelling due to {err}") if anursery._children: with trio.CancelScope(shield=True): await anursery.cancel() raise finally: # No errors were raised while awaiting ".run_in_actor()" # actors but those actors may have returned remote errors as # results (meaning they errored remotely and have relayed # those errors back to this parent actor). The errors are # collected in ``errors`` so cancel all actors, summarize # all errors and re-raise. if errors: if anursery._children: with trio.CancelScope(shield=True): await anursery.cancel() if len(errors) > 1: raise trio.MultiError(tuple(errors.values())) else: raise list(errors.values())[0] # ria_nursery scope end log.debug(f"Nursery teardown complete")
import _common import trio def exc1_fn(): try: raise ValueError except Exception as exc: return exc def exc2_fn(): try: raise KeyError except Exception as exc: return exc # This should be printed nicely, because trio overrode sys.excepthook raise trio.MultiError([exc1_fn(), exc2_fn()])
class MyExceptionBase(Exception): pass class MyException(MyExceptionBase): pass @pytest.mark.parametrize( "context, to_raise, expected_exception", [ # simple exception (defer_to_cancelled(ValueError), ValueError, ValueError), # MultiError gets deferred (defer_to_cancelled(ValueError), trio.MultiError([_cancelled(), ValueError()]), trio.Cancelled), # multiple exception types (defer_to_cancelled(ValueError, KeyError), trio.MultiError([_cancelled(), ValueError(), KeyError()]), trio.Cancelled), # nested MultiError (defer_to_cancelled(ValueError), trio.MultiError( [ValueError(), trio.MultiError([_cancelled(), _cancelled()])]), trio.Cancelled), # non-matching exception (defer_to_cancelled(ValueError), trio.MultiError([_cancelled(), KeyError()]), trio.MultiError), ]) async def test_defer_to_cancelled(context, to_raise, expected_exception): with pytest.raises(expected_exception):
async def open_tcp_stream( host, port, *, # No trailing comma b/c bpo-9232 (fixed in py36) happy_eyeballs_delay=DEFAULT_DELAY, ): """Connect to the given host and port over TCP. If the given ``host`` has multiple IP addresses associated with it, then we have a problem: which one do we use? One approach would be to attempt to connect to the first one, and then if that fails, attempt to connect to the second one ... until we've tried all of them. But the problem with this is that if the first IP address is unreachable (for example, because it's an IPv6 address and our network discards IPv6 packets), then we might end up waiting tens of seconds for the first connection attempt to timeout before we try the second address. Another approach would be to attempt to connect to all of the addresses at the same time, in parallel, and then use whichever connection succeeds first, abandoning the others. This would be fast, but create a lot of unnecessary load on the network and the remote server. This function strikes a balance between these two extremes: it works its way through the available addresses one at a time, like the first approach; but, if ``happy_eyeballs_delay`` seconds have passed and it's still waiting for an attempt to succeed or fail, then it gets impatient and starts the next connection attempt in parallel. As soon as any one connection attempt succeeds, all the other attempts are cancelled. This avoids unnecessary load because most connections will succeed after just one or two attempts, but if one of the addresses is unreachable then it doesn't slow us down too much. This is known as a "happy eyeballs" algorithm, and our particular variant is modelled after how Chrome connects to webservers; see `RFC 6555 <https://tools.ietf.org/html/rfc6555>`__ for more details. Args: host (str or bytes): The host to connect to. Can be an IPv4 address, IPv6 address, or a hostname. port (int): The port to connect to. happy_eyeballs_delay (float): How many seconds to wait for each connection attempt to succeed or fail before getting impatient and starting another one in parallel. Set to `math.inf` if you want to limit to only one connection attempt at a time (like :func:`socket.create_connection`). Default: 0.25 (250 ms). Returns: SocketStream: a :class:`~trio.abc.Stream` connected to the given server. Raises: OSError: if the connection fails. See also: open_ssl_over_tcp_stream """ # To keep our public API surface smaller, rule out some cases that # getaddrinfo will accept in some circumstances, but that act weird or # have non-portable behavior or are just plain not useful. # No type check on host though b/c we want to allow bytes-likes. if host is None: raise ValueError("host cannot be None") if not isinstance(port, int): raise TypeError("port must be int, not {!r}".format(port)) if happy_eyeballs_delay is None: happy_eyeballs_delay = DEFAULT_DELAY targets = await getaddrinfo(host, port, type=SOCK_STREAM) # I don't think this can actually happen -- if there are no results, # getaddrinfo should have raised OSError instead of returning an empty # list. But let's be paranoid and handle it anyway: if not targets: msg = "no results found for hostname lookup: {}".format( format_host_port(host, port)) raise OSError(msg) reorder_for_rfc_6555_section_5_4(targets) # This list records all the connection failures that we ignored. oserrors = [] # Keeps track of the socket that we're going to complete with, # need to make sure this isn't automatically closed winning_socket = None # Try connecting to the specified address. Possible outcomes: # - success: record connected socket in winning_socket and cancel # concurrent attempts # - failure: record exception in oserrors, set attempt_failed allowing # the next connection attempt to start early # code needs to ensure sockets can be closed appropriately in the # face of crash or cancellation async def attempt_connect(socket_args, sockaddr, attempt_failed): nonlocal winning_socket try: sock = socket(*socket_args) open_sockets.add(sock) await sock.connect(sockaddr) # Success! Save the winning socket and cancel all outstanding # connection attempts. winning_socket = sock nursery.cancel_scope.cancel() except OSError as exc: # This connection attempt failed, but the next one might # succeed. Save the error for later so we can report it if # everything fails, and tell the next attempt that it should go # ahead (if it hasn't already). oserrors.append(exc) attempt_failed.set() with close_all() as open_sockets: # nursery spawns a task for each connection attempt, will be # cancelled by the task that gets a successful connection async with trio.open_nursery() as nursery: for *sa, _, addr in targets: # create an event to indicate connection failure, # allowing the next target to be tried early attempt_failed = trio.Event() nursery.start_soon(attempt_connect, sa, addr, attempt_failed) # give this attempt at most this time before moving on with trio.move_on_after(happy_eyeballs_delay): await attempt_failed.wait() # nothing succeeded if winning_socket is None: assert len(oserrors) == len(targets) msg = "all attempts to connect to {} failed".format( format_host_port(host, port)) raise OSError(msg) from trio.MultiError(oserrors) else: stream = trio.SocketStream(winning_socket) open_sockets.remove(winning_socket) return stream
async def foo(): raise trio.MultiError([trio.Cancelled._create(), ValueError()])
async def open_tcp_stream( host, port, *, happy_eyeballs_delay=DEFAULT_DELAY, local_address=None, ): """Connect to the given host and port over TCP. If the given ``host`` has multiple IP addresses associated with it, then we have a problem: which one do we use? One approach would be to attempt to connect to the first one, and then if that fails, attempt to connect to the second one ... until we've tried all of them. But the problem with this is that if the first IP address is unreachable (for example, because it's an IPv6 address and our network discards IPv6 packets), then we might end up waiting tens of seconds for the first connection attempt to timeout before we try the second address. Another approach would be to attempt to connect to all of the addresses at the same time, in parallel, and then use whichever connection succeeds first, abandoning the others. This would be fast, but create a lot of unnecessary load on the network and the remote server. This function strikes a balance between these two extremes: it works its way through the available addresses one at a time, like the first approach; but, if ``happy_eyeballs_delay`` seconds have passed and it's still waiting for an attempt to succeed or fail, then it gets impatient and starts the next connection attempt in parallel. As soon as any one connection attempt succeeds, all the other attempts are cancelled. This avoids unnecessary load because most connections will succeed after just one or two attempts, but if one of the addresses is unreachable then it doesn't slow us down too much. This is known as a "happy eyeballs" algorithm, and our particular variant is modelled after how Chrome connects to webservers; see `RFC 6555 <https://tools.ietf.org/html/rfc6555>`__ for more details. Args: host (str or bytes): The host to connect to. Can be an IPv4 address, IPv6 address, or a hostname. port (int): The port to connect to. happy_eyeballs_delay (float): How many seconds to wait for each connection attempt to succeed or fail before getting impatient and starting another one in parallel. Set to `math.inf` if you want to limit to only one connection attempt at a time (like :func:`socket.create_connection`). Default: 0.25 (250 ms). local_address (None or str): The local IP address or hostname to use as the source for outgoing connections. If ``None``, we let the OS pick the source IP. This is useful in some exotic networking configurations where your host has multiple IP addresses, and you want to force the use of a specific one. Note that if you pass an IPv4 ``local_address``, then you won't be able to connect to IPv6 hosts, and vice-versa. If you want to take advantage of this to force the use of IPv4 or IPv6 without specifying an exact source address, you can use the IPv4 wildcard address ``local_address="0.0.0.0"``, or the IPv6 wildcard address ``local_address="::"``. Returns: SocketStream: a :class:`~trio.abc.Stream` connected to the given server. Raises: OSError: if the connection fails. See also: open_ssl_over_tcp_stream """ # To keep our public API surface smaller, rule out some cases that # getaddrinfo will accept in some circumstances, but that act weird or # have non-portable behavior or are just plain not useful. # No type check on host though b/c we want to allow bytes-likes. if host is None: raise ValueError("host cannot be None") if not isinstance(port, int): raise TypeError("port must be int, not {!r}".format(port)) if happy_eyeballs_delay is None: happy_eyeballs_delay = DEFAULT_DELAY targets = await getaddrinfo(host, port, type=SOCK_STREAM) # I don't think this can actually happen -- if there are no results, # getaddrinfo should have raised OSError instead of returning an empty # list. But let's be paranoid and handle it anyway: if not targets: msg = "no results found for hostname lookup: {}".format( format_host_port(host, port)) raise OSError(msg) reorder_for_rfc_6555_section_5_4(targets) # This list records all the connection failures that we ignored. oserrors = [] # Keeps track of the socket that we're going to complete with, # need to make sure this isn't automatically closed winning_socket = None # Try connecting to the specified address. Possible outcomes: # - success: record connected socket in winning_socket and cancel # concurrent attempts # - failure: record exception in oserrors, set attempt_failed allowing # the next connection attempt to start early # code needs to ensure sockets can be closed appropriately in the # face of crash or cancellation async def attempt_connect(socket_args, sockaddr, attempt_failed): nonlocal winning_socket try: sock = socket(*socket_args) open_sockets.add(sock) if local_address is not None: # TCP connections are identified by a 4-tuple: # # (local IP, local port, remote IP, remote port) # # So if a single local IP wants to make multiple connections # to the same (remote IP, remote port) pair, then those # connections have to use different local ports, or else TCP # won't be able to tell them apart. OTOH, if you have multiple # connections to different remote IP/ports, then those # connections can share a local port. # # Normally, when you call bind(), the kernel will immediately # assign a specific local port to your socket. At this point # the kernel doesn't know which (remote IP, remote port) # you're going to use, so it has to pick a local port that # *no* other connection is using. That's the only way to # guarantee that this local port will be usable later when we # call connect(). (Alternatively, you can set SO_REUSEADDR to # allow multiple nascent connections to share the same port, # but then connect() might fail with EADDRNOTAVAIL if we get # unlucky and our TCP 4-tuple ends up colliding with another # unrelated connection.) # # So calling bind() before connect() works, but it disables # sharing of local ports. This is inefficient: it makes you # more likely to run out of local ports. # # But on some versions of Linux, we can re-enable sharing of # local ports by setting a special flag. This flag tells # bind() to only bind the IP, and not the port. That way, # connect() is allowed to pick the the port, and it can do a # better job of it because it knows the remote IP/port. try: sock.setsockopt(trio.socket.IPPROTO_IP, trio.socket.IP_BIND_ADDRESS_NO_PORT, 1) except (OSError, AttributeError): pass try: await sock.bind((local_address, 0)) except OSError: raise OSError( f"local_address={local_address!r} is incompatible " f"with remote address {sockaddr}") await sock.connect(sockaddr) # Success! Save the winning socket and cancel all outstanding # connection attempts. winning_socket = sock nursery.cancel_scope.cancel() except OSError as exc: # This connection attempt failed, but the next one might # succeed. Save the error for later so we can report it if # everything fails, and tell the next attempt that it should go # ahead (if it hasn't already). oserrors.append(exc) attempt_failed.set() with close_all() as open_sockets: # nursery spawns a task for each connection attempt, will be # cancelled by the task that gets a successful connection async with trio.open_nursery() as nursery: for *sa, _, addr in targets: # create an event to indicate connection failure, # allowing the next target to be tried early attempt_failed = trio.Event() nursery.start_soon(attempt_connect, sa, addr, attempt_failed) # give this attempt at most this time before moving on with trio.move_on_after(happy_eyeballs_delay): await attempt_failed.wait() # nothing succeeded if winning_socket is None: assert len(oserrors) == len(targets) msg = "all attempts to connect to {} failed".format( format_host_port(host, port)) raise OSError(msg) from trio.MultiError(oserrors) else: stream = trio.SocketStream(winning_socket) open_sockets.remove(winning_socket) return stream
async def wait(self) -> None: """Wait for all subactors to complete. This is probably the most complicated (and confusing, sorry) function that does all the clever crap to deal with cancellation, error propagation, and graceful subprocess tear down. """ async def exhaust_portal(portal, actor): """Pull final result from portal (assuming it has one). If the main task is an async generator do our best to consume what's left of it. """ try: log.debug(f"Waiting on final result from {actor.uid}") final = res = await portal.result() # if it's an async-gen then alert that we're cancelling it if inspect.isasyncgen(res): final = [] log.warning( f"Blindly consuming asyncgen for {actor.uid}") with trio.fail_after(1): async with aclosing(res) as agen: async for item in agen: log.debug(f"Consuming item {item}") final.append(item) except (Exception, trio.MultiError) as err: # we reraise in the parent task via a ``trio.MultiError`` return err else: return final async def cancel_on_completion( portal: Portal, actor: Actor, task_status=trio.TASK_STATUS_IGNORED, ) -> None: """Cancel actor gracefully once it's "main" portal's result arrives. Should only be called for actors spawned with `run_in_actor()`. """ with trio.CancelScope() as cs: task_status.started(cs) # if this call errors we store the exception for later # in ``errors`` which will be reraised inside # a MultiError and we still send out a cancel request result = await exhaust_portal(portal, actor) if isinstance(result, Exception): errors.append(result) log.warning( f"Cancelling {portal.channel.uid} after error {result}" ) else: log.info(f"Cancelling {portal.channel.uid} gracefully") # cancel the process now that we have a final result await portal.cancel_actor() # XXX: lol, this will never get run without a shield above.. # if cs.cancelled_caught: # log.warning( # "Result waiter was cancelled, process may have died") async def wait_for_proc( proc: mp.Process, actor: Actor, cancel_scope: Optional[trio.CancelScope] = None, ) -> None: # TODO: timeout block here? if proc.is_alive(): await proc_waiter(proc) # please god don't hang proc.join() log.debug(f"Joined {proc}") # indicate we are no longer managing this subactor self._children.pop(actor.uid) # proc terminated, cancel result waiter that may have # been spawned in tandem if not done already if cancel_scope: log.warning( f"Cancelling existing result waiter task for {actor.uid}") cancel_scope.cancel() log.debug(f"Waiting on all subactors to complete") # since we pop each child subactor on termination, # iterate a copy children = self._children.copy() errors: List[Exception] = [] # wait on run_in_actor() tasks, unblocks when all complete async with trio.open_nursery() as nursery: for subactor, proc, portal in children.values(): cs = None # portal from ``run_in_actor()`` if portal in self._cancel_after_result_on_exit: assert portal cs = await nursery.start( cancel_on_completion, portal, subactor) # TODO: how do we handle remote host spawned actors? nursery.start_soon( wait_for_proc, proc, subactor, cs) if errors: if not self.cancelled: # bubble up error(s) here and expect to be called again # once the nursery has been cancelled externally (ex. # from within __aexit__() if an error is caught around # ``self.wait()`` then, ``self.cancel()`` is called # immediately, in the default supervisor strat, after # which in turn ``self.wait()`` is called again.) raise trio.MultiError(errors) # wait on all `start_actor()` subactors to complete # if errors were captured above and we have not been cancelled # then these ``start_actor()`` spawned actors will block until # cancelled externally children = self._children.copy() async with trio.open_nursery() as nursery: for subactor, proc, portal in children.values(): # TODO: how do we handle remote host spawned actors? assert portal nursery.start_soon(wait_for_proc, proc, subactor, cs) log.debug(f"All subactors for {self} have terminated") if errors: # always raise any error if we're also cancelled raise trio.MultiError(errors)