def __init__(self, name: str):
super().__init__(name)
self._running = trio.Event()
self._stopped = trio.Event()
# Temporary storage for SendChannels used in the `stream` API. Uses a
# `WeakSet` to automate cleanup.
self._stream_channels = collections.defaultdict(set)
self._pending_requests = {}
self._sync_handlers = collections.defaultdict(list)
self._async_handlers = collections.defaultdict(list)
self._run_lock = trio.Lock()
# Signal when a new remote connection is established
self._remote_connections_changed = trio.Condition() # type: ignore
# Signal when at least one remote has had a subscription change.
self._remote_subscriptions_changed = trio.Condition() # type: ignore
# events used to signal that the endpoint has fully booted up.
self._message_processing_loop_running = trio.Event()
self._connection_loop_running = trio.Event()
self._process_broadcasts_running = trio.Event()
# internal signal that local subscriptions have changed.
self._subscriptions_changed = trio.Event()
self._socket_bound = trio.Event()
self._server_stopped = trio.Event()
async def adaptive_timeout(
*tasks: Tuple[Callable[..., Awaitable[None]], Sequence[Any]],
threshold: int = 1,
variance: float = 2,
) -> None:
"""
Given a set of tasks this function will run them concurrently. Once at
least `threshold` have completed, the average completion time is measured.
The remaining tasks are then given `avg_task_time * variance` to complete after
which they will be cancelled
"""
if threshold >= len(tasks):
raise ValueError(
"The `threshold` value must be less than the number of tasks")
elif threshold < 1:
raise ValueError("The `threshold` value must be 1 or greater")
# a mutable list to track the average task time
task_times: List[float] = []
condition = trio.Condition()
# a thin wrapper around the provided tasks which measures their execution
# time.
async def task_wrapper(task_fn: Callable[..., Awaitable[None]],
args: Sequence[Any]) -> None:
nonlocal task_times
start_at = trio.current_time()
await task_fn(*args)
elapsed = trio.current_time() - start_at
async with condition:
task_times.append(elapsed)
condition.notify_all()
async with trio.open_nursery() as nursery:
for task_fn, task_args in tasks:
nursery.start_soon(task_wrapper, task_fn, task_args)
start_at = trio.current_time()
# wait for `threshold` tasks to complete
while len(task_times) < threshold:
async with condition:
await condition.wait()
# measure the average task completion time and calculate the remaining
# timeout for the remaining tasks.
avg_task_time = sum(task_times) / len(task_times)
timeout_at = start_at + (avg_task_time * variance)
timeout_remaining = timeout_at - trio.current_time()
# apply the calculated timeout on the remaining tasks
if timeout_remaining > 0:
with trio.move_on_after(timeout_remaining):
while len(task_times) < len(tasks):
async with condition:
await condition.wait()
# cancel any remaining tasks.
nursery.cancel_scope.cancel()
def __init__(self, *, nursery):
self.nursery = nursery
self.broadcaster = ca.Broadcaster(our_role=ca.CLIENT)
self.log = self.broadcaster.log
self.command_bundle_queue = trio.Queue(capacity=1000)
self.broadcaster_command_condition = trio.Condition()
self._cleanup_condition = trio.Condition()
self._cleanup_event = trio.Event()
# UDP socket broadcasting to CA servers
self.udp_sock = None
self.registered = False # refers to RepeaterRegisterRequest
self.unanswered_searches = {} # map search id (cid) to name
self.search_results = {} # map name to address
self.new_id = ThreadsafeCounter(
dont_clash_with=self.unanswered_searches)
def __init__(self, backend_pool, db, cachedir, max_size, max_entries=768):
log.debug('Initializing')
self.path = cachedir
self.db = db
self.backend_pool = backend_pool
self.cache = CacheDict(max_size, max_entries)
self.mlock = MultiLock()
self.in_transit = set()
self.upload_threads = []
self.removal_threads = []
self.transfer_completed = trio.Condition()
# Will be initialized once threads are available
self.to_upload = None
self.to_remove = None
if os.path.exists(self.path):
self.load_cache()
log.info('Loaded %d entries from cache', len(self.cache))
else:
os.mkdir(self.path)
# Initialized fromt the outside to prevent cyclic dependency
self.fs = None
def __init__(self,
network: NetworkProtocol,
target: NodeID,
concurrency: int = 3) -> None:
self.logger = get_extended_debug_logger("ddht.Explorer")
self._network = network
self.target = target
self._concurrency = concurrency
self._condition = trio.Condition()
self.in_flight = set()
self.seen = set()
self.queried = {self._network.local_node_id}
self.unresponsive = set()
self.unreachable = set()
self.invalid = set()
# Using a relatively small buffer size here ensures that we are applying
# back-pressure against the workers. If the consumer is only consuming a
# few nodes, we don't need to continue issuing requests.
self._send_channel, self._receive_channel = trio.open_memory_channel[
ENRAPI](16)
# signal that the initial set of nodes for exploration has been seeded.
self._exploration_seeded = trio.Event()
# signal that the service is up and running and ready for nodes to be streamed.
self._ready = trio.Event()
def provision_condition(cls) -> Any:
if cls.has_trio_loop():
return trio.Condition()
if cls.has_asyncio_loop():
return asyncio.Condition()
else:
return threading.Condition()
def __init__(self, port: int) -> None:
"""
:param port: The port that a server wants to bind to on this machine, and
make publicly accessible.
"""
self.port = port
self._has_ip_addresses = trio.Event()
self._ip_changed = trio.Condition()
def __init__(self, pool_init, _n, temperr, persistent, listen_addr):
self.pool = pool_init
self._n = _n
self.temperr = {*set(temperr), BrokenSubprocessError}
self.persistent = persistent
self.listen_addr = listen_addr
self.cond_res = trio.Condition()
self.soacceptor = None
def __init__(self, circuit, client, context):
super().__init__(circuit, client, context)
self.nursery = context.nursery
self.QueueFull = trio.WouldBlock
self.command_queue = trio.Queue(ca.MAX_COMMAND_BACKLOG)
self.new_command_condition = trio.Condition()
self.subscription_queue = trio.Queue(ca.MAX_TOTAL_SUBSCRIPTION_BACKLOG)
self.write_event = Event()
self.events_on = trio.Event()
def __init__(self, *, nursery):
self.nursery = nursery
self.broadcaster = ca.Broadcaster(our_role=ca.CLIENT)
self.log = self.broadcaster.log
self.command_chan = open_memory_channel(1000)
self.broadcaster_command_condition = trio.Condition()
self._cleanup_condition = trio.Condition()
self._cleanup_event = trio.Event()
# UDP socket broadcasting to CA servers
self.udp_sock = None
self.registered = False # refers to RepeaterRegisterRequest
self.unanswered_searches = {} # map search id (cid) to name
self.search_results = {} # map name to address
self.new_id = ThreadsafeCounter(
dont_clash_with=self.unanswered_searches)
self.environ = get_environment_variables()
self.ca_server_port = self.environ['EPICS_CA_SERVER_PORT']
def __init__(self, circuit, *, nursery):
self.circuit = circuit # a caproto.VirtualCircuit
self.log = self.circuit.log
self.nursery = nursery
self.channels = {} # map cid to Channel
self.ioids = {} # map ioid to Channel
self.ioid_data = {} # map ioid to server response
self.subscriptionids = {} # map subscriptionid to Channel
self.connected = True
self.socket = None
self.command_chan = open_memory_channel(1000)
self.new_command_condition = trio.Condition()
self._socket_lock = trio.Lock()
def __init__(
self,
local_name: str,
conn: ConnectionAPI,
subscriptions_changed: ConditionAPI,
new_msg_func: Callable[[Broadcast], Awaitable[Any]],
) -> None:
super().__init__(local_name, conn, new_msg_func)
self._notify_lock = trio.Lock() # type: ignore
self._received_response = trio.Condition() # type: ignore
self._received_subscription = trio.Condition() # type: ignore
self._running = trio.Event() # type: ignore
self._stopped = trio.Event() # type: ignore
self._received_subscription = subscriptions_changed
self._subscriptions_initialized = trio.Event() # type: ignore
self._running = trio.Event() # type: ignore
self._stopped = trio.Event() # type: ignore
self._ready = trio.Event() # type: ignore
def __init__(self, circuit, client, context):
super().__init__(circuit, client, context)
self.nursery = context.nursery
self.QueueFull = trio.WouldBlock
self.command_chan = open_memory_channel(ca.MAX_COMMAND_BACKLOG)
# For compatibility with server common:
self.command_queue = self.command_chan.send
self.new_command_condition = trio.Condition()
self.subscription_chan = open_memory_channel(ca.MAX_TOTAL_SUBSCRIPTION_BACKLOG)
# For compatibility with server common:
self.subscription_queue = self.subscription_chan.send
self.write_event = Event()
self.events_on = trio.Event()
async def start_async_works(self):
async with trio.open_nursery() as nursery:
cond = trio.Condition()
for key, af in reversed(self.async_funcs.items()):
nursery.start_soon(af, cond)
logger.info("Work all done in {}".format(time.time() - self.t_start))
def __init__(self, circuit, client, context):
super().__init__(circuit, client, context)
self.nursery = context.nursery
self.command_queue = trio.Queue(1000)
self.new_command_condition = trio.Condition()
def __init__(self):
self.locked_keys = set()
self.cond = trio.Condition()
async def common_recursive_find_nodes(
network: NetworkProtocol,
target: NodeID,
*,
concurrency: int = 3,
unresponsive_cache: Dict[NodeID, float] = UNRESPONSIVE_CACHE,
) -> AsyncIterator[trio.abc.ReceiveChannel[ENRAPI]]:
"""
An optimized version of the recursive lookup algorithm for a kademlia
network.
Continually lookup nodes in the target part of the network, keeping track
of all of the nodes we have seen.
Exit once we have queried all of the `k` closest nodes to the target.
The concurrency structure here is optimized to minimize the effect of
unresponsive nodes on the total time it takes to perform the recursive
lookup. Some requests will hang for up to 10 seconds. The
`adaptive_timeout` combined with the multiple concurrent workers helps
mitigate the overall slowdown caused by a few unresponsive nodes since the
other queries can be issues concurrently.
"""
network.logger.debug2("Recursive find nodes: %s", target.hex())
start_at = trio.current_time()
# The set of NodeID values we have already queried.
queried_node_ids: Set[NodeID] = set()
# The set of NodeID that timed out
#
# The `local_node_id` is
# included in this as a convenience mechanism so that we don't have to
# continually fiter it out of the various filters
unresponsive_node_ids: Set[NodeID] = {network.local_node_id}
# We maintain a cache of nodes that were recently deemed unresponsive
# within the last 10 minutes.
unresponsive_node_ids.update(
node_id
for node_id, last_unresponsive_at in unresponsive_cache.items()
if trio.current_time() - last_unresponsive_at < 300
)
# Accumulator of the node_ids we have seen
received_node_ids: Set[NodeID] = set()
# Tracker for node_ids that are actively being requested.
in_flight: Set[NodeID] = set()
condition = trio.Condition()
def get_unqueried_node_ids() -> Tuple[NodeID, ...]:
"""
Get the three nodes that are closest to the target such that the node
is in the closest `k` nodes which haven't been deemed unresponsive.
"""
# Construct an iterable of *all* the nodes we know about ordered by
# closeness to the target.
candidates = iter_closest_nodes(
target, network.routing_table, received_node_ids
)
# Remove any unresponsive nodes from that iterable
responsive_candidates = itertools.filterfalse(
lambda node_id: node_id in unresponsive_node_ids, candidates
)
# Grab the closest K
closest_k_candidates = take(
network.routing_table.bucket_size, responsive_candidates,
)
# Filter out any from the closest K that we've already queried or that are in-flight
closest_k_unqueried = itertools.filterfalse(
lambda node_id: node_id in queried_node_ids or node_id in in_flight,
closest_k_candidates,
)
return tuple(take(3, closest_k_unqueried))
async def do_lookup(
node_id: NodeID, send_channel: trio.abc.SendChannel[ENRAPI]
) -> None:
"""
Perform an individual lookup on the target part of the network from the
given `node_id`
"""
if node_id == target:
distance = 0
else:
distance = compute_log_distance(node_id, target)
try:
found_enrs = await network.find_nodes(node_id, distance)
except (trio.TooSlowError, MissingEndpointFields, ValidationError):
unresponsive_node_ids.add(node_id)
unresponsive_cache[node_id] = trio.current_time()
return
except trio.Cancelled:
# We don't add these to the unresponsive cache since they didn't
# necessarily exceed the fulle 10s request/response timeout.
unresponsive_node_ids.add(node_id)
raise
for enr in found_enrs:
try:
network.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
async with condition:
new_enrs = tuple(
enr for enr in found_enrs if enr.node_id not in received_node_ids
)
received_node_ids.update(enr.node_id for enr in new_enrs)
for enr in new_enrs:
try:
await send_channel.send(enr)
except (trio.BrokenResourceError, trio.ClosedResourceError):
# In the event that the consumer of `recursive_find_nodes`
# exits early before the lookup has completed we can end up
# operating on a closed channel.
return
async def worker(
worker_id: NodeID, send_channel: trio.abc.SendChannel[ENRAPI]
) -> None:
"""
Pulls unqueried nodes from the closest k nodes and performs a
concurrent lookup on them.
"""
for round in itertools.count():
async with condition:
node_ids = get_unqueried_node_ids()
if not node_ids:
await condition.wait()
continue
# Mark the node_ids as having been queried.
queried_node_ids.update(node_ids)
# Mark the node_ids as being in-flight.
in_flight.update(node_ids)
# Some of the node ids may have come from our routing table.
# These won't be present in the `received_node_ids` so we
# detect this here and send them over the channel.
try:
for node_id in node_ids:
if node_id not in received_node_ids:
enr = network.enr_db.get_enr(node_id)
received_node_ids.add(node_id)
await send_channel.send(enr)
except (trio.BrokenResourceError, trio.ClosedResourceError):
# In the event that the consumer of `recursive_find_nodes`
# exits early before the lookup has completed we can end up
# operating on a closed channel.
return
if len(node_ids) == 1:
await do_lookup(node_ids[0], send_channel)
else:
tasks = tuple(
(do_lookup, (node_id, send_channel)) for node_id in node_ids
)
try:
await adaptive_timeout(*tasks, threshold=1, variance=2.0)
except trio.TooSlowError:
pass
async with condition:
# Remove the `node_ids` from the in_flight set.
in_flight.difference_update(node_ids)
condition.notify_all()
async def _monitor_done(send_channel: trio.abc.SendChannel[ENRAPI]) -> None:
async with send_channel:
async with condition:
while True:
# this `fail_after` is a failsafe to prevent deadlock situations
# which are possible with `Condition` objects.
with trio.move_on_after(60) as scope:
node_ids = get_unqueried_node_ids()
if not node_ids and not in_flight:
break
else:
await condition.wait()
if scope.cancelled_caught:
network.logger.error("Deadlock")
send_channel, receive_channel = trio.open_memory_channel[ENRAPI](256)
async with trio.open_nursery() as nursery:
nursery.start_soon(_monitor_done, send_channel)
for worker_id in range(concurrency):
nursery.start_soon(worker, worker_id, send_channel)
async with receive_channel:
yield receive_channel
nursery.cancel_scope.cancel()
elapsed = trio.current_time() - start_at
network.logger.debug(
"Lookup for %s finished in %f seconds: seen=%d queried=%d unresponsive=%d",
target.hex(),
elapsed,
len(received_node_ids),
len(queried_node_ids),
len(unresponsive_node_ids),
)
def condition(cls, lock=None):
return trio.Condition(lock)
def __init__(self, *args):
super().__init__(*args)
self._log_message_queue = []
self._measurements = {}
self._trio_token = None
self._condition = trio.Condition()
