async def test_alexandria_network_find_nodes_api(alice, bob,
alice_alexandria_network,
bob_alexandria_client):
distances = {0}
bob_alexandria_routing_table = KademliaRoutingTable(
bob.enr.node_id, ROUTING_TABLE_BUCKET_SIZE)
for _ in range(200):
enr = ENRFactory()
bob.enr_db.set_enr(enr)
bob_alexandria_routing_table.update(enr.node_id)
distances.add(compute_log_distance(enr.node_id, bob.node_id))
if distances.issuperset({0, 256, 255}):
break
else:
raise Exception("failed")
async with bob_alexandria_client.subscribe(
FindNodesMessage) as subscription:
async with trio.open_nursery() as nursery:
async def _respond():
request = await subscription.receive()
response_enrs = []
for distance in request.message.payload.distances:
if distance == 0:
response_enrs.append(bob.enr)
else:
for (
node_id
) in bob_alexandria_routing_table.get_nodes_at_log_distance(
distance):
response_enrs.append(bob.enr_db.get_enr(node_id))
await bob_alexandria_client.send_found_nodes(
request.sender_node_id,
request.sender_endpoint,
enrs=response_enrs,
request_id=request.request_id,
)
nursery.start_soon(_respond)
with trio.fail_after(2):
enrs = await alice_alexandria_network.find_nodes(
bob.node_id, 0, 255, 256)
assert any(enr.node_id == bob.node_id for enr in enrs)
response_distances = {
compute_log_distance(enr.node_id, bob.node_id)
for enr in enrs if enr.node_id != bob.node_id
}
assert response_distances == {256, 255}
async def test_client_request_response_find_nodes_found_nodes(
alice,
bob,
alice_alexandria_client,
bob_alexandria_client,
):
table = KademliaRoutingTable(bob.node_id, 256)
for i in range(1000):
enr = ENRFactory()
table.update(enr.node_id)
bob.enr_db.set_enr(enr)
checked_bucket_indexes = []
for distance in range(256, 0, -1):
bucket = table.buckets[distance - 1]
if not len(bucket):
break
async with trio.open_nursery() as nursery:
async with bob_alexandria_client.subscribe(
FindNodesMessage) as subscription:
expected_enrs = tuple(
bob.enr_db.get_enr(node_id) for node_id in bucket)
async def _send_response():
request = await subscription.receive()
checked_bucket_indexes.append(distance)
await bob_alexandria_client.send_found_nodes(
alice.node_id,
alice.endpoint,
enrs=expected_enrs,
request_id=request.request_id,
)
nursery.start_soon(_send_response)
with trio.fail_after(2):
found_nodes_messages = await alice_alexandria_client.find_nodes(
bob.node_id,
bob.endpoint,
distances=[distance],
)
found_node_ids = {
enr.node_id
for message in found_nodes_messages
for enr in message.message.payload.enrs
}
expected_node_ids = {enr.node_id for enr in expected_enrs}
assert found_node_ids == expected_node_ids
assert len(checked_bucket_indexes) > 4
def __init__(
self,
client: ClientAPI,
bootnodes: Collection[ENRAPI],
) -> None:
self.client = client
self._bootnodes = tuple(bootnodes)
self.routing_table = KademliaRoutingTable(
self.client.enr_manager.enr.node_id,
ROUTING_TABLE_BUCKET_SIZE,
)
self._routing_table_ready = trio.Event()
self._last_pong_at = LRU(2048)
def __init__(self, client: ClientAPI, bootnodes: Collection[ENRAPI],) -> None:
self.logger = get_extended_debug_logger("ddht.Network")
self.client = client
self._bootnodes = tuple(bootnodes)
self.routing_table = KademliaRoutingTable(
self.client.enr_manager.enr.node_id, ROUTING_TABLE_BUCKET_SIZE,
)
self._routing_table_ready = trio.Event()
self._last_pong_at = LRU(2048)
self._talk_protocols = {}
self._ping_handler_ready = trio.Event()
self._find_nodes_handler_ready = trio.Event()
def __init__(
self,
network: NetworkAPI,
bootnodes: Collection[ENRAPI],
storage: ContentStorageAPI,
) -> None:
self.logger = get_extended_debug_logger("ddht.Alexandria")
self._bootnodes = tuple(bootnodes)
self.client = AlexandriaClient(network)
self.routing_table = KademliaRoutingTable(
self.enr_manager.enr.node_id, ROUTING_TABLE_BUCKET_SIZE,
)
self._last_pong_at = LRU(2048)
self._routing_table_ready = trio.Event()
self._ping_handler_ready = trio.Event()
self._find_nodes_handler_ready = trio.Event()
self._find_content_handler_ready = trio.Event()
self.storage = storage
class Network(Service, NetworkAPI):
logger = logging.getLogger("ddht.Network")
_bootnodes: Tuple[ENRAPI, ...]
def __init__(
self,
client: ClientAPI,
bootnodes: Collection[ENRAPI],
) -> None:
self.client = client
self._bootnodes = tuple(bootnodes)
self.routing_table = KademliaRoutingTable(
self.client.enr_manager.enr.node_id,
ROUTING_TABLE_BUCKET_SIZE,
)
self._routing_table_ready = trio.Event()
self._last_pong_at = LRU(2048)
#
# Proxied ClientAPI properties
#
@property
def local_node_id(self) -> NodeID:
return self.client.local_node_id
@property
def events(self) -> EventsAPI:
return self.client.events
@property
def dispatcher(self) -> DispatcherAPI:
return self.client.dispatcher
@property
def enr_manager(self) -> ENRManagerAPI:
return self.client.enr_manager
@property
def pool(self) -> PoolAPI:
return self.client.pool
@property
def enr_db(self) -> ENRDatabaseAPI:
return self.client.enr_db
#
# High Level API
#
async def bond(self,
node_id: NodeID,
*,
endpoint: Optional[Endpoint] = None) -> bool:
try:
pong = await self.ping(node_id, endpoint=endpoint)
except trio.TooSlowError:
self.logger.debug("Bonding with %s timed out during ping",
humanize_node_id(node_id))
return False
try:
enr = self.enr_db.get_enr(node_id)
except KeyError:
try:
enr = await self.get_enr(node_id, endpoint=endpoint)
except trio.TooSlowError:
self.logger.debug(
"Bonding with %s timed out during ENR retrieval",
humanize_node_id(node_id),
)
return False
else:
if pong.enr_seq > enr.sequence_number:
try:
enr = await self.get_enr(node_id, endpoint=endpoint)
except trio.TooSlowError:
self.logger.debug(
"Bonding with %s timed out during ENR retrieval",
humanize_node_id(node_id),
)
else:
self.enr_db.set_enr(enr)
self.routing_table.update(enr.node_id)
self._routing_table_ready.set()
return True
async def _bond(self, node_id: NodeID, endpoint: Endpoint) -> None:
await self.bond(node_id, endpoint=endpoint)
async def ping(self,
node_id: NodeID,
*,
endpoint: Optional[Endpoint] = None) -> PongMessage:
if endpoint is None:
endpoint = self._endpoint_for_node_id(node_id)
response = await self.client.ping(endpoint, node_id)
return response.message
async def find_nodes(
self,
node_id: NodeID,
*distances: int,
endpoint: Optional[Endpoint] = None,
) -> Tuple[ENRAPI, ...]:
if not distances:
raise TypeError("Must provide at least one distance")
if endpoint is None:
endpoint = self._endpoint_for_node_id(node_id)
responses = await self.client.find_nodes(endpoint,
node_id,
distances=distances)
return tuple(enr for response in responses
for enr in response.message.enrs)
async def get_enr(self,
node_id: NodeID,
*,
endpoint: Optional[Endpoint] = None) -> ENRAPI:
enrs = await self.find_nodes(node_id, 0, endpoint=endpoint)
if not enrs:
raise Exception("Invalid response")
# This reduce accounts for
return _reduce_enrs(enrs)[0]
async def recursive_find_nodes(self, target: NodeID) -> Tuple[ENRAPI, ...]:
self.logger.debug("Recursive find nodes: %s", humanize_node_id(target))
queried_node_ids = set()
unresponsive_node_ids = set()
received_enrs: List[ENRAPI] = []
received_node_ids: Set[NodeID] = set()
async def do_lookup(node_id: NodeID) -> None:
queried_node_ids.add(node_id)
distance = compute_log_distance(node_id, target)
try:
enrs = await self.find_nodes(node_id, distance)
except trio.TooSlowError:
unresponsive_node_ids.add(node_id)
return
for enr in enrs:
received_node_ids.add(enr.node_id)
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
received_enrs.append(self.enr_db.get_enr(enr.node_id))
else:
received_enrs.append(enr)
for lookup_round_counter in itertools.count():
candidates = iter_closest_nodes(target, self.routing_table,
received_node_ids)
responsive_candidates = itertools.dropwhile(
lambda node: node in unresponsive_node_ids, candidates)
closest_k_candidates = take(self.routing_table.bucket_size,
responsive_candidates)
closest_k_unqueried_candidates = (
candidate for candidate in closest_k_candidates
if candidate not in queried_node_ids)
nodes_to_query = tuple(take(3, closest_k_unqueried_candidates))
if nodes_to_query:
self.logger.debug(
"Starting lookup round %d for %s",
lookup_round_counter + 1,
humanize_node_id(target),
)
async with trio.open_nursery() as nursery:
for peer in nodes_to_query:
nursery.start_soon(do_lookup, peer)
else:
self.logger.debug(
"Lookup for %s finished in %d rounds",
humanize_node_id(target),
lookup_round_counter,
)
break
# now sort and return the ENR records in order of closesness to the target.
return tuple(
sorted(
_reduce_enrs(received_enrs),
key=lambda enr: compute_distance(enr.node_id, target),
))
#
# Long Running Processes
#
async def run(self) -> None:
self.manager.run_daemon_child_service(self.client)
await self.client.wait_listening()
self.manager.run_daemon_task(self._ping_oldest_routing_table_entry)
self.manager.run_daemon_task(self._track_last_pong)
self.manager.run_daemon_task(self._manage_routing_table)
self.manager.run_daemon_task(self._pong_when_pinged)
self.manager.run_daemon_task(self._serve_find_nodes)
await self.manager.wait_finished()
async def _periodically_report_routing_table(self) -> None:
async for _ in every(30, initial_delay=30):
non_empty_buckets = tuple((idx, bucket)
for idx, bucket in enumerate(
reversed(self.routing_table.buckets))
if bucket)
total_size = sum(len(bucket) for idx, bucket in non_empty_buckets)
bucket_info = "|".join(
tuple(f"{idx}:{len(bucket)}"
for idx, bucket in non_empty_buckets))
self.logger.debug(
"routing-table-info: size=%d buckets=%s",
total_size,
bucket_info,
)
async def _ping_oldest_routing_table_entry(self) -> None:
await self._routing_table_ready.wait()
while self.manager.is_running:
# Here we preserve the lazy iteration while still checking that the
# iterable is not empty before passing it into `min` below which
# throws an ambiguous `ValueError` otherwise if the iterable is
# empty.
nodes_iter = self.routing_table.iter_all_random()
try:
first_node_id = first(nodes_iter)
except StopIteration:
await trio.sleep(ROUTING_TABLE_KEEP_ALIVE)
continue
else:
least_recently_ponged_node_id = min(
cons(first_node_id, nodes_iter),
key=lambda node_id: self._last_pong_at.get(node_id, 0),
)
too_old_at = trio.current_time() - ROUTING_TABLE_KEEP_ALIVE
try:
last_pong_at = self._last_pong_at[
least_recently_ponged_node_id]
except KeyError:
pass
else:
if last_pong_at > too_old_at:
await trio.sleep(last_pong_at - too_old_at)
continue
did_bond = await self.bond(least_recently_ponged_node_id)
if not did_bond:
self.routing_table.remove(least_recently_ponged_node_id)
async def _track_last_pong(self) -> None:
async with self.dispatcher.subscribe(PongMessage) as subscription:
async for message in subscription:
self._last_pong_at[
message.sender_node_id] = trio.current_time()
async def _manage_routing_table(self) -> None:
# First load all the bootnode ENRs into our database
for enr in self._bootnodes:
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
# Now repeatedly try to bond with each bootnode until one succeeds.
async with trio.open_nursery() as nursery:
while self.manager.is_running:
for enr in self._bootnodes:
if enr.node_id == self.local_node_id:
continue
endpoint = self._endpoint_for_enr(enr)
nursery.start_soon(self._bond, enr.node_id, endpoint)
with trio.move_on_after(10):
await self._routing_table_ready.wait()
break
# TODO: Need better logic here for more quickly populating the
# routing table. Should start off aggressively filling in the
# table, only backing off once the table contains some minimum
# number of records **or** searching for new records fails to find
# new nodes. Maybe use a TokenBucket
async for _ in every(30):
async with trio.open_nursery() as nursery:
target_node_id = NodeID(secrets.token_bytes(32))
found_enrs = await self.recursive_find_nodes(target_node_id)
for enr in found_enrs:
endpoint = self._endpoint_for_enr(enr)
nursery.start_soon(self._bond, enr.node_id, endpoint)
async def _pong_when_pinged(self) -> None:
async with self.dispatcher.subscribe(PingMessage) as subscription:
async for request in subscription:
await self.dispatcher.send_message(
request.to_response(
PongMessage(
request.message.request_id,
self.enr_manager.enr.sequence_number,
request.sender_endpoint.ip_address,
request.sender_endpoint.port,
)))
async def _serve_find_nodes(self) -> None:
async with self.dispatcher.subscribe(FindNodeMessage) as subscription:
async for request in subscription:
response_enrs: List[ENRAPI] = []
distances = set(request.message.distances)
if len(distances) != len(request.message.distances):
self.logger.debug(
"Ignoring invalid FindNodeMessage from %s@%s: duplicate distances",
humanize_node_id(request.sender_node_id),
request.sender_endpoint,
)
continue
elif not distances:
self.logger.debug(
"Ignoring invalid FindNodeMessage from %s@%s: empty distances",
humanize_node_id(request.sender_node_id),
request.sender_endpoint,
)
continue
elif any(distance > self.routing_table.num_buckets
for distance in distances):
self.logger.debug(
"Ignoring invalid FindNodeMessage from %s@%s: distances: %s",
humanize_node_id(request.sender_node_id),
request.sender_endpoint,
distances,
)
continue
for distance in distances:
if distance == 0:
response_enrs.append(self.enr_manager.enr)
elif distance <= self.routing_table.num_buckets:
node_ids_at_distance = self.routing_table.get_nodes_at_log_distance(
distance, )
for node_id in node_ids_at_distance:
response_enrs.append(self.enr_db.get_enr(node_id))
else:
raise Exception("Should be unreachable")
await self.client.send_found_nodes(
request.sender_endpoint,
request.sender_node_id,
enrs=response_enrs,
request_id=request.message.request_id,
)
#
# Utility
#
def _endpoint_for_enr(self, enr: ENRAPI) -> Endpoint:
try:
ip_address = enr[IP_V4_ADDRESS_ENR_KEY]
port = enr[UDP_PORT_ENR_KEY]
except KeyError:
raise Exception("Missing endpoint address information: ")
return Endpoint(ip_address, port)
def _endpoint_for_node_id(self, node_id: NodeID) -> Endpoint:
enr = self.enr_db.get_enr(node_id)
return self._endpoint_for_enr(enr)
class Network(Service, NetworkAPI):
_bootnodes: Tuple[ENRAPI, ...]
_talk_protocols: Dict[bytes, TalkProtocolAPI]
def __init__(self, client: ClientAPI, bootnodes: Collection[ENRAPI],) -> None:
self.logger = get_extended_debug_logger("ddht.Network")
self.client = client
self._bootnodes = tuple(bootnodes)
self.routing_table = KademliaRoutingTable(
self.client.enr_manager.enr.node_id, ROUTING_TABLE_BUCKET_SIZE,
)
self._routing_table_ready = trio.Event()
self._last_pong_at = LRU(2048)
self._talk_protocols = {}
self._ping_handler_ready = trio.Event()
self._find_nodes_handler_ready = trio.Event()
async def ready(self) -> None:
await self._ping_handler_ready.wait()
await self._find_nodes_handler_ready.wait()
#
# Proxied ClientAPI properties
#
@property
def local_node_id(self) -> NodeID:
return self.client.local_node_id
@property
def events(self) -> EventsAPI:
return self.client.events
@property
def dispatcher(self) -> DispatcherAPI:
return self.client.dispatcher
@property
def enr_manager(self) -> ENRManagerAPI:
return self.client.enr_manager
@property
def pool(self) -> PoolAPI:
return self.client.pool
@property
def enr_db(self) -> QueryableENRDatabaseAPI:
return self.client.enr_db
#
# TALK API
#
def add_talk_protocol(self, protocol: TalkProtocolAPI) -> None:
if protocol.protocol_id in self._talk_protocols:
raise DuplicateProtocol(
f"A protocol is already registered for '{protocol.protocol_id!r}'"
)
self._talk_protocols[protocol.protocol_id] = protocol
#
# High Level API
#
async def bond(
self, node_id: NodeID, *, endpoint: Optional[Endpoint] = None
) -> bool:
self.logger.debug2(
"Bonding with %s", node_id.hex(),
)
try:
pong = await self.ping(node_id, endpoint=endpoint)
except trio.TooSlowError:
self.logger.debug("Bonding with %s timed out during ping", node_id.hex())
return False
except MissingEndpointFields:
self.logger.debug(
"Bonding with %s failed due to missing endpoint information",
node_id.hex(),
)
return False
try:
enr = await self.lookup_enr(
node_id, enr_seq=pong.enr_seq, endpoint=endpoint
)
except trio.TooSlowError:
self.logger.debug(
"Bonding with %s timed out during ENR retrieval", node_id.hex(),
)
return False
except EmptyFindNodesResponse:
self.logger.debug(
"Bonding with %s failed due to them not returing their ENR record",
node_id.hex(),
)
return False
self.routing_table.update(enr.node_id)
self.logger.debug(
"Bonded with %s successfully", node_id.hex(),
)
self._routing_table_ready.set()
return True
async def _bond(self, node_id: NodeID, endpoint: Optional[Endpoint] = None) -> None:
await self.bond(node_id, endpoint=endpoint)
async def ping(
self,
node_id: NodeID,
*,
endpoint: Optional[Endpoint] = None,
request_id: Optional[bytes] = None,
) -> PongMessage:
if endpoint is None:
endpoint = await self.endpoint_for_node_id(node_id)
response = await self.client.ping(node_id, endpoint, request_id=request_id)
return response.message
async def find_nodes(
self,
node_id: NodeID,
*distances: int,
endpoint: Optional[Endpoint] = None,
request_id: Optional[bytes] = None,
) -> Tuple[ENRAPI, ...]:
if not distances:
raise TypeError("Must provide at least one distance")
if endpoint is None:
endpoint = await self.endpoint_for_node_id(node_id)
responses = await self.client.find_nodes(
node_id, endpoint, distances=distances, request_id=request_id
)
# Validate that all responses are indeed at one of the
# specified distances.
for response in responses:
validate_found_nodes_distances(response.message.enrs, node_id, distances)
return tuple(enr for response in responses for enr in response.message.enrs)
def stream_find_nodes(
self,
node_id: NodeID,
endpoint: Endpoint,
distances: Collection[int],
*,
request_id: Optional[bytes] = None,
) -> AsyncContextManager[trio.abc.ReceiveChannel[ENRAPI]]:
return common_network_stream_find_nodes(
self, node_id, endpoint, distances, request_id=request_id
)
async def talk(
self,
node_id: NodeID,
*,
protocol: bytes,
payload: bytes,
endpoint: Optional[Endpoint] = None,
request_id: Optional[bytes] = None,
) -> bytes:
if endpoint is None:
endpoint = await self.endpoint_for_node_id(node_id)
response = await self.client.talk(
node_id, endpoint, protocol, payload, request_id=request_id
)
payload = response.message.payload
if not payload:
raise ProtocolNotSupported(protocol)
return response.message.payload
async def lookup_enr(
self, node_id: NodeID, *, enr_seq: int = 0, endpoint: Optional[Endpoint] = None
) -> ENRAPI:
if node_id == self.local_node_id:
raise Exception(f"Cannot lookup local ENR: node_id={node_id.hex()}")
try:
enr = self.enr_db.get_enr(node_id)
except KeyError:
if endpoint is None:
# Try to use a recursive network lookup to find the desired
# node.
async with self.recursive_find_nodes(node_id) as enr_aiter:
async for found_enr in enr_aiter:
if found_enr.node_id == node_id:
endpoint = Endpoint.from_enr(found_enr)
break
else:
# we weren't given an endpoint and we don't have an enr which would give
# us an endpoint, there's no way to reach this node.
raise KeyError(f"Could not find ENR: node_id={node_id.hex()}")
else:
if enr.sequence_number >= enr_seq:
return enr
enr = await self._fetch_enr(node_id, endpoint=endpoint)
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
return enr
async def _fetch_enr(
self, node_id: NodeID, *, endpoint: Optional[Endpoint]
) -> ENRAPI:
enrs = await self.find_nodes(node_id, 0, endpoint=endpoint)
if not enrs:
raise EmptyFindNodesResponse(f"{node_id.hex()} did not return its ENR")
# Assuming we're given enrs for a single node, this reduce returns the enr for
# that node with the highest sequence number
return reduce_enrs(enrs)[0]
def recursive_find_nodes(
self, target: NodeID
) -> AsyncContextManager[trio.abc.ReceiveChannel[ENRAPI]]:
return common_recursive_find_nodes(self, target)
@asynccontextmanager
async def explore(
self, target: NodeID, concurrency: int = 3,
) -> AsyncIterator[trio.abc.ReceiveChannel[ENRAPI]]:
explorer = Explorer(self, target, concurrency)
with trio.move_on_after(300) as scope:
async with background_trio_service(explorer):
await explorer.ready()
async with explorer.stream() as receive_channel:
yield receive_channel
if scope.cancelled_caught:
self.logger.error("Timeout from `stream_locate`")
#
# Long Running Processes
#
async def run(self) -> None:
self.manager.run_daemon_child_service(self.client)
await self.client.wait_listening()
self.manager.run_daemon_task(self._periodically_report_routing_table)
self.manager.run_daemon_task(self._ping_oldest_routing_table_entry)
self.manager.run_daemon_task(self._track_last_pong)
self.manager.run_daemon_task(self._manage_routing_table)
self.manager.run_daemon_task(self._pong_when_pinged)
self.manager.run_daemon_task(self._serve_find_nodes)
self.manager.run_daemon_task(self._handle_unhandled_talk_requests)
await self.manager.wait_finished()
async def _periodically_report_routing_table(self) -> None:
async for _ in every(30, initial_delay=10):
non_empty_buckets = tuple(
reversed(
tuple(
(idx, bucket)
for idx, bucket in enumerate(self.routing_table.buckets, 1)
if bucket
)
)
)
total_size = sum(len(bucket) for idx, bucket in non_empty_buckets)
bucket_info = "|".join(
tuple(
f"{idx}:{'F' if len(bucket) == self.routing_table.bucket_size else len(bucket)}"
for idx, bucket in non_empty_buckets
)
)
self.logger.debug(
"routing-table-info: size=%d buckets=%s", total_size, bucket_info,
)
async def _ping_oldest_routing_table_entry(self) -> None:
await self._routing_table_ready.wait()
while self.manager.is_running:
# Here we preserve the lazy iteration while still checking that the
# iterable is not empty before passing it into `min` below which
# throws an ambiguous `ValueError` otherwise if the iterable is
# empty.
nodes_iter = self.routing_table.iter_all_random()
try:
first_node_id = first(nodes_iter)
except StopIteration:
await trio.sleep(ROUTING_TABLE_KEEP_ALIVE)
continue
else:
least_recently_ponged_node_id = min(
cons(first_node_id, nodes_iter),
key=lambda node_id: self._last_pong_at.get(node_id, 0),
)
too_old_at = trio.current_time() - ROUTING_TABLE_KEEP_ALIVE
try:
last_pong_at = self._last_pong_at[least_recently_ponged_node_id]
except KeyError:
pass
else:
if last_pong_at > too_old_at:
await trio.sleep(last_pong_at - too_old_at)
continue
did_bond = await self.bond(least_recently_ponged_node_id)
if not did_bond:
self.routing_table.remove(least_recently_ponged_node_id)
async def _track_last_pong(self) -> None:
async with self.dispatcher.subscribe(PongMessage) as subscription:
async for message in subscription:
self._last_pong_at[message.sender_node_id] = trio.current_time()
async def _manage_routing_table(self) -> None:
# First load all the bootnode ENRs into our database
for enr in self._bootnodes:
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
# Now repeatedly try to bond with each bootnode until one succeeds.
while self.manager.is_running:
with trio.move_on_after(20):
async with trio.open_nursery() as nursery:
for enr in self._bootnodes:
if enr.node_id == self.local_node_id:
continue
endpoint = Endpoint.from_enr(enr)
nursery.start_soon(self._bond, enr.node_id, endpoint)
await self._routing_table_ready.wait()
break
# Now we enter into an infinite loop that continually probes the
# network to beep the routing table fresh. We both perform completely
# random lookups, as well as targeted lookups on the outermost routing
# table buckets which are not full.
#
# The `TokenBucket` allows us to burst at the beginning, making quick
# successive probes, then slowing down once the
#
# TokenBucket starts with 10 tokens, refilling at 1 token every 30
# seconds.
token_bucket = TokenBucket(1 / 30, 10)
async with trio.open_nursery() as nursery:
while self.manager.is_running:
await token_bucket.take()
# Get the logarithmic distance to the "largest" buckets
# that are not full.
non_full_bucket_distances = tuple(
idx + 1
for idx, bucket in enumerate(self.routing_table.buckets)
if len(bucket) < self.routing_table.bucket_size # noqa: E501
)[-16:]
# Probe one of the not-full-buckets with a weighted preference
# towards the largest buckets.
distance_to_probe = weighted_choice(non_full_bucket_distances)
target_node_id = at_log_distance(self.local_node_id, distance_to_probe)
async with self.recursive_find_nodes(target_node_id) as enr_aiter:
async for enr in enr_aiter:
if enr.node_id == self.local_node_id:
continue
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
nursery.start_soon(self._bond, enr.node_id)
async def _pong_when_pinged(self) -> None:
async def _maybe_add_to_routing_table(
request: InboundMessage[PingMessage],
) -> None:
try:
enr = await self.lookup_enr(
request.sender_node_id,
enr_seq=request.message.enr_seq,
endpoint=request.sender_endpoint,
)
except (trio.TooSlowError, EmptyFindNodesResponse):
return
self.routing_table.update(enr.node_id)
self._routing_table_ready.set()
async with trio.open_nursery() as nursery:
async with self.dispatcher.subscribe(PingMessage) as subscription:
self._ping_handler_ready.set()
async for request in subscription:
await self.dispatcher.send_message(
request.to_response(
PongMessage(
request.request_id,
self.enr_manager.enr.sequence_number,
request.sender_endpoint.ip_address,
request.sender_endpoint.port,
)
)
)
nursery.start_soon(_maybe_add_to_routing_table, request)
async def _serve_find_nodes(self) -> None:
async with self.dispatcher.subscribe(FindNodeMessage) as subscription:
self._find_nodes_handler_ready.set()
async for request in subscription:
response_enrs: List[ENRAPI] = []
distances = set(request.message.distances)
if len(distances) != len(request.message.distances):
self.logger.debug(
"Ignoring invalid FindNodeMessage from %s@%s: duplicate distances",
request.sender_node_id.hex(),
request.sender_endpoint,
)
continue
elif not distances:
self.logger.debug(
"Ignoring invalid FindNodeMessage from %s@%s: empty distances",
request.sender_node_id.hex(),
request.sender_endpoint,
)
continue
elif any(
distance > self.routing_table.num_buckets for distance in distances
):
self.logger.debug(
"Ignoring invalid FindNodeMessage from %s@%s: distances: %s",
request.sender_node_id.hex(),
request.sender_endpoint,
distances,
)
continue
for distance in distances:
if distance == 0:
response_enrs.append(self.enr_manager.enr)
elif distance <= self.routing_table.num_buckets:
node_ids_at_distance = self.routing_table.get_nodes_at_log_distance(
distance,
)
for node_id in node_ids_at_distance:
response_enrs.append(self.enr_db.get_enr(node_id))
else:
raise Exception("Should be unreachable")
await self.client.send_found_nodes(
request.sender_node_id,
request.sender_endpoint,
enrs=response_enrs,
request_id=request.request_id,
)
async def _handle_unhandled_talk_requests(self) -> None:
async with self.dispatcher.subscribe(TalkRequestMessage) as subscription:
async for request in subscription:
if request.message.protocol not in self._talk_protocols:
await self.client.send_talk_response(
request.sender_node_id,
request.sender_endpoint,
payload=b"",
request_id=request.message.request_id,
)
#
# Utility
#
async def endpoint_for_node_id(self, node_id: NodeID) -> Endpoint:
try:
enr = self.enr_db.get_enr(node_id)
except KeyError:
enr = await self.lookup_enr(node_id)
return Endpoint.from_enr(enr)
def routing_table(center_node_id, bucket_size):
return KademliaRoutingTable(center_node_id, bucket_size)
async def run(self) -> None:
identity_scheme_registry = default_identity_scheme_registry
message_type_registry = v5_registry
enr_database_dir = self._boot_info.base_dir / ENR_DATABASE_DIR_NAME
enr_database_dir.mkdir(exist_ok=True)
enr_db = ENRDB(LevelDB(enr_database_dir), identity_scheme_registry)
self.enr_db = enr_db
local_private_key = get_local_private_key(self._boot_info)
enr_manager = ENRManager(
private_key=local_private_key,
enr_db=enr_db,
)
port = self._boot_info.port
if b"udp" not in enr_manager.enr:
enr_manager.update((b"udp", port))
listen_on: AnyIPAddress
if self._boot_info.listen_on is None:
listen_on = DEFAULT_LISTEN
else:
listen_on = self._boot_info.listen_on
# Update the ENR if an explicit listening address was provided
enr_manager.update((IP_V4_ADDRESS_ENR_KEY, listen_on.packed))
if self._boot_info.is_upnp_enabled:
upnp_service = UPnPService(port)
self.manager.run_daemon_child_service(upnp_service)
routing_table = KademliaRoutingTable(enr_manager.enr.node_id,
ROUTING_TABLE_BUCKET_SIZE)
self.routing_table = routing_table
for enr in self._boot_info.bootnodes:
try:
enr_db.set_enr(enr)
except OldSequenceNumber:
pass
routing_table.update(enr.node_id)
sock = trio.socket.socket(family=trio.socket.AF_INET,
type=trio.socket.SOCK_DGRAM)
outbound_datagram_channels = trio.open_memory_channel[
OutboundDatagram](0)
inbound_datagram_channels = trio.open_memory_channel[InboundDatagram](
0)
outbound_packet_channels = trio.open_memory_channel[OutboundPacket](0)
inbound_packet_channels = trio.open_memory_channel[InboundPacket](0)
outbound_message_channels = trio.open_memory_channel[
AnyOutboundMessage](0)
inbound_message_channels = trio.open_memory_channel[AnyInboundMessage](
0)
endpoint_vote_channels = trio.open_memory_channel[EndpointVote](0)
# types ignored due to https://github.com/ethereum/async-service/issues/5
datagram_sender = DatagramSender( # type: ignore
outbound_datagram_channels[1], sock)
datagram_receiver = DatagramReceiver( # type: ignore
sock, inbound_datagram_channels[0])
packet_encoder = PacketEncoder( # type: ignore
outbound_packet_channels[1], outbound_datagram_channels[0])
packet_decoder = PacketDecoder( # type: ignore
inbound_datagram_channels[1], inbound_packet_channels[0])
packer = Packer(
local_private_key=local_private_key.to_bytes(),
local_node_id=enr_manager.enr.node_id,
enr_db=enr_db,
message_type_registry=message_type_registry,
inbound_packet_receive_channel=inbound_packet_channels[1],
inbound_message_send_channel=inbound_message_channels[0],
outbound_message_receive_channel=outbound_message_channels[1],
outbound_packet_send_channel=outbound_packet_channels[0],
)
message_dispatcher = MessageDispatcher(
enr_db=enr_db,
inbound_message_receive_channel=inbound_message_channels[1],
outbound_message_send_channel=outbound_message_channels[0],
)
endpoint_tracker = EndpointTracker(
local_private_key=local_private_key.to_bytes(),
local_node_id=enr_manager.enr.node_id,
enr_db=enr_db,
identity_scheme_registry=identity_scheme_registry,
vote_receive_channel=endpoint_vote_channels[1],
)
routing_table_manager = RoutingTableManager(
local_node_id=enr_manager.enr.node_id,
routing_table=routing_table,
message_dispatcher=message_dispatcher,
enr_db=enr_db,
outbound_message_send_channel=outbound_message_channels[0],
endpoint_vote_send_channel=endpoint_vote_channels[0],
)
self.logger.info(f"DDHT base dir: {self._boot_info.base_dir}")
self.logger.info("Starting discovery service...")
self.logger.info(f"Listening on {listen_on}:{port}")
self.logger.info(
f"Local Node ID: {encode_hex(enr_manager.enr.node_id)}")
self.logger.info(f"Local ENR: {enr_manager.enr}")
services = (
datagram_sender,
datagram_receiver,
packet_encoder,
packet_decoder,
packer,
message_dispatcher,
endpoint_tracker,
routing_table_manager,
)
await sock.bind((str(listen_on), port))
with sock:
for service in services:
self.manager.run_daemon_child_service(service)
await self.manager.wait_finished()
async def run(self) -> None:
identity_scheme_registry = default_identity_scheme_registry
enr_database_file = self._boot_info.base_dir / ENR_DATABASE_FILENAME
enr_db = QueryableENRDB(sqlite3.connect(enr_database_file),
identity_scheme_registry)
self.enr_db = enr_db
local_private_key = get_local_private_key(self._boot_info)
enr_manager = ENRManager(
private_key=local_private_key,
enr_db=enr_db,
)
port = self._boot_info.port
if b"udp" not in enr_manager.enr:
enr_manager.update((b"udp", port))
listen_on: AnyIPAddress
if self._boot_info.listen_on is None:
listen_on = DEFAULT_LISTEN
else:
listen_on = self._boot_info.listen_on
# Update the ENR if an explicit listening address was provided
enr_manager.update((IP_V4_ADDRESS_ENR_KEY, listen_on.packed))
if self._boot_info.is_upnp_enabled:
upnp_service = UPnPService(port)
self.manager.run_daemon_child_service(upnp_service)
routing_table = KademliaRoutingTable(enr_manager.enr.node_id,
ROUTING_TABLE_BUCKET_SIZE)
self.routing_table = routing_table
for enr in self._boot_info.bootnodes:
try:
enr_db.set_enr(enr)
except OldSequenceNumber:
pass
routing_table.update(enr.node_id)
sock = trio.socket.socket(family=trio.socket.AF_INET,
type=trio.socket.SOCK_DGRAM)
client = Client(local_private_key, enr_db, enr_manager.enr.node_id,
sock)
endpoint_vote_channels = trio.open_memory_channel[EndpointVote](0)
endpoint_tracker = EndpointTracker(
local_private_key=local_private_key.to_bytes(),
local_node_id=enr_manager.enr.node_id,
enr_db=enr_db,
identity_scheme_registry=identity_scheme_registry,
vote_receive_channel=endpoint_vote_channels[1],
)
routing_table_manager = RoutingTableManager(
local_node_id=enr_manager.enr.node_id,
routing_table=routing_table,
message_dispatcher=client.message_dispatcher,
enr_db=enr_db,
outbound_message_send_channel=client.outbound_message_send_channel,
endpoint_vote_send_channel=endpoint_vote_channels[0],
)
self.logger.info(f"DDHT base dir: {self._boot_info.base_dir}")
self.logger.info("Starting discovery service...")
self.logger.info(f"Listening on {listen_on}:{port}")
self.logger.info(
f"Local Node ID: {encode_hex(enr_manager.enr.node_id)}")
self.logger.info(f"Local ENR: {enr_manager.enr}")
services = (
client,
endpoint_tracker,
routing_table_manager,
)
await sock.bind((str(listen_on), port))
with sock:
for service in services:
self.manager.run_daemon_child_service(service)
await self.manager.wait_finished()
def empty_routing_table(local_enr):
routing_table = KademliaRoutingTable(local_enr.node_id, 16)
return routing_table
def routing_table(enr):
return KademliaRoutingTable(enr.node_id, ROUTING_TABLE_BUCKET_SIZE)
class AlexandriaNetwork(Service, AlexandriaNetworkAPI):
# Delegate to the AlexandriaClient for determining `protocol_id`
protocol_id = AlexandriaClient.protocol_id
def __init__(
self,
network: NetworkAPI,
bootnodes: Collection[ENRAPI],
storage: ContentStorageAPI,
) -> None:
self.logger = get_extended_debug_logger("ddht.Alexandria")
self._bootnodes = tuple(bootnodes)
self.client = AlexandriaClient(network)
self.routing_table = KademliaRoutingTable(
self.enr_manager.enr.node_id, ROUTING_TABLE_BUCKET_SIZE,
)
self._last_pong_at = LRU(2048)
self._routing_table_ready = trio.Event()
self._ping_handler_ready = trio.Event()
self._find_nodes_handler_ready = trio.Event()
self._find_content_handler_ready = trio.Event()
self.storage = storage
async def routing_table_ready(self) -> None:
await self._routing_table_ready.wait()
async def ready(self) -> None:
await self._ping_handler_ready.wait()
await self._find_nodes_handler_ready.wait()
await self._find_content_handler_ready.wait()
@property
def network(self) -> NetworkAPI:
return self.client.network
@property
def local_node_id(self) -> NodeID:
return self.network.local_node_id
@property
def enr_manager(self) -> ENRManagerAPI:
return self.network.enr_manager
@property
def enr_db(self) -> QueryableENRDatabaseAPI:
return self.network.enr_db
async def run(self) -> None:
# Long running processes
self.manager.run_daemon_task(self._periodically_report_routing_table)
self.manager.run_daemon_task(self._ping_oldest_routing_table_entry)
self.manager.run_daemon_task(self._track_last_pong)
self.manager.run_daemon_task(self._manage_routing_table)
self.manager.run_daemon_task(self._pong_when_pinged)
self.manager.run_daemon_task(self._serve_find_nodes)
self.manager.run_daemon_task(self._serve_find_content)
# Child services
self.manager.run_daemon_child_service(self.client)
# self.manager.run_daemon_child_service(self.radius_tracker)
await self.manager.wait_finished()
#
# Local properties
#
@property
def local_advertisement_radius(self) -> int:
return MAX_RADIUS
#
# High Level API
#
async def bond(
self, node_id: NodeID, *, endpoint: Optional[Endpoint] = None
) -> bool:
self.logger.debug2(
"Bonding with %s", node_id.hex(),
)
try:
pong = await self.ping(node_id, endpoint=endpoint)
except trio.TooSlowError:
self.logger.debug("Bonding with %s timed out during ping", node_id.hex())
return False
except KeyError:
self.logger.debug(
"Unable to lookup endpoint information for node: %s", node_id.hex()
)
return False
try:
enr = await self.lookup_enr(
node_id, enr_seq=pong.enr_seq, endpoint=endpoint
)
except trio.TooSlowError:
self.logger.debug(
"Bonding with %s timed out during ENR retrieval", node_id.hex(),
)
return False
self.routing_table.update(enr.node_id)
self.logger.debug(
"Bonded with %s successfully", node_id.hex(),
)
self._routing_table_ready.set()
return True
async def _bond(self, node_id: NodeID, endpoint: Optional[Endpoint] = None) -> None:
await self.bond(node_id, endpoint=endpoint)
async def lookup_enr(
self, node_id: NodeID, *, enr_seq: int = 0, endpoint: Optional[Endpoint] = None
) -> ENRAPI:
return await self.network.lookup_enr(
node_id, enr_seq=enr_seq, endpoint=endpoint
)
async def ping(
self,
node_id: NodeID,
*,
enr_seq: Optional[int] = None,
advertisement_radius: Optional[int] = None,
endpoint: Optional[Endpoint] = None,
request_id: Optional[bytes] = None,
) -> PongPayload:
if endpoint is None:
endpoint = await self.network.endpoint_for_node_id(node_id)
if enr_seq is None:
enr_seq = self.network.enr_manager.enr.sequence_number
if advertisement_radius is None:
advertisement_radius = self.local_advertisement_radius
response = await self.client.ping(
node_id,
enr_seq=enr_seq,
advertisement_radius=advertisement_radius,
endpoint=endpoint,
request_id=request_id,
)
return response.payload
async def find_nodes(
self,
node_id: NodeID,
*distances: int,
endpoint: Optional[Endpoint] = None,
request_id: Optional[bytes] = None,
) -> Tuple[ENRAPI, ...]:
if not distances:
raise TypeError("Must provide at least one distance")
if endpoint is None:
endpoint = await self.network.endpoint_for_node_id(node_id)
responses = await self.client.find_nodes(
node_id, endpoint, distances=distances, request_id=request_id
)
return tuple(
enr for response in responses for enr in response.message.payload.enrs
)
def recursive_find_nodes(
self, target: Union[NodeID, ContentID],
) -> AsyncContextManager[trio.abc.ReceiveChannel[ENRAPI]]:
return common_recursive_find_nodes(self, NodeID(target))
@asynccontextmanager
async def explore(
self, target: NodeID, concurrency: int = 3,
) -> AsyncIterator[trio.abc.ReceiveChannel[ENRAPI]]:
explorer = Explorer(self, target, concurrency)
with trio.move_on_after(300) as scope:
async with background_trio_service(explorer):
await explorer.ready()
async with explorer.stream() as receive_channel:
yield receive_channel
if scope.cancelled_caught:
self.logger.error("Timeout from explore")
async def find_content(
self,
node_id: NodeID,
*,
content_key: ContentKey,
endpoint: Optional[Endpoint] = None,
request_id: Optional[bytes] = None,
) -> FoundContentPayload:
if endpoint is None:
endpoint = await self.network.endpoint_for_node_id(node_id)
response = await self.client.find_content(
node_id, endpoint, content_key=content_key, request_id=request_id,
)
if response.payload.is_content and response.payload.encoded_enrs:
raise ValidationError("Content response with non-empty ENR payload")
elif response.payload.is_content and not response.payload.content:
raise ValidationError("Content response with empty content")
return response.payload
@asynccontextmanager
async def recursive_find_content(
self, content_key: ContentKey,
) -> AsyncIterator[trio.abc.ReceiveChannel[bytes]]:
seeker = Seeker(self, content_key)
async with background_trio_service(seeker):
yield seeker.content_receive
async def retrieve_content(self, content_key: ContentKey) -> bytes:
async with self.recursive_find_content(content_key) as content_aiter:
return await content_aiter.receive()
#
# Long Running Processes
#
async def _periodically_report_routing_table(self) -> None:
async for _ in every(30, initial_delay=30):
non_empty_buckets = tuple(
(idx, bucket)
for idx, bucket in enumerate(reversed(self.routing_table.buckets))
if bucket
)
total_size = sum(len(bucket) for idx, bucket in non_empty_buckets)
bucket_info = "|".join(
tuple(f"{idx}:{len(bucket)}" for idx, bucket in non_empty_buckets)
)
self.logger.debug(
"routing-table-info: size=%d buckets=%s", total_size, bucket_info,
)
async def _pong_when_pinged(self) -> None:
async with self.client.subscribe(PingMessage) as subscription:
self._ping_handler_ready.set()
async for request in subscription:
await self.client.send_pong(
request.sender_node_id,
request.sender_endpoint,
enr_seq=self.enr_manager.enr.sequence_number,
advertisement_radius=self.local_advertisement_radius,
request_id=request.request_id,
)
enr = await self.lookup_enr(
request.sender_node_id,
enr_seq=request.message.payload.enr_seq,
endpoint=request.sender_endpoint,
)
self.routing_table.update(enr.node_id)
self._routing_table_ready.set()
def _source_nodes(self, distances: Tuple[int, ...]) -> Tuple[ENRAPI, ...]:
response_enrs: List[ENRAPI] = []
unique_distances = set(distances)
if len(unique_distances) != len(distances):
raise ValidationError("duplicate distances")
elif not distances:
raise ValidationError("empty distances")
elif any(distance > self.routing_table.num_buckets for distance in distances):
raise ValidationError("invalid distances")
for distance in distances:
if distance == 0:
response_enrs.append(self.enr_manager.enr)
elif distance <= self.routing_table.num_buckets:
node_ids_at_distance = self.routing_table.get_nodes_at_log_distance(
distance,
)
for node_id in node_ids_at_distance:
response_enrs.append(self.enr_db.get_enr(node_id))
else:
raise Exception("Should be unreachable")
return tuple(response_enrs)
async def _serve_find_nodes(self) -> None:
async with self.client.subscribe(FindNodesMessage) as subscription:
self._find_nodes_handler_ready.set()
async for request in subscription:
try:
response_enrs = self._source_nodes(
request.message.payload.distances
)
except ValidationError as err:
self.logger.debug(
"Ignoring invalid FindNodesMessage from %s@%s: %s",
request.sender_node_id.hex(),
request.sender_endpoint,
err,
)
else:
await self.client.send_found_nodes(
request.sender_node_id,
request.sender_endpoint,
enrs=response_enrs,
request_id=request.request_id,
)
async def _serve_find_content(self) -> None:
async with self.client.subscribe(FindContentMessage) as subscription:
self._find_content_handler_ready.set()
async for request in subscription:
# if content in storage, serve it....
# else serve ENR records that we know of which are *closest*
content_key = request.message.payload.content_key
if self.storage.has_content(content_key):
content = self.storage.get_content(content_key)
await self.client.send_found_content(
request.sender_node_id,
request.sender_endpoint,
enrs=None,
content=content,
request_id=request.request_id,
)
else:
content_id = content_key_to_content_id(content_key)
distance = compute_content_distance(self.local_node_id, content_id)
try:
response_enrs = self._source_nodes((distance,))
except ValidationError as err:
self.logger.debug(
"Ignoring invalid FindNodesMessage from %s@%s: %s",
request.sender_node_id.hex(),
request.sender_endpoint,
err,
)
else:
await self.client.send_found_content(
request.sender_node_id,
request.sender_endpoint,
enrs=response_enrs,
content=None,
request_id=request.request_id,
)
async def _ping_oldest_routing_table_entry(self) -> None:
await self._routing_table_ready.wait()
while self.manager.is_running:
# Here we preserve the lazy iteration while still checking that the
# iterable is not empty before passing it into `min` below which
# throws an ambiguous `ValueError` otherwise if the iterable is
# empty.
nodes_iter = self.routing_table.iter_all_random()
try:
first_node_id = first(nodes_iter)
except StopIteration:
await trio.sleep(ROUTING_TABLE_KEEP_ALIVE)
continue
else:
least_recently_ponged_node_id = min(
cons(first_node_id, nodes_iter),
key=lambda node_id: self._last_pong_at.get(node_id, 0),
)
too_old_at = trio.current_time() - ROUTING_TABLE_KEEP_ALIVE
try:
last_pong_at = self._last_pong_at[least_recently_ponged_node_id]
except KeyError:
pass
else:
if last_pong_at > too_old_at:
await trio.sleep(last_pong_at - too_old_at)
continue
did_bond = await self.bond(least_recently_ponged_node_id)
if not did_bond:
self.routing_table.remove(least_recently_ponged_node_id)
async def _track_last_pong(self) -> None:
async with self.client.subscribe(PongMessage) as subscription:
async for message in subscription:
self._last_pong_at[message.sender_node_id] = trio.current_time()
async def _manage_routing_table(self) -> None:
# First load all the bootnode ENRs into our database
for enr in self._bootnodes:
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
# Now repeatedly try to bond with each bootnode until one succeeds.
while self.manager.is_running:
with trio.move_on_after(10):
async with trio.open_nursery() as nursery:
for enr in self._bootnodes:
if enr.node_id == self.local_node_id:
continue
endpoint = Endpoint.from_enr(enr)
nursery.start_soon(self._bond, enr.node_id, endpoint)
await self._routing_table_ready.wait()
break
# Now we enter into an infinite loop that continually probes the
# network to beep the routing table fresh. We both perform completely
# random lookups, as well as targeted lookups on the outermost routing
# table buckets which are not full.
#
# The `TokenBucket` allows us to burst at the beginning, making quick
# successive probes, then slowing down once the
#
# TokenBucket starts with 10 tokens, refilling at 1 token every 30
# seconds.
token_bucket = TokenBucket(1 / 30, 10)
async with trio.open_nursery() as nursery:
while self.manager.is_running:
await token_bucket.take()
# Get the logarithmic distance to the "largest" buckets
# that are not full.
non_full_bucket_distances = tuple(
idx + 1
for idx, bucket in enumerate(self.routing_table.buckets)
if len(bucket) < self.routing_table.bucket_size # noqa: E501
)[-16:]
# Probe one of the not-full-buckets with a weighted preference
# towards the largest buckets.
distance_to_probe = weighted_choice(non_full_bucket_distances)
target_node_id = at_log_distance(self.local_node_id, distance_to_probe)
async with self.recursive_find_nodes(target_node_id) as enr_aiter:
async for enr in enr_aiter:
if enr.node_id == self.local_node_id:
continue
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
pass
nursery.start_soon(self._bond, enr.node_id)