def test_routingtable_split_bucket():
table = RoutingTable(NodeFactory().id)
assert len(table.buckets) == 1
old_bucket = table.buckets[0]
table.split_bucket(0)
assert len(table.buckets) == 2
assert old_bucket not in table.buckets
def test_routingtable_iter_random():
table = RoutingTable(NodeFactory().id)
nodes_in_insertion_order = []
# Use a relatively high number of nodes here otherwise we could have two consecutive calls
# yielding nodes in the same order.
for _ in range(100):
node = NodeFactory()
assert table.add_node(node) is None
nodes_in_insertion_order.append(node)
nodes_in_iteration_order = [node for node in table.iter_random()]
# We iterate over all nodes
assert len(nodes_in_iteration_order) == len(table) == len(
nodes_in_insertion_order)
# No repeated nodes are returned
assert len(set(nodes_in_iteration_order)) == len(nodes_in_iteration_order)
# The order in which we iterate is not the same as the one in which nodes were inserted.
assert nodes_in_iteration_order != nodes_in_insertion_order
second_iteration_order = [node for node in table.iter_random()]
# Multiple calls should yield the same nodes, but in a different order.
assert set(nodes_in_iteration_order) == set(second_iteration_order)
assert nodes_in_iteration_order != second_iteration_order
def test_routingtable_remove_node():
table = RoutingTable(NodeFactory().id)
node1 = NodeFactory()
assert table.add_node(node1) is None
assert node1 in table
table.remove_node(node1)
assert node1 not in table
def test_routingtable_add_node():
table = RoutingTable(NodeFactory().id)
for i in range(table.buckets[0].size):
# As long as the bucket is not full, the new node is added to the bucket and None is
# returned.
assert table.add_node(NodeFactory()) is None
assert len(table.buckets) == 1
assert len(table) == i + 1
assert table.buckets[0].is_full
# Now that the bucket is full, an add_node() should cause it to be split.
assert table.add_node(NodeFactory()) is None
def test_routingtable_get_random_nodes():
table = RoutingTable(NodeFactory())
for _ in range(100):
assert table.add_node(NodeFactory()) is None
nodes = list(table.get_random_nodes(50))
assert len(nodes) == 50
assert len(set(nodes)) == 50
# If we ask for more nodes than what the routing table contains, we'll get only what the
# routing table contains, without duplicates.
nodes = list(table.get_random_nodes(200))
assert len(nodes) == 100
assert len(set(nodes)) == 100
def test_routingtable_neighbours():
table = RoutingTable(NodeFactory().id)
for i in range(1000):
assert table.add_node(NodeFactory()) is None
assert i == len(table) - 1
for _ in range(100):
node = NodeFactory()
nearest_bucket = table.buckets_by_distance_to(node._id_int)[0]
if not nearest_bucket.nodes:
continue
# Change nodeid to something that is in this bucket's range.
node_a = nearest_bucket.nodes[0]
node_b = NodeFactory.with_nodeid(node_a._id_int + 1)
assert node_a == table.neighbours(node_b.id)[0]
def __init__(self,
privkey: datatypes.PrivateKey,
address: AddressAPI,
bootstrap_nodes: Sequence[NodeAPI],
cancel_token: CancelToken) -> None:
self.privkey = privkey
self.address = address
self.bootstrap_nodes = bootstrap_nodes
self.this_node = Node(self.pubkey, address)
self.routing = RoutingTable(self.this_node)
self.topic_table = TopicTable(self.logger)
self.pong_callbacks = CallbackManager()
self.ping_callbacks = CallbackManager()
self.neighbours_callbacks = CallbackManager()
self.topic_nodes_callbacks = CallbackManager()
self.parity_pong_tokens: Dict[Hash32, Hash32] = {}
self.cancel_token = CancelToken('DiscoveryProtocol').chain(cancel_token)
def __init__(self, privkey: datatypes.PrivateKey, address: AddressAPI,
bootstrap_nodes: Sequence[NodeAPI], event_bus: EndpointAPI,
socket: trio.socket.SocketType) -> None:
self.privkey = privkey
self.address = address
self.bootstrap_nodes = bootstrap_nodes
self._event_bus = event_bus
self.this_node = Node(self.pubkey, address)
self.routing = RoutingTable(self.this_node)
self.pong_callbacks = CallbackManager()
self.ping_callbacks = CallbackManager()
self.neighbours_callbacks = CallbackManager()
self.parity_pong_tokens: Dict[Hash32, Hash32] = {}
if socket.family != trio.socket.AF_INET:
raise ValueError("Invalid socket family")
elif socket.type != trio.socket.SOCK_DGRAM:
raise ValueError("Invalid socket type")
self.socket = socket
def __init__(
self,
privkey: datatypes.PrivateKey,
address: AddressAPI,
bootstrap_nodes: Sequence[NodeAPI],
event_bus: EndpointAPI,
socket: trio.socket.SocketType,
enr_field_providers: Sequence[ENR_FieldProvider] = tuple(),
) -> None:
self.privkey = privkey
self.address = address
self.bootstrap_nodes = bootstrap_nodes
self._event_bus = event_bus
self.this_node = Node(self.pubkey, address)
self.routing = RoutingTable(self.this_node)
self.enr_response_channels = ExpectedResponseChannels[Tuple[ENR,
Hash32]]()
self.pong_channels = ExpectedResponseChannels[Tuple[Hash32, int]]()
self.neighbours_channels = ExpectedResponseChannels[List[NodeAPI]]()
self.ping_channels = ExpectedResponseChannels[None]()
self.enr_field_providers = enr_field_providers
# FIXME: Use a persistent EnrDb implementation.
self._enr_db = MemoryEnrDb(default_identity_scheme_registry)
# FIXME: Use a concurrency-safe EnrDb implementation.
self._enr_db_lock = trio.Lock()
self._local_enr: ENR = None
self._local_enr_next_refresh: float = time.monotonic()
self._local_enr_lock = trio.Lock()
self.parity_pong_tokens: Dict[Hash32, Hash32] = {}
if socket.family != trio.socket.AF_INET:
raise ValueError("Invalid socket family")
elif socket.type != trio.socket.SOCK_DGRAM:
raise ValueError("Invalid socket type")
self.socket = socket
self.pending_enrs_producer, self.pending_enrs_consumer = trio.open_memory_channel[
Tuple[NodeAPI, int]](self._max_pending_enrs)
def test_routingtable_add_node_error():
table = RoutingTable(NodeFactory().id)
with pytest.raises(ValueError):
table.add_node(NodeFactory.with_nodeid(KADEMLIA_MAX_NODE_ID + 1))
class DiscoveryService(Service):
_refresh_interval: int = 30
_max_neighbours_per_packet_cache = None
logger = get_extended_debug_logger('p2p.discovery.DiscoveryService')
def __init__(self, privkey: datatypes.PrivateKey, address: AddressAPI,
bootstrap_nodes: Sequence[NodeAPI], event_bus: EndpointAPI,
socket: trio.socket.SocketType) -> None:
self.privkey = privkey
self.address = address
self.bootstrap_nodes = bootstrap_nodes
self._event_bus = event_bus
self.this_node = Node(self.pubkey, address)
self.routing = RoutingTable(self.this_node)
self.pong_callbacks = CallbackManager()
self.ping_callbacks = CallbackManager()
self.neighbours_callbacks = CallbackManager()
self.parity_pong_tokens: Dict[Hash32, Hash32] = {}
if socket.family != trio.socket.AF_INET:
raise ValueError("Invalid socket family")
elif socket.type != trio.socket.SOCK_DGRAM:
raise ValueError("Invalid socket type")
self.socket = socket
async def consume_datagrams(self) -> None:
while self.manager.is_running:
await self.consume_datagram()
async def handle_get_peer_candidates_requests(self) -> None:
async for event in self._event_bus.stream(PeerCandidatesRequest):
nodes = tuple(self.get_nodes_to_connect(event.max_candidates))
self.logger.debug2("Broadcasting peer candidates (%s)", nodes)
await self._event_bus.broadcast(
event.expected_response_type()(nodes),
event.broadcast_config())
async def handle_get_random_bootnode_requests(self) -> None:
async for event in self._event_bus.stream(RandomBootnodeRequest):
nodes = tuple(self.get_random_bootnode())
self.logger.debug2("Broadcasting random boot nodes (%s)", nodes)
await self._event_bus.broadcast(
event.expected_response_type()(nodes),
event.broadcast_config())
async def run(self) -> None:
self.manager.run_daemon_task(self.handle_get_peer_candidates_requests)
self.manager.run_daemon_task(self.handle_get_random_bootnode_requests)
self.manager.run_daemon_task(self.periodically_refresh)
self.manager.run_daemon_task(self.consume_datagrams)
self.manager.run_task(self.bootstrap)
await self.manager.wait_finished()
async def periodically_refresh(self) -> None:
async for _ in trio_utils.every(self._refresh_interval):
await self.lookup_random()
def update_routing_table(self, node: NodeAPI) -> None:
"""Update the routing table entry for the given node."""
eviction_candidate = self.routing.add_node(node)
if eviction_candidate:
# This means we couldn't add the node because its bucket is full, so schedule a bond()
# with the least recently seen node on that bucket. If the bonding fails the node will
# be removed from the bucket and a new one will be picked from the bucket's
# replacement cache.
self.manager.run_task(self.bond, eviction_candidate)
async def bond(self, node: NodeAPI) -> bool:
"""Bond with the given node.
Bonding consists of pinging the node, waiting for a pong and maybe a ping as well.
It is necessary to do this at least once before we send find_node requests to a node.
"""
if node in self.routing:
return True
elif node == self.this_node:
return False
token = self.send_ping_v4(node)
log_version = "v4"
try:
got_pong = await self.wait_pong_v4(node, token)
except AlreadyWaitingDiscoveryResponse:
self.logger.debug("bonding failed, awaiting %s pong from %s",
log_version, node)
return False
if not got_pong:
self.logger.debug("bonding failed, didn't receive %s pong from %s",
log_version, node)
self.routing.remove_node(node)
return False
try:
# Give the remote node a chance to ping us before we move on and
# start sending find_node requests. It is ok for wait_ping() to
# timeout and return false here as that just means the remote
# remembers us.
await self.wait_ping(node)
except AlreadyWaitingDiscoveryResponse:
self.logger.debug("bonding failed, already waiting for ping")
return False
self.logger.debug("bonding completed successfully with %s", node)
self.update_routing_table(node)
return True
async def wait_ping(self, remote: NodeAPI) -> bool:
"""Wait for a ping from the given remote.
This coroutine adds a callback to ping_callbacks and yields control until that callback is
called or a timeout (k_request_timeout) occurs. At that point it returns whether or not
a ping was received from the given node.
"""
event = trio.Event()
with self.ping_callbacks.acquire(remote, event.set):
with trio.move_on_after(
constants.KADEMLIA_REQUEST_TIMEOUT) as cancel_scope:
await event.wait()
if cancel_scope.cancelled_caught:
self.logger.debug2('timed out waiting for ping from %s',
remote)
got_ping = False
else:
self.logger.debug2('got expected ping from %s', remote)
got_ping = True
return got_ping
async def wait_pong_v4(self, remote: NodeAPI, token: Hash32) -> bool:
event = trio.Event()
callback = event.set
return await self._wait_pong(remote, token, event, callback)
async def _wait_pong(self, remote: NodeAPI, token: Hash32,
event: trio.Event, callback: Callable[...,
Any]) -> bool:
"""Wait for a pong from the given remote containing the given token.
This coroutine adds a callback to pong_callbacks and yields control until the given event
is set or a timeout (k_request_timeout) occurs. At that point it returns whether or not
a pong was received with the given pingid.
"""
pingid = self._mkpingid(token, remote)
with self.pong_callbacks.acquire(pingid, callback):
with trio.move_on_after(
constants.KADEMLIA_REQUEST_TIMEOUT) as cancel_scope:
await event.wait()
if cancel_scope.cancelled_caught:
got_pong = False
self.logger.debug2(
'timed out waiting for pong from %s (token == %s)',
remote,
encode_hex(token),
)
else:
got_pong = True
self.logger.debug2('got expected pong with token %s',
encode_hex(token))
return got_pong
async def wait_neighbours(self, remote: NodeAPI) -> Tuple[NodeAPI, ...]:
"""Wait for a neihgbours packet from the given node.
Returns the list of neighbours received.
"""
event = trio.Event()
neighbours: List[NodeAPI] = []
def process(response: List[NodeAPI]) -> None:
neighbours.extend(response)
# This callback is expected to be called multiple times because nodes usually
# split the neighbours replies into multiple packets, so we only call event.set() once
# we've received enough neighbours.
if len(neighbours) >= constants.KADEMLIA_BUCKET_SIZE:
event.set()
with self.neighbours_callbacks.acquire(remote, process):
with trio.move_on_after(
constants.KADEMLIA_REQUEST_TIMEOUT) as cancel_scope:
await event.wait()
self.logger.debug2('got expected neighbours response from %s',
remote)
if cancel_scope.cancelled_caught:
self.logger.debug2(
'timed out waiting for %d neighbours from %s',
constants.KADEMLIA_BUCKET_SIZE,
remote,
)
return tuple(n for n in neighbours if n != self.this_node)
def _mkpingid(self, token: Hash32, node: NodeAPI) -> Hash32:
return Hash32(token + node.pubkey.to_bytes())
def _send_find_node(self, node: NodeAPI, target_node_id: int) -> None:
self.send_find_node_v4(node, target_node_id)
async def lookup(self, node_id: int) -> Tuple[NodeAPI, ...]:
"""Lookup performs a network search for nodes close to the given target.
It approaches the target by querying nodes that are closer to it on each iteration. The
given target does not need to be an actual node identifier.
"""
nodes_asked: Set[NodeAPI] = set()
nodes_seen: Set[NodeAPI] = set()
async def _find_node(node_id: int,
remote: NodeAPI) -> Tuple[NodeAPI, ...]:
self._send_find_node(remote, node_id)
candidates = await self.wait_neighbours(remote)
if not candidates:
self.logger.debug("got no candidates from %s, returning",
remote)
return tuple()
all_candidates = tuple(c for c in candidates
if c not in nodes_seen)
candidates = tuple(c for c in all_candidates
if (not self.ping_callbacks.locked(c)
and not self.pong_callbacks.locked(c)))
self.logger.debug2("got %s new candidates", len(candidates))
# Add new candidates to nodes_seen so that we don't attempt to bond with failing ones
# in the future.
nodes_seen.update(candidates)
bonded = await trio_utils.gather(*((self.bond, c)
for c in candidates))
self.logger.debug2("bonded with %s candidates", bonded.count(True))
return tuple(c for c in candidates if bonded[candidates.index(c)])
def _exclude_if_asked(nodes: Iterable[NodeAPI]) -> List[NodeAPI]:
nodes_to_ask = list(set(nodes).difference(nodes_asked))
return sort_by_distance(
nodes_to_ask, node_id)[:constants.KADEMLIA_FIND_CONCURRENCY]
closest = self.routing.neighbours(node_id)
self.logger.debug("starting lookup; initial neighbours: %s", closest)
nodes_to_ask = _exclude_if_asked(closest)
while nodes_to_ask:
self.logger.debug2("node lookup; querying %s", nodes_to_ask)
nodes_asked.update(nodes_to_ask)
next_find_node_queries = (
(_find_node, node_id, n) for n in nodes_to_ask
if not self.neighbours_callbacks.locked(n))
results = await trio_utils.gather(*next_find_node_queries)
for candidates in results:
closest.extend(candidates)
closest = sort_by_distance(
closest, node_id)[:constants.KADEMLIA_BUCKET_SIZE]
nodes_to_ask = _exclude_if_asked(closest)
self.logger.debug(
"lookup finished for target %s; closest neighbours: %s",
to_hex(node_id), closest)
return tuple(closest)
async def lookup_random(self) -> Tuple[NodeAPI, ...]:
return await self.lookup(
random.randint(0, constants.KADEMLIA_MAX_NODE_ID))
def get_random_bootnode(self) -> Iterator[NodeAPI]:
if self.bootstrap_nodes:
yield random.choice(self.bootstrap_nodes)
else:
self.logger.warning('No bootnodes available')
def get_nodes_to_connect(self, count: int) -> Iterator[NodeAPI]:
return self.routing.get_random_nodes(count)
@property
def pubkey(self) -> datatypes.PublicKey:
return self.privkey.public_key
def _get_handler(self, cmd: DiscoveryCommand) -> V4_HANDLER_TYPE:
if cmd == CMD_PING:
return self.recv_ping_v4
elif cmd == CMD_PONG:
return self.recv_pong_v4
elif cmd == CMD_FIND_NODE:
return self.recv_find_node_v4
elif cmd == CMD_NEIGHBOURS:
return self.recv_neighbours_v4
else:
raise ValueError(f"Unknown command: {cmd}")
@classmethod
def _get_max_neighbours_per_packet(cls) -> int:
if cls._max_neighbours_per_packet_cache is not None:
return cls._max_neighbours_per_packet_cache
cls._max_neighbours_per_packet_cache = _get_max_neighbours_per_packet()
return cls._max_neighbours_per_packet_cache
async def bootstrap(self) -> None:
for node in self.bootstrap_nodes:
uri = node.uri()
pubkey, _, uri_tail = uri.partition('@')
pubkey_head = pubkey[:16]
pubkey_tail = pubkey[-8:]
self.logger.debug("full-bootnode: %s", uri)
self.logger.debug("bootnode: %s...%s@%s", pubkey_head, pubkey_tail,
uri_tail)
bonding_queries = ((self.bond, n) for n in self.bootstrap_nodes
if (not self.ping_callbacks.locked(n)
and not self.pong_callbacks.locked(n)))
bonded = await trio_utils.gather(*bonding_queries)
if not any(bonded):
self.logger.info("Failed to bond with bootstrap nodes %s",
self.bootstrap_nodes)
return
await self.lookup_random()
def send(self, node: NodeAPI, msg_type: DiscoveryCommand,
payload: Sequence[Any]) -> bytes:
message = _pack_v4(msg_type.id, payload, self.privkey)
self.manager.run_task(self.socket.sendto, message,
(node.address.ip, node.address.udp_port))
return message
async def consume_datagram(self) -> None:
datagram, (ip_address, port) = await self.socket.recvfrom(
constants.DISCOVERY_DATAGRAM_BUFFER_SIZE)
address = Address(ip_address, port)
self.logger.debug2("Received datagram from %s", address)
self.receive(address, datagram)
def receive(self, address: AddressAPI, message: bytes) -> None:
try:
remote_pubkey, cmd_id, payload, message_hash = _unpack_v4(message)
except DefectiveMessage as e:
self.logger.error('error unpacking message (%s) from %s: %s',
message, address, e)
return
# As of discovery version 4, expiration is the last element for all packets, so
# we can validate that here, but if it changes we may have to do so on the
# handler methods.
expiration = rlp.sedes.big_endian_int.deserialize(payload[-1])
if time.time() > expiration:
self.logger.debug('received message already expired')
return
cmd = CMD_ID_MAP[cmd_id]
if len(payload) != cmd.elem_count:
self.logger.error('invalid %s payload: %s', cmd.name, payload)
return
node = Node(remote_pubkey, address)
handler = self._get_handler(cmd)
handler(node, payload, message_hash)
def recv_pong_v4(self, node: NodeAPI, payload: Sequence[Any],
_: Hash32) -> None:
# The pong payload should have 3 elements: to, token, expiration
_, token, _ = payload
self.logger.debug2('<<< pong (v4) from %s (token == %s)', node,
encode_hex(token))
self.process_pong_v4(node, token)
def recv_neighbours_v4(self, node: NodeAPI, payload: Sequence[Any],
_: Hash32) -> None:
# The neighbours payload should have 2 elements: nodes, expiration
nodes, _ = payload
neighbours = _extract_nodes_from_payload(node.address, nodes,
self.logger)
self.logger.debug2('<<< neighbours from %s: %s', node, neighbours)
self.process_neighbours(node, neighbours)
def recv_ping_v4(self, node: NodeAPI, _: Any,
message_hash: Hash32) -> None:
self.logger.debug2('<<< ping(v4) from %s', node)
self.process_ping(node, message_hash)
self.send_pong_v4(node, message_hash)
def recv_find_node_v4(self, node: NodeAPI, payload: Sequence[Any],
_: Hash32) -> None:
# The find_node payload should have 2 elements: node_id, expiration
self.logger.debug2('<<< find_node from %s', node)
node_id, _ = payload
if node not in self.routing:
# FIXME: This is not correct; a node we've bonded before may have become unavailable
# and thus removed from self.routing, but once it's back online we should accept
# find_nodes from them.
self.logger.debug(
'Ignoring find_node request from unknown node %s', node)
return
self.update_routing_table(node)
found = self.routing.neighbours(big_endian_to_int(node_id))
self.send_neighbours_v4(node, found)
def send_ping_v4(self, node: NodeAPI) -> Hash32:
version = rlp.sedes.big_endian_int.serialize(PROTO_VERSION)
payload = (version, self.address.to_endpoint(),
node.address.to_endpoint())
message = self.send(node, CMD_PING, payload)
# Return the msg hash, which is used as a token to identify pongs.
token = Hash32(message[:MAC_SIZE])
self.logger.debug2('>>> ping (v4) %s (token == %s)', node,
encode_hex(token))
# XXX: This hack is needed because there are lots of parity 1.10 nodes out there that send
# the wrong token on pong msgs (https://github.com/paritytech/parity/issues/8038). We
# should get rid of this once there are no longer too many parity 1.10 nodes out there.
parity_token = keccak(message[HEAD_SIZE + 1:])
self.parity_pong_tokens[parity_token] = token
return token
def send_find_node_v4(self, node: NodeAPI, target_node_id: int) -> None:
node_id = int_to_big_endian(target_node_id).rjust(
constants.KADEMLIA_PUBLIC_KEY_SIZE // 8, b'\0')
self.logger.debug2('>>> find_node to %s', node)
self.send(node, CMD_FIND_NODE, tuple([node_id]))
def send_pong_v4(self, node: NodeAPI, token: Hash32) -> None:
self.logger.debug2('>>> pong %s', node)
payload = (node.address.to_endpoint(), token)
self.send(node, CMD_PONG, payload)
def send_neighbours_v4(self, node: NodeAPI,
neighbours: List[NodeAPI]) -> None:
nodes = []
neighbours = sorted(neighbours)
for n in neighbours:
nodes.append(n.address.to_endpoint() + [n.pubkey.to_bytes()])
max_neighbours = self._get_max_neighbours_per_packet()
for i in range(0, len(nodes), max_neighbours):
self.logger.debug2('>>> neighbours to %s: %s', node,
neighbours[i:i + max_neighbours])
self.send(node, CMD_NEIGHBOURS,
tuple([nodes[i:i + max_neighbours]]))
def process_neighbours(self, remote: NodeAPI,
neighbours: List[NodeAPI]) -> None:
"""Process a neighbours response.
Neighbours responses should only be received as a reply to a find_node, and that is only
done as part of node lookup, so the actual processing is left to the callback from
neighbours_callbacks, which is added (and removed after it's done or timed out) in
wait_neighbours().
"""
try:
callback = self.neighbours_callbacks.get_callback(remote)
except KeyError:
self.logger.debug(
'unexpected neighbours from %s, probably came too late',
remote)
else:
callback(neighbours)
def process_pong_v4(self, remote: NodeAPI, token: Hash32) -> None:
"""Process a pong packet.
Pong packets should only be received as a response to a ping, so the actual processing is
left to the callback from pong_callbacks, which is added (and removed after it's done
or timed out) in wait_pong().
"""
# XXX: This hack is needed because there are lots of parity 1.10 nodes out there that send
# the wrong token on pong msgs (https://github.com/paritytech/parity/issues/8038). We
# should get rid of this once there are no longer too many parity 1.10 nodes out there.
if token in self.parity_pong_tokens:
# This is a pong from a buggy parity node, so need to lookup the actual token we're
# expecting.
token = self.parity_pong_tokens.pop(token)
else:
# This is a pong from a non-buggy node, so just cleanup self.parity_pong_tokens.
self.parity_pong_tokens = eth_utils.toolz.valfilter(
lambda val: val != token, self.parity_pong_tokens)
pingid = self._mkpingid(token, remote)
try:
callback = self.pong_callbacks.get_callback(pingid)
except KeyError:
self.logger.debug('unexpected v4 pong from %s (token == %s)',
remote, encode_hex(token))
else:
callback()
def process_ping(self, remote: NodeAPI, hash_: Hash32) -> None:
"""Process a received ping packet.
A ping packet may come any time, unrequested, or may be prompted by us bond()ing with a
new node. In the former case we'll just update the sender's entry in our routing table and
reply with a pong, whereas in the latter we'll also fire a callback from ping_callbacks.
"""
if remote == self.this_node:
self.logger.info('Invariant: received ping from this_node: %s',
remote)
return
else:
self.update_routing_table(remote)
# Sometimes a ping will be sent to us as part of the bonding
# performed the first time we see a node, and it is in those cases that
# a callback will exist.
try:
callback = self.ping_callbacks.get_callback(remote)
except KeyError:
pass
else:
callback()
class DiscoveryService(Service):
_refresh_interval: int = 30
_max_neighbours_per_packet_cache = None
# Maximum number of ENR retrieval requests active at any moment. Need to be a relatively high
# number as during a lookup() we'll bond and fetch ENRs of many nodes.
_max_pending_enrs: int = 20
_local_enr_refresh_interval: int = 60
logger = get_extended_debug_logger('p2p.discovery.DiscoveryService')
def __init__(
self,
privkey: datatypes.PrivateKey,
address: AddressAPI,
bootstrap_nodes: Sequence[NodeAPI],
event_bus: EndpointAPI,
socket: trio.socket.SocketType,
enr_field_providers: Sequence[ENR_FieldProvider] = tuple(),
) -> None:
self.privkey = privkey
self.address = address
self.bootstrap_nodes = bootstrap_nodes
self._event_bus = event_bus
self.this_node = Node(self.pubkey, address)
self.routing = RoutingTable(self.this_node)
self.enr_response_channels = ExpectedResponseChannels[Tuple[ENR,
Hash32]]()
self.pong_channels = ExpectedResponseChannels[Tuple[Hash32, int]]()
self.neighbours_channels = ExpectedResponseChannels[List[NodeAPI]]()
self.ping_channels = ExpectedResponseChannels[None]()
self.enr_field_providers = enr_field_providers
# FIXME: Use a persistent EnrDb implementation.
self._enr_db = MemoryEnrDb(default_identity_scheme_registry)
# FIXME: Use a concurrency-safe EnrDb implementation.
self._enr_db_lock = trio.Lock()
self._local_enr: ENR = None
self._local_enr_next_refresh: float = time.monotonic()
self._local_enr_lock = trio.Lock()
self.parity_pong_tokens: Dict[Hash32, Hash32] = {}
if socket.family != trio.socket.AF_INET:
raise ValueError("Invalid socket family")
elif socket.type != trio.socket.SOCK_DGRAM:
raise ValueError("Invalid socket type")
self.socket = socket
self.pending_enrs_producer, self.pending_enrs_consumer = trio.open_memory_channel[
Tuple[NodeAPI, int]](self._max_pending_enrs)
async def consume_datagrams(self) -> None:
while self.manager.is_running:
await self.consume_datagram()
async def handle_get_peer_candidates_requests(self) -> None:
async for event in self._event_bus.stream(PeerCandidatesRequest):
nodes = tuple(self.get_nodes_to_connect(event.max_candidates))
self.logger.debug2("Broadcasting peer candidates (%s)", nodes)
await self._event_bus.broadcast(
event.expected_response_type()(nodes),
event.broadcast_config())
async def handle_get_random_bootnode_requests(self) -> None:
async for event in self._event_bus.stream(RandomBootnodeRequest):
nodes = tuple(self.get_random_bootnode())
self.logger.debug2("Broadcasting random boot nodes (%s)", nodes)
await self._event_bus.broadcast(
event.expected_response_type()(nodes),
event.broadcast_config())
async def run(self) -> None:
await self.load_local_enr()
self.run_daemons_and_bootstrap()
await self.manager.wait_finished()
def run_daemons_and_bootstrap(self) -> None:
self.manager.run_daemon_task(self.handle_get_peer_candidates_requests)
self.manager.run_daemon_task(self.handle_get_random_bootnode_requests)
self.manager.run_daemon_task(self.periodically_refresh)
self.manager.run_daemon_task(self.report_stats)
self.manager.run_daemon_task(self.fetch_enrs)
self.manager.run_daemon_task(self.consume_datagrams)
self.manager.run_task(self.bootstrap)
async def periodically_refresh(self) -> None:
async for _ in trio_utils.every(self._refresh_interval):
await self.lookup_random()
async def fetch_enrs(self) -> None:
async with self.pending_enrs_consumer:
async for (remote, enr_seq) in self.pending_enrs_consumer:
self.logger.debug2("Received request to fetch ENR for %s",
remote)
self.manager.run_task(self._ensure_enr, remote, enr_seq)
async def _ensure_enr(self, node: NodeAPI, enr_seq: int) -> None:
# TODO: Check that we've recently bonded with the remote. For now it shouldn't be a
# problem as this is only triggered once we successfully bonded with a peer.
async with self._enr_db_lock:
try:
enr = await self._enr_db.get(NodeID(node.id_bytes))
except KeyError:
pass
else:
if enr.sequence_number >= enr_seq:
self.logger.debug2("Already got latest ENR for %s", node)
return
try:
await self.get_enr(node)
except CouldNotRetrieveENR as e:
self.logger.info("Failed to retrieve ENR for %s: %s", node, e)
async def report_stats(self) -> None:
async for _ in trio_utils.every(self._refresh_interval):
self.logger.debug(
"============================= Stats =======================")
full_buckets = [
bucket for bucket in self.routing.buckets if bucket.is_full
]
total_nodes = sum([len(bucket) for bucket in self.routing.buckets])
nodes_in_replacement_cache = sum([
len(bucket.replacement_cache)
for bucket in self.routing.buckets
])
self.logger.debug(
"Routing table has %s nodes in %s buckets (%s of which are full), and %s nodes "
"are in the replacement cache", total_nodes,
len(self.routing.buckets), len(full_buckets),
nodes_in_replacement_cache)
self.logger.debug("ENR DB has a total of %s entries",
len(self._enr_db))
self.logger.debug(
"===========================================================")
def update_routing_table(self, node: NodeAPI) -> None:
"""Update the routing table entry for the given node."""
eviction_candidate = self.routing.add_node(node)
if eviction_candidate:
# This means we couldn't add the node because its bucket is full, so schedule a bond()
# with the least recently seen node on that bucket. If the bonding fails the node will
# be removed from the bucket and a new one will be picked from the bucket's
# replacement cache.
self.logger.debug2(
"Routing table's bucket is full, couldn't add %s. "
"Checking if %s is still responding, will evict if not", node,
eviction_candidate)
self.manager.run_task(self.bond, eviction_candidate)
async def bond(self, node: NodeAPI) -> bool:
"""Bond with the given node.
Bonding consists of pinging the node, waiting for a pong and maybe a ping as well.
It is necessary to do this at least once before we send find_node requests to a node.
"""
if node in self.routing:
return True
elif node == self.this_node:
return False
token = await self.send_ping_v4(node)
send_chan, recv_chan = trio.open_memory_channel[Tuple[Hash32, int]](1)
try:
with trio.fail_after(constants.KADEMLIA_REQUEST_TIMEOUT):
received_token, enr_seq = await self.pong_channels.receive_one(
node, send_chan, recv_chan)
except AlreadyWaitingDiscoveryResponse:
self.logger.debug("Bonding failed, already waiting pong from %s",
node)
return False
except trio.TooSlowError:
self.logger.debug("Bonding with %s timed out", node)
return False
if received_token != token:
self.logger.info(
"Bonding with %s failed, expected pong with token %s, but got %s",
node, token, received_token)
self.routing.remove_node(node)
return False
ping_send_chan, ping_recv_chan = trio.open_memory_channel[None](1)
try:
# Give the remote node a chance to ping us before we move on and
# start sending find_node requests. It is ok for us to timeout
# here as that just means the remote remembers us -- that's why we use
# move_on_after() instead of fail_after().
with trio.move_on_after(constants.KADEMLIA_REQUEST_TIMEOUT):
await self.ping_channels.receive_one(node, ping_send_chan,
ping_recv_chan)
except AlreadyWaitingDiscoveryResponse:
self.logger.debug("bonding failed, already waiting for ping")
return False
self.logger.debug("bonding completed successfully with %s", node)
self.update_routing_table(node)
if enr_seq is not None:
self.schedule_enr_retrieval(node, enr_seq)
return True
def schedule_enr_retrieval(self, node: NodeAPI, enr_seq: int) -> None:
self.logger.debug("scheduling ENR retrieval from %s", node)
try:
self.pending_enrs_producer.send_nowait((node, enr_seq))
except trio.WouldBlock:
self.logger.warning(
"Failed to schedule ENR retrieval; channel buffer is full")
async def request_enr(self, remote: NodeAPI) -> ENR:
# No need to use a timeout because bond() takes care of that internally.
await self.bond(remote)
token = self.send_enr_request(remote)
send_chan, recv_chan = trio.open_memory_channel[Tuple[ENR, Hash32]](1)
try:
with trio.fail_after(constants.KADEMLIA_REQUEST_TIMEOUT):
enr, received_token = await self.enr_response_channels.receive_one(
remote, send_chan, recv_chan)
except trio.TooSlowError:
raise CouldNotRetrieveENR(
f"Timed out waiting for ENR from {remote}")
except AlreadyWaitingDiscoveryResponse:
raise CouldNotRetrieveENR(f"Already waiting for ENR from {remote}")
if received_token != token:
raise CouldNotRetrieveENR(
f"Got ENR from {remote} with token {received_token!r} but expected {token!r}"
)
return enr
async def _generate_local_enr(self, sequence_number: int) -> ENR:
kv_pairs = {
IDENTITY_SCHEME_ENR_KEY: V4IdentityScheme.id,
V4IdentityScheme.public_key_enr_key:
self.pubkey.to_compressed_bytes(),
IP_V4_ADDRESS_ENR_KEY: self.address.ip_packed,
UDP_PORT_ENR_KEY: self.address.udp_port,
TCP_PORT_ENR_KEY: self.address.tcp_port,
}
for field_provider in self.enr_field_providers:
key, value = await field_provider()
if key in kv_pairs:
raise AssertionError(
"ENR field provider attempted to override already used key: %s",
key)
kv_pairs[key] = value
unsigned_enr = UnsignedENR(sequence_number, kv_pairs)
return unsigned_enr.to_signed_enr(self.privkey.to_bytes())
async def load_local_enr(self) -> ENR:
"""
Load our own ENR from the database, or create a new one if none exists.
"""
async with self._enr_db_lock:
try:
self._local_enr = await self._enr_db.get(
NodeID(self.this_node.id_bytes))
except KeyError:
self._local_enr = await self._generate_local_enr(
sequence_number=1)
await self._enr_db.insert(self._local_enr)
return self._local_enr
async def get_local_enr(self) -> ENR:
"""
Get our own ENR.
If the cached version of our ENR (self._local_enr) has been updated less than
self._local_enr_next_refresh seconds ago, return it. Otherwise we generate a fresh ENR
(with our current sequence number), compare it to the current one and then either:
1. Return the current one if they're identical
2. Create a new ENR with a new sequence number (current + 1), assign that to
self._local_enr and return it
"""
# Most times self._get_local_enr() will return self._local_enr immediately but if multiple
# coroutines call us and end up trying to update our local ENR we'd have plenty of race
# conditions, so use a lock here.
async with self._local_enr_lock:
return await self._get_local_enr()
async def _get_local_enr(self) -> ENR:
if self._local_enr is None:
raise AssertionError("Local ENR must be loaded on startup")
if self._local_enr_next_refresh > time.monotonic():
return self._local_enr
self._local_enr_next_refresh = time.monotonic(
) + self._local_enr_refresh_interval
# Re-generate our ENR from scratch and compare it to the current one to see if we
# must create a new one with a higher sequence number.
current_enr = await self._generate_local_enr(
self._local_enr.sequence_number)
if current_enr == self._local_enr:
return self._local_enr
# Either our node's details (e.g. IP address) or one of our ENR fields have changed, so
# generate a new one with a higher sequence number.
self._local_enr = await self._generate_local_enr(
self._local_enr.sequence_number + 1)
self.logger.info(
"Node details changed, generated new local ENR with sequence number %d",
self._local_enr.sequence_number)
async with self._enr_db_lock:
await self._enr_db.update(self._local_enr)
return self._local_enr
async def get_local_enr_seq(self) -> int:
enr = await self.get_local_enr()
return enr.sequence_number
async def get_enr(self, remote: NodeAPI) -> ENR:
"""Get the most recent ENR for the given node and update our local DB if necessary.
Raises CouldNotRetrieveENR if we can't get a ENR from the remote node.
"""
enr = await self.request_enr(remote)
self.logger.debug2("Got ENR with seq-id %s for %s",
enr.sequence_number, remote)
async with self._enr_db_lock:
await self._enr_db.insert_or_update(enr)
return await self._enr_db.get(NodeID(remote.id_bytes))
async def wait_neighbours(self, remote: NodeAPI) -> Tuple[NodeAPI, ...]:
"""Wait for a neihgbours packet from the given node.
Returns the list of neighbours received.
"""
neighbours: List[NodeAPI] = []
send_chan, recv_chan = trio.open_memory_channel[List[NodeAPI]](1)
with trio.move_on_after(
constants.KADEMLIA_REQUEST_TIMEOUT) as cancel_scope:
# Responses to a FIND_NODE request are usually split between multiple
# NEIGHBOURS packets, so we may have to read from the channel multiple times.
gen = self.neighbours_channels.receive(remote, send_chan,
recv_chan)
# mypy thinks wrapping our generator turns it into something else, so ignore.
async with aclosing(gen): # type: ignore
async for batch in gen:
self.logger.debug2(
f'got expected neighbours response from {remote}: {batch}'
)
neighbours.extend(batch)
if len(neighbours) >= constants.KADEMLIA_BUCKET_SIZE:
break
self.logger.debug2(
f'got expected neighbours response from {remote}')
if cancel_scope.cancelled_caught:
self.logger.debug2(
f'timed out waiting for {constants.KADEMLIA_BUCKET_SIZE} neighbours from '
f'{remote}, got only {len(neighbours)}')
return tuple(n for n in neighbours if n != self.this_node)
def _send_find_node(self, node: NodeAPI, target_node_id: int) -> None:
self.send_find_node_v4(node, target_node_id)
async def lookup(self, node_id: int) -> Tuple[NodeAPI, ...]:
"""Lookup performs a network search for nodes close to the given target.
It approaches the target by querying nodes that are closer to it on each iteration. The
given target does not need to be an actual node identifier.
"""
nodes_asked: Set[NodeAPI] = set()
nodes_seen: Set[NodeAPI] = set()
async def _find_node(node_id: int,
remote: NodeAPI) -> Tuple[NodeAPI, ...]:
self._send_find_node(remote, node_id)
candidates = await self.wait_neighbours(remote)
if not candidates:
self.logger.debug("got no candidates from %s, returning",
remote)
return tuple()
all_candidates = tuple(c for c in candidates
if c not in nodes_seen)
candidates = tuple(
c for c in all_candidates
if (not self.ping_channels.already_waiting_for(c)
and not self.pong_channels.already_waiting_for(c)))
self.logger.debug2("got %s new candidates", len(candidates))
# Add new candidates to nodes_seen so that we don't attempt to bond with failing ones
# in the future.
nodes_seen.update(candidates)
bonded = await trio_utils.gather(*((self.bond, c)
for c in candidates))
self.logger.debug2("bonded with %s candidates", bonded.count(True))
return tuple(c for c in candidates if bonded[candidates.index(c)])
def _exclude_if_asked(nodes: Iterable[NodeAPI]) -> List[NodeAPI]:
nodes_to_ask = list(set(nodes).difference(nodes_asked))
return sort_by_distance(
nodes_to_ask, node_id)[:constants.KADEMLIA_FIND_CONCURRENCY]
closest = self.routing.neighbours(node_id)
self.logger.debug("starting lookup; initial neighbours: %s", closest)
nodes_to_ask = _exclude_if_asked(closest)
while nodes_to_ask:
self.logger.debug2("node lookup; querying %s", nodes_to_ask)
nodes_asked.update(nodes_to_ask)
next_find_node_queries = (
(_find_node, node_id, n) for n in nodes_to_ask
if not self.neighbours_channels.already_waiting_for(n))
results = await trio_utils.gather(*next_find_node_queries)
for candidates in results:
closest.extend(candidates)
# Need to sort again and pick just the closest k nodes to ensure we converge.
closest = sort_by_distance(
eth_utils.toolz.unique(closest),
node_id)[:constants.KADEMLIA_BUCKET_SIZE]
nodes_to_ask = _exclude_if_asked(closest)
self.logger.debug(
"lookup finished for target %s; closest neighbours: %s",
to_hex(node_id), closest)
return tuple(closest)
async def lookup_random(self) -> Tuple[NodeAPI, ...]:
return await self.lookup(
random.randint(0, constants.KADEMLIA_MAX_NODE_ID))
def get_random_bootnode(self) -> Iterator[NodeAPI]:
if self.bootstrap_nodes:
yield random.choice(self.bootstrap_nodes)
else:
self.logger.warning('No bootnodes available')
def get_nodes_to_connect(self, count: int) -> Iterator[NodeAPI]:
return self.routing.get_random_nodes(count)
@property
def pubkey(self) -> datatypes.PublicKey:
return self.privkey.public_key
def _get_handler(self, cmd: DiscoveryCommand) -> V4_HANDLER_TYPE:
if cmd == CMD_PING:
return self.recv_ping_v4
elif cmd == CMD_PONG:
return self.recv_pong_v4
elif cmd == CMD_FIND_NODE:
return self.recv_find_node_v4
elif cmd == CMD_NEIGHBOURS:
return self.recv_neighbours_v4
elif cmd == CMD_ENR_REQUEST:
return self.recv_enr_request
elif cmd == CMD_ENR_RESPONSE:
return self.recv_enr_response
else:
raise ValueError(f"Unknown command: {cmd}")
@classmethod
def _get_max_neighbours_per_packet(cls) -> int:
if cls._max_neighbours_per_packet_cache is not None:
return cls._max_neighbours_per_packet_cache
cls._max_neighbours_per_packet_cache = _get_max_neighbours_per_packet()
return cls._max_neighbours_per_packet_cache
async def bootstrap(self) -> None:
for node in self.bootstrap_nodes:
uri = node.uri()
pubkey, _, uri_tail = uri.partition('@')
pubkey_head = pubkey[:16]
pubkey_tail = pubkey[-8:]
self.logger.debug("full-bootnode: %s", uri)
self.logger.debug("bootnode: %s...%s@%s", pubkey_head, pubkey_tail,
uri_tail)
bonding_queries = (
(self.bond, n) for n in self.bootstrap_nodes
if (not self.ping_channels.already_waiting_for(n)
and not self.pong_channels.already_waiting_for(n)))
bonded = await trio_utils.gather(*bonding_queries)
if not any(bonded):
self.logger.info("Failed to bond with bootstrap nodes %s",
self.bootstrap_nodes)
return
await self.lookup_random()
def send(self, node: NodeAPI, msg_type: DiscoveryCommand,
payload: Sequence[Any]) -> bytes:
message = _pack_v4(msg_type.id, payload, self.privkey)
self.manager.run_task(self.socket.sendto, message,
(node.address.ip, node.address.udp_port))
return message
async def consume_datagram(self) -> None:
datagram, (ip_address, port) = await self.socket.recvfrom(
constants.DISCOVERY_DATAGRAM_BUFFER_SIZE)
address = Address(ip_address, port)
self.logger.debug2("Received datagram from %s", address)
self.manager.run_task(self.receive, address, datagram)
async def receive(self, address: AddressAPI, message: bytes) -> None:
try:
remote_pubkey, cmd_id, payload, message_hash = _unpack_v4(message)
except DefectiveMessage as e:
self.logger.error('error unpacking message (%s) from %s: %s',
message, address, e)
return
try:
cmd = CMD_ID_MAP[cmd_id]
except KeyError:
self.logger.warning("Ignoring uknown msg type: %s; payload=%s",
cmd_id, payload)
return
self.logger.debug2("Received %s with payload: %s", cmd.name, payload)
node = Node(remote_pubkey, address)
handler = self._get_handler(cmd)
await handler(node, payload, message_hash)
def _is_msg_expired(self, rlp_expiration: bytes) -> bool:
expiration = rlp.sedes.big_endian_int.deserialize(rlp_expiration)
if time.time() > expiration:
self.logger.debug('Received message already expired')
return True
return False
async def recv_pong_v4(self, node: NodeAPI, payload: Sequence[Any],
_: Hash32) -> None:
# The pong payload should have at least 3 elements: to, token, expiration
if len(payload) < 3:
self.logger.warning('Ignoring PONG msg with invalid payload: %s',
payload)
return
elif len(payload) == 3:
_, token, expiration = payload[:3]
enr_seq = None
else:
_, token, expiration, enr_seq = payload[:4]
enr_seq = big_endian_to_int(enr_seq)
if self._is_msg_expired(expiration):
return
self.logger.debug2('<<< pong (v4) from %s (token == %s)', node,
encode_hex(token))
await self.process_pong_v4(node, token, enr_seq)
async def recv_neighbours_v4(self, node: NodeAPI, payload: Sequence[Any],
_: Hash32) -> None:
# The neighbours payload should have 2 elements: nodes, expiration
if len(payload) < 2:
self.logger.warning(
'Ignoring NEIGHBOURS msg with invalid payload: %s', payload)
return
nodes, expiration = payload[:2]
if self._is_msg_expired(expiration):
return
neighbours = _extract_nodes_from_payload(node.address, nodes,
self.logger)
self.logger.debug2('<<< neighbours from %s: %s', node, neighbours)
try:
channel = self.neighbours_channels.get_channel(node)
except KeyError:
self.logger.debug(
f'unexpected neighbours from {node}, probably came too late')
return
try:
await channel.send(neighbours)
except trio.BrokenResourceError:
# This means the receiver has already closed, probably because it timed out.
pass
async def recv_ping_v4(self, node: NodeAPI, payload: Sequence[Any],
message_hash: Hash32) -> None:
# The ping payload should have at least 4 elements: [version, from, to, expiration], with
# an optional 5th element for the node's ENR sequence number.
if len(payload) < 4:
self.logger.warning('Ignoring PING msg with invalid payload: %s',
payload)
return
elif len(payload) == 4:
_, _, _, expiration = payload[:4]
enr_seq = None
else:
_, _, _, expiration, enr_seq = payload[:5]
enr_seq = big_endian_to_int(enr_seq)
self.logger.debug2('<<< ping(v4) from %s, enr_seq=%s', node, enr_seq)
if self._is_msg_expired(expiration):
return
await self.process_ping(node, message_hash)
await self.send_pong_v4(node, message_hash)
async def recv_find_node_v4(self, node: NodeAPI, payload: Sequence[Any],
_: Hash32) -> None:
# The find_node payload should have 2 elements: node_id, expiration
if len(payload) < 2:
self.logger.warning(
'Ignoring FIND_NODE msg with invalid payload: %s', payload)
return
node_id, expiration = payload[:2]
self.logger.debug2('<<< find_node from %s', node)
if self._is_msg_expired(expiration):
return
if node not in self.routing:
# FIXME: This is not correct; a node we've bonded before may have become unavailable
# and thus removed from self.routing, but once it's back online we should accept
# find_nodes from them.
self.logger.debug(
'Ignoring find_node request from unknown node %s', node)
return
self.update_routing_table(node)
found = self.routing.neighbours(big_endian_to_int(node_id))
self.send_neighbours_v4(node, found)
async def recv_enr_request(self, node: NodeAPI, payload: Sequence[Any],
msg_hash: Hash32) -> None:
# The enr_request payload should have at least one element: expiration.
if len(payload) < 1:
self.logger.warning(
'Ignoring ENR_REQUEST msg with invalid payload: %s', payload)
return
expiration = payload[0]
if self._is_msg_expired(expiration):
return
# XXX: Maybe reconsider this and accept all ENR requests until we have a persistent
# routing store of nodes we've bonded with? Otherwise if a node request our ENR across a
# restart, we'll not reply to them.
if node not in self.routing:
self.logger.info('Ignoring ENR_REQUEST from unknown node %s', node)
return
enr = await self.get_local_enr()
self.logger.debug("Sending local ENR to %s: %s", node, enr)
payload = (msg_hash, ENR.serialize(enr))
self.send(node, CMD_ENR_RESPONSE, payload)
async def recv_enr_response(self, node: NodeAPI, payload: Sequence[Any],
msg_hash: Hash32) -> None:
# The enr_response payload should have at least two elements: request_hash, enr.
if len(payload) < 2:
self.logger.warning(
'Ignoring ENR_RESPONSE msg with invalid payload: %s', payload)
return
token, serialized_enr = payload[:2]
try:
enr = ENR.deserialize(serialized_enr)
except DeserializationError as error:
raise ValidationError(
"ENR in response is not properly encoded") from error
try:
channel = self.enr_response_channels.get_channel(node)
except KeyError:
self.logger.debug("Unexpected ENR_RESPONSE from %s", node)
return
enr.validate_signature()
self.logger.debug(
"Received ENR %s (%s) with expected response token: %s", enr,
enr.items(), encode_hex(token))
try:
await channel.send((enr, token))
except trio.BrokenResourceError:
# This means the receiver has already closed, probably because it timed out.
pass
def send_enr_request(self, node: NodeAPI) -> Hash32:
message = self.send(node, CMD_ENR_REQUEST, [_get_msg_expiration()])
token = Hash32(message[:MAC_SIZE])
self.logger.debug("Sending ENR request with token: %s",
encode_hex(token))
return token
async def send_ping_v4(self, node: NodeAPI) -> Hash32:
version = rlp.sedes.big_endian_int.serialize(PROTO_VERSION)
expiration = _get_msg_expiration()
local_enr_seq = await self.get_local_enr_seq()
payload = (version, self.address.to_endpoint(),
node.address.to_endpoint(), expiration,
int_to_big_endian(local_enr_seq))
message = self.send(node, CMD_PING, payload)
# Return the msg hash, which is used as a token to identify pongs.
token = Hash32(message[:MAC_SIZE])
self.logger.debug2('>>> ping (v4) %s (token == %s)', node,
encode_hex(token))
# XXX: This hack is needed because there are lots of parity 1.10 nodes out there that send
# the wrong token on pong msgs (https://github.com/paritytech/parity/issues/8038). We
# should get rid of this once there are no longer too many parity 1.10 nodes out there.
parity_token = keccak(message[HEAD_SIZE + 1:])
self.parity_pong_tokens[parity_token] = token
return token
def send_find_node_v4(self, node: NodeAPI, target_node_id: int) -> None:
expiration = _get_msg_expiration()
node_id = int_to_big_endian(target_node_id).rjust(
constants.KADEMLIA_PUBLIC_KEY_SIZE // 8, b'\0')
self.logger.debug2('>>> find_node to %s', node)
self.send(node, CMD_FIND_NODE, (node_id, expiration))
async def send_pong_v4(self, node: NodeAPI, token: Hash32) -> None:
expiration = _get_msg_expiration()
self.logger.debug2('>>> pong %s', node)
local_enr_seq = await self.get_local_enr_seq()
payload = (node.address.to_endpoint(), token, expiration,
int_to_big_endian(local_enr_seq))
self.send(node, CMD_PONG, payload)
def send_neighbours_v4(self, node: NodeAPI,
neighbours: List[NodeAPI]) -> None:
nodes = []
neighbours = sorted(neighbours)
for n in neighbours:
nodes.append(n.address.to_endpoint() + [n.pubkey.to_bytes()])
expiration = _get_msg_expiration()
max_neighbours = self._get_max_neighbours_per_packet()
for i in range(0, len(nodes), max_neighbours):
self.logger.debug2('>>> neighbours to %s: %s', node,
neighbours[i:i + max_neighbours])
payload = NeighboursPacket(neighbours=nodes[i:i + max_neighbours],
expiration=expiration)
self.send(node, CMD_NEIGHBOURS, payload)
async def process_pong_v4(self, remote: NodeAPI, token: Hash32,
enr_seq: int) -> None:
# XXX: This hack is needed because there are lots of parity 1.10 nodes out there that send
# the wrong token on pong msgs (https://github.com/paritytech/parity/issues/8038). We
# should get rid of this once there are no longer too many parity 1.10 nodes out there.
if token in self.parity_pong_tokens:
# This is a pong from a buggy parity node, so need to lookup the actual token we're
# expecting.
token = self.parity_pong_tokens.pop(token)
else:
# This is a pong from a non-buggy node, so just cleanup self.parity_pong_tokens.
self.parity_pong_tokens = eth_utils.toolz.valfilter(
lambda val: val != token, self.parity_pong_tokens)
try:
channel = self.pong_channels.get_channel(remote)
except KeyError:
# This is probably a Node which changed its identity since it was added to the DHT,
# causing us to expect a pong signed with a certain key when in fact it's using
# a different one. Another possibility is that the pong came after we've given up
# waiting.
self.logger.debug(
f'Unexpected pong from {remote} with token {encode_hex(token)}'
)
return
try:
await channel.send((token, enr_seq))
except trio.BrokenResourceError:
# This means the receiver has already closed, probably because it timed out.
pass
async def process_ping(self, remote: NodeAPI, hash_: Hash32) -> None:
"""Process a received ping packet.
A ping packet may come any time, unrequested, or may be prompted by us bond()ing with a
new node. In the former case we'll just update the sender's entry in our routing table and
reply with a pong, whereas in the latter we'll also send an empty msg on the appropriate
channel from ping_channels, to notify any coroutine waiting for a ping.
"""
if remote == self.this_node:
self.logger.info('Invariant: received ping from this_node: %s',
remote)
return
else:
self.update_routing_table(remote)
try:
channel = self.ping_channels.get_channel(remote)
except KeyError:
return
try:
await channel.send(None)
except trio.BrokenResourceError:
# This means the receiver has already closed, probably because it timed out.
pass