def __init__(self, args: argparse.Namespace, boot_info: BootInfo) -> None:
super().__init__(args, boot_info)
self.concurrency = args.crawl_concurrency
self.enr_db = ENRDB(dict(), default_identity_scheme_registry)
self.sock = trio.socket.socket(
family=trio.socket.AF_INET, type=trio.socket.SOCK_DGRAM
)
self.private_key = get_local_private_key(boot_info)
self.enr_manager = ENRManager(private_key=self.private_key, enr_db=self.enr_db)
self.enr_manager.update((b"udp", boot_info.port))
my_node_id = self.enr_manager.enr.node_id
self.client = Client(self.private_key, self.enr_db, my_node_id, self.sock)
self.active_tasks = ActiveTaskCounter()
self.seen_nodeids: Set[NodeID] = set()
self.responsive_peers: Set[NodeID] = set()
self.enrqueue_send, self.enrqueue_recv = trio.open_memory_channel[ENRAPI](
10_000
)
async def _update_enr_ip_from_upnp(self, enr_manager: ENRManager,
upnp_service: UPnPService) -> None:
await upnp_service.ready()
with trio.move_on_after(10):
_, external_ip = await upnp_service.get_ip_addresses()
enr_manager.update((IP_V4_ADDRESS_ENR_KEY, external_ip.packed))
while self.manager.is_running:
_, external_ip = await upnp_service.wait_ip_changed()
enr_manager.update((IP_V4_ADDRESS_ENR_KEY, external_ip.packed))
def new_enr_manager():
enr_db = QueryableENRDB(sqlite3.connect(":memory:"))
private_key = PrivateKeyFactory()
base_enr = ENRFactory(
private_key=private_key.to_bytes(),
sequence_number=secrets.randbelow(100) + 1,
)
enr_db.set_enr(base_enr)
return ENRManager(private_key, enr_db)
async def test_network_lookup_many_enr_response(bob, alice_network, alice,
bob_client):
enr_manager = ENRManager(
private_key=bob.private_key,
enr_db=bob.enr_db,
)
enr_manager.update(
(b"udp", bob.endpoint.port),
(b"ip", bob.endpoint.ip_address),
)
first_enr = enr_manager.enr
enr_manager.update((b"eth2", b"\x00\x00"))
second_enr = enr_manager.enr
# quick sanity check, if these are equal the next assertion would be testing nothing
assert first_enr != second_enr
async def return_duplicate_enr_response():
async with bob.events.find_nodes_received.subscribe() as subscription:
find_nodes = await subscription.receive()
await bob_client.send_found_nodes(
alice.node_id,
alice.endpoint,
enrs=[second_enr, first_enr],
request_id=find_nodes.message.request_id,
)
with trio.fail_after(2):
async with trio.open_nursery() as nursery:
nursery.start_soon(return_duplicate_enr_response)
enr = await alice_network.lookup_enr(bob.node_id, enr_seq=101)
assert enr == second_enr # the one with the highest sequence number
async def run(self) -> None:
self.manager.run_daemon_child_service(Canary())
identity_scheme_registry = default_identity_scheme_registry
message_type_registry = v51_registry
enr_database_file = self._boot_info.base_dir / ENR_DATABASE_FILENAME
enr_db: QueryableENRDatabaseAPI = QueryableENRDB(
sqlite3.connect(enr_database_file), identity_scheme_registry)
local_private_key = get_local_private_key(self._boot_info)
enr_manager = ENRManager(
enr_db=enr_db,
private_key=local_private_key,
)
port = self._boot_info.port
events = Events()
if b"udp" not in enr_manager.enr:
enr_manager.update((b"udp", port))
elif enr_manager.enr[b"udp"] != port:
enr_manager.update((b"udp", port))
listen_on_ip_address: AnyIPAddress
if self._boot_info.listen_on is None:
listen_on_ip_address = DEFAULT_LISTEN
else:
listen_on_ip_address = self._boot_info.listen_on
# Update the ENR if an explicit listening address was provided
enr_manager.update(
(IP_V4_ADDRESS_ENR_KEY, listen_on_ip_address.packed))
listen_on = Endpoint(listen_on_ip_address.packed, self._boot_info.port)
if self._boot_info.is_upnp_enabled:
upnp_service = UPnPService(port)
self.manager.run_daemon_child_service(upnp_service)
self.manager.run_daemon_task(self._update_enr_ip_from_upnp,
enr_manager, upnp_service)
bootnodes = self._boot_info.bootnodes
self.client = Client(
local_private_key=local_private_key,
listen_on=listen_on,
enr_db=enr_db,
session_cache_size=self._args.session_cache_size,
events=events,
message_type_registry=message_type_registry,
)
self.network = Network(
client=self.client,
bootnodes=bootnodes,
)
if self._boot_info.is_rpc_enabled:
handlers = merge(
get_core_rpc_handlers(enr_manager, self.network.routing_table),
get_v51_rpc_handlers(self.network),
)
self.rpc_server = RPCServer(self._boot_info.ipc_path, handlers)
self.manager.run_daemon_child_service(self.rpc_server)
self.logger.info("Starting DDHT...")
self.logger.info("Protocol-Version: %s",
self._boot_info.protocol_version.value)
self.logger.info("DDHT base dir : %s", self._boot_info.base_dir)
self.logger.info("Listening on : %s", listen_on)
self.logger.info("Local Node ID : %s",
encode_hex(enr_manager.enr.node_id))
self.logger.info(
"Local ENR : seq=%d enr=%s",
enr_manager.enr.sequence_number,
enr_manager.enr,
)
self.manager.run_daemon_child_service(self.network)
self._ready.set()
await self.manager.wait_finished()
def __init__(
self,
local_private_key: keys.PrivateKey,
listen_on: Endpoint,
enr_db: ENRDatabaseAPI,
events: EventsAPI = None,
message_type_registry: MessageTypeRegistry = v51_registry,
) -> None:
self._local_private_key = local_private_key
self.listen_on = listen_on
self._listening = trio.Event()
self.enr_manager = ENRManager(
private_key=local_private_key,
enr_db=enr_db,
)
self.enr_db = enr_db
self._registry = message_type_registry
# Datagrams
(
self._outbound_datagram_send_channel,
self._outbound_datagram_receive_channel,
) = trio.open_memory_channel[OutboundDatagram](256)
(
self._inbound_datagram_send_channel,
self._inbound_datagram_receive_channel,
) = trio.open_memory_channel[InboundDatagram](256)
# EnvelopePair
(
self._outbound_envelope_send_channel,
self._outbound_envelope_receive_channel,
) = trio.open_memory_channel[OutboundEnvelope](256)
(
self._inbound_envelope_send_channel,
self._inbound_envelope_receive_channel,
) = trio.open_memory_channel[InboundEnvelope](256)
# Messages
(
self._outbound_message_send_channel,
self._outbound_message_receive_channel,
) = trio.open_memory_channel[AnyOutboundMessage](256)
(
self._inbound_message_send_channel,
self._inbound_message_receive_channel,
) = trio.open_memory_channel[AnyInboundMessage](256)
if events is None:
events = Events()
self.events = events
self.pool = Pool(
local_private_key=self._local_private_key,
local_node_id=self.enr_manager.enr.node_id,
enr_db=self.enr_db,
outbound_envelope_send_channel=self.
_outbound_envelope_send_channel,
inbound_message_send_channel=self._inbound_message_send_channel,
message_type_registry=self._registry,
events=self.events,
)
self.dispatcher = Dispatcher(
self._inbound_envelope_receive_channel,
self._inbound_message_receive_channel,
self.pool,
self.enr_db,
self._registry,
self.events,
)
self.envelope_decoder = EnvelopeEncoder(
self._outbound_envelope_receive_channel,
self._outbound_datagram_send_channel,
)
self.envelope_encoder = EnvelopeDecoder(
self._inbound_datagram_receive_channel,
self._inbound_envelope_send_channel,
self.enr_manager.enr.node_id,
)
self._ready = trio.Event()
async def run(self) -> None:
identity_scheme_registry = default_identity_scheme_registry
message_type_registry = v51_registry
enr_database_dir = self._boot_info.base_dir / ENR_DATABASE_DIR_NAME
enr_database_dir.mkdir(exist_ok=True)
enr_db = ENRDB(LevelDB(enr_database_dir), identity_scheme_registry)
local_private_key = get_local_private_key(self._boot_info)
enr_manager = ENRManager(
enr_db=enr_db,
private_key=local_private_key,
)
port = self._boot_info.port
events = Events()
if b"udp" not in enr_manager.enr:
enr_manager.update((b"udp", port))
listen_on_ip_address: AnyIPAddress
if self._boot_info.listen_on is None:
listen_on_ip_address = DEFAULT_LISTEN
else:
listen_on_ip_address = self._boot_info.listen_on
# Update the ENR if an explicit listening address was provided
enr_manager.update(
(IP_V4_ADDRESS_ENR_KEY, listen_on_ip_address.packed))
listen_on = Endpoint(listen_on_ip_address.packed, self._boot_info.port)
if self._boot_info.is_upnp_enabled:
upnp_service = UPnPService(port)
self.manager.run_daemon_child_service(upnp_service)
self.manager.run_daemon_task(self._update_enr_ip_from_upnp,
enr_manager, upnp_service)
bootnodes = self._boot_info.bootnodes
client = Client(
local_private_key=local_private_key,
listen_on=listen_on,
enr_db=enr_db,
events=events,
message_type_registry=message_type_registry,
)
network = Network(
client=client,
bootnodes=bootnodes,
)
logger.info("Protocol-Version: %s",
self._boot_info.protocol_version.value)
logger.info("DDHT base dir: %s", self._boot_info.base_dir)
logger.info("Starting discovery service...")
logger.info("Listening on %s:%d", listen_on, port)
logger.info("Local Node ID: %s", encode_hex(enr_manager.enr.node_id))
logger.info("Local ENR: seq=%d enr=%s",
enr_manager.enr.sequence_number, enr_manager.enr)
await run_trio_service(network)
async def run(self) -> None:
identity_scheme_registry = default_identity_scheme_registry
message_type_registry = v5_registry
enr_database_dir = self._boot_info.base_dir / ENR_DATABASE_DIR_NAME
enr_database_dir.mkdir(exist_ok=True)
enr_db = ENRDB(LevelDB(enr_database_dir), identity_scheme_registry)
self.enr_db = enr_db
local_private_key = get_local_private_key(self._boot_info)
enr_manager = ENRManager(
private_key=local_private_key,
enr_db=enr_db,
)
port = self._boot_info.port
if b"udp" not in enr_manager.enr:
enr_manager.update((b"udp", port))
listen_on: AnyIPAddress
if self._boot_info.listen_on is None:
listen_on = DEFAULT_LISTEN
else:
listen_on = self._boot_info.listen_on
# Update the ENR if an explicit listening address was provided
enr_manager.update((IP_V4_ADDRESS_ENR_KEY, listen_on.packed))
if self._boot_info.is_upnp_enabled:
upnp_service = UPnPService(port)
self.manager.run_daemon_child_service(upnp_service)
routing_table = KademliaRoutingTable(enr_manager.enr.node_id,
ROUTING_TABLE_BUCKET_SIZE)
self.routing_table = routing_table
for enr in self._boot_info.bootnodes:
try:
enr_db.set_enr(enr)
except OldSequenceNumber:
pass
routing_table.update(enr.node_id)
sock = trio.socket.socket(family=trio.socket.AF_INET,
type=trio.socket.SOCK_DGRAM)
outbound_datagram_channels = trio.open_memory_channel[
OutboundDatagram](0)
inbound_datagram_channels = trio.open_memory_channel[InboundDatagram](
0)
outbound_packet_channels = trio.open_memory_channel[OutboundPacket](0)
inbound_packet_channels = trio.open_memory_channel[InboundPacket](0)
outbound_message_channels = trio.open_memory_channel[
AnyOutboundMessage](0)
inbound_message_channels = trio.open_memory_channel[AnyInboundMessage](
0)
endpoint_vote_channels = trio.open_memory_channel[EndpointVote](0)
# types ignored due to https://github.com/ethereum/async-service/issues/5
datagram_sender = DatagramSender( # type: ignore
outbound_datagram_channels[1], sock)
datagram_receiver = DatagramReceiver( # type: ignore
sock, inbound_datagram_channels[0])
packet_encoder = PacketEncoder( # type: ignore
outbound_packet_channels[1], outbound_datagram_channels[0])
packet_decoder = PacketDecoder( # type: ignore
inbound_datagram_channels[1], inbound_packet_channels[0])
packer = Packer(
local_private_key=local_private_key.to_bytes(),
local_node_id=enr_manager.enr.node_id,
enr_db=enr_db,
message_type_registry=message_type_registry,
inbound_packet_receive_channel=inbound_packet_channels[1],
inbound_message_send_channel=inbound_message_channels[0],
outbound_message_receive_channel=outbound_message_channels[1],
outbound_packet_send_channel=outbound_packet_channels[0],
)
message_dispatcher = MessageDispatcher(
enr_db=enr_db,
inbound_message_receive_channel=inbound_message_channels[1],
outbound_message_send_channel=outbound_message_channels[0],
)
endpoint_tracker = EndpointTracker(
local_private_key=local_private_key.to_bytes(),
local_node_id=enr_manager.enr.node_id,
enr_db=enr_db,
identity_scheme_registry=identity_scheme_registry,
vote_receive_channel=endpoint_vote_channels[1],
)
routing_table_manager = RoutingTableManager(
local_node_id=enr_manager.enr.node_id,
routing_table=routing_table,
message_dispatcher=message_dispatcher,
enr_db=enr_db,
outbound_message_send_channel=outbound_message_channels[0],
endpoint_vote_send_channel=endpoint_vote_channels[0],
)
self.logger.info(f"DDHT base dir: {self._boot_info.base_dir}")
self.logger.info("Starting discovery service...")
self.logger.info(f"Listening on {listen_on}:{port}")
self.logger.info(
f"Local Node ID: {encode_hex(enr_manager.enr.node_id)}")
self.logger.info(f"Local ENR: {enr_manager.enr}")
services = (
datagram_sender,
datagram_receiver,
packet_encoder,
packet_decoder,
packer,
message_dispatcher,
endpoint_tracker,
routing_table_manager,
)
await sock.bind((str(listen_on), port))
with sock:
for service in services:
self.manager.run_daemon_child_service(service)
await self.manager.wait_finished()
async def run(self) -> None:
identity_scheme_registry = default_identity_scheme_registry
enr_database_file = self._boot_info.base_dir / ENR_DATABASE_FILENAME
enr_db = QueryableENRDB(sqlite3.connect(enr_database_file),
identity_scheme_registry)
self.enr_db = enr_db
local_private_key = get_local_private_key(self._boot_info)
enr_manager = ENRManager(
private_key=local_private_key,
enr_db=enr_db,
)
port = self._boot_info.port
if b"udp" not in enr_manager.enr:
enr_manager.update((b"udp", port))
listen_on: AnyIPAddress
if self._boot_info.listen_on is None:
listen_on = DEFAULT_LISTEN
else:
listen_on = self._boot_info.listen_on
# Update the ENR if an explicit listening address was provided
enr_manager.update((IP_V4_ADDRESS_ENR_KEY, listen_on.packed))
if self._boot_info.is_upnp_enabled:
upnp_service = UPnPService(port)
self.manager.run_daemon_child_service(upnp_service)
routing_table = KademliaRoutingTable(enr_manager.enr.node_id,
ROUTING_TABLE_BUCKET_SIZE)
self.routing_table = routing_table
for enr in self._boot_info.bootnodes:
try:
enr_db.set_enr(enr)
except OldSequenceNumber:
pass
routing_table.update(enr.node_id)
sock = trio.socket.socket(family=trio.socket.AF_INET,
type=trio.socket.SOCK_DGRAM)
client = Client(local_private_key, enr_db, enr_manager.enr.node_id,
sock)
endpoint_vote_channels = trio.open_memory_channel[EndpointVote](0)
endpoint_tracker = EndpointTracker(
local_private_key=local_private_key.to_bytes(),
local_node_id=enr_manager.enr.node_id,
enr_db=enr_db,
identity_scheme_registry=identity_scheme_registry,
vote_receive_channel=endpoint_vote_channels[1],
)
routing_table_manager = RoutingTableManager(
local_node_id=enr_manager.enr.node_id,
routing_table=routing_table,
message_dispatcher=client.message_dispatcher,
enr_db=enr_db,
outbound_message_send_channel=client.outbound_message_send_channel,
endpoint_vote_send_channel=endpoint_vote_channels[0],
)
self.logger.info(f"DDHT base dir: {self._boot_info.base_dir}")
self.logger.info("Starting discovery service...")
self.logger.info(f"Listening on {listen_on}:{port}")
self.logger.info(
f"Local Node ID: {encode_hex(enr_manager.enr.node_id)}")
self.logger.info(f"Local ENR: {enr_manager.enr}")
services = (
client,
endpoint_tracker,
routing_table_manager,
)
await sock.bind((str(listen_on), port))
with sock:
for service in services:
self.manager.run_daemon_child_service(service)
await self.manager.wait_finished()
class Crawler(BaseApplication):
def __init__(self, args: argparse.Namespace, boot_info: BootInfo) -> None:
super().__init__(args, boot_info)
self.concurrency = args.crawl_concurrency
self.enr_db = ENRDB(dict(), default_identity_scheme_registry)
self.sock = trio.socket.socket(
family=trio.socket.AF_INET, type=trio.socket.SOCK_DGRAM
)
self.private_key = get_local_private_key(boot_info)
self.enr_manager = ENRManager(private_key=self.private_key, enr_db=self.enr_db)
self.enr_manager.update((b"udp", boot_info.port))
my_node_id = self.enr_manager.enr.node_id
self.client = Client(self.private_key, self.enr_db, my_node_id, self.sock)
self.active_tasks = ActiveTaskCounter()
self.seen_nodeids: Set[NodeID] = set()
self.responsive_peers: Set[NodeID] = set()
self.enrqueue_send, self.enrqueue_recv = trio.open_memory_channel[ENRAPI](
10_000
)
async def fetch_enr_bucket(
self, remote_enr: ENRAPI, bucket: int
) -> Tuple[ENRAPI, ...]:
peer_id = remote_enr.node_id
logger.debug(f"sending FindNode. nodeid={encode_hex(peer_id)} bucket={bucket}")
try:
with trio.fail_after(2):
find_node = FindNodeMessage(
request_id=self.client.message_dispatcher.get_free_request_id(
peer_id
),
distance=bucket,
)
responses = await self.client.message_dispatcher.request_nodes(
peer_id, find_node
)
return tuple(
enr for response in responses for enr in response.message.enrs
)
except trio.TooSlowError:
logger.info(
f"no response from peer. nodeid={encode_hex(peer_id)} bucket={bucket}"
)
raise _StopScanning()
except UnexpectedMessage:
logger.exception(
"received a bad message from the peer. " f"nodeid={encode_hex(peer_id)}"
)
raise _StopScanning()
async def scan_enr_buckets(self, remote_enr: ENRAPI) -> None:
"""
Attempts to read ENRs from the outermost few buckets of the remote node.
"""
peerid = remote_enr.node_id
logger.debug(f"scanning. nodeid={encode_hex(peerid)}")
SCAN_DEPTH = 100
consecutive_empty_buckets = 0
try:
for bucket in range(256, 256 - SCAN_DEPTH, -1):
enrs = await self.fetch_enr_bucket(remote_enr, bucket)
novel_enrs = len(
[enr for enr in enrs if enr.node_id not in self.seen_nodeids]
)
logger.info(
f"received ENRs. nodeid={encode_hex(peerid)} bucket={bucket} "
f"enrs={len(enrs)} novel={novel_enrs}"
)
if len(enrs) == 0:
consecutive_empty_buckets += 1
else:
self.responsive_peers.add(peerid)
consecutive_empty_buckets = 0
# as we get deeper into the peer's routing table the chance that there are
# any peers in the given bucket decreases exponentially. we save some time
# by moving on to the next peer once we notice that the buckets have
# started to think out.
if consecutive_empty_buckets >= 5:
return
for enr in enrs:
await self.schedule_enr_to_be_visited(enr)
await trio.sleep(0.1) # don't overburden this peer
except _StopScanning:
pass
finally:
# once this coro finishes we will never talk to this peer again, do some
# cleanup now or we'll leak memory.
self.client.discard_peer(remote_enr.node_id)
async def read_from_queue_until_done(self) -> None:
while True:
# CAUTION: Do not insert any code here. There must not be any checkpoints
# between leaving active_tasks (in the previous loop iteration) and
# trying to read from the queue.
try:
enr = self.enrqueue_recv.receive_nowait()
except trio.WouldBlock:
if len(self.active_tasks) > 0:
# some tasks are still active so it's too soon to quit. Instead, block
# until new work comes in. If we've truly run out of work then this
# will block forever but that's okay, another task will notice and
# cause our cancellation.
enr = await self.enrqueue_recv.receive()
else:
# nobody else is performing any work and there's also no work left to
# be done. It's time to quit! Calling this causes all other tasks to
# be canceled.
await self.manager.stop()
return
# CAUTION: Do not insert any code here. There must not be any checkpoints
# between popping from the queue and entering active_tasks.
with self.active_tasks.enter():
await self.scan_enr_buckets(enr)
async def schedule_enr_to_be_visited(self, enr: ENRAPI) -> None:
if enr.node_id in self.seen_nodeids:
# since each trip to a peer dumps their entire routing table there's no need
# to visit a peer twice. This assumes that nodeids don't move around between
# servers which is wrong but likely close enough.
return
if IP_V4_ADDRESS_ENR_KEY not in enr:
logger.info(f"dropping ENR without IP address enr={enr} kv={enr.items()}")
return
self.seen_nodeids.add(enr.node_id)
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
logger.info(f"dropping old ENR. enr={enr} kv={enr.items()}")
return
enqueued = self.enrqueue_send.statistics().current_buffer_used
logger.info(
f"found ENR. count={len(self.seen_nodeids)} enr={enr} node_id={enr.node_id.hex()}"
f" sequence_number={enr.sequence_number} kv={enr.items()} queue_len={enqueued}"
)
await self.enrqueue_send.send(enr)
async def run(self) -> None:
logger.info("crawling!")
boot_info = self._boot_info
if boot_info.is_upnp_enabled:
logger.info(
"uPNP will not be used; crawling does not require listening for "
"incoming connections."
)
for bootnode in boot_info.bootnodes:
await self.schedule_enr_to_be_visited(bootnode)
listen_on = boot_info.listen_on or DEFAULT_LISTEN
logger.info(f"about to bind to port. bind={listen_on}:{boot_info.port}")
await self.sock.bind((str(listen_on), boot_info.port))
try:
with self.sock:
self.manager.run_daemon_child_service(self.client)
for _ in range(self.concurrency):
self.manager.run_daemon_task(self.read_from_queue_until_done)
# When it is time to quit one of the `read_from_queue_until_done` tasks will
# notice and trigger a shutdown.
await self.manager.wait_finished()
finally:
logger.info(
f"scaning finished. found_peers={len(self.seen_nodeids)} "
f"responsive_peers={len(self.responsive_peers)} "
)