def get_unqueried_node_ids() -> Tuple[NodeID, ...]:
"""
Get the three nodes that are closest to the target such that the node
is in the closest `k` nodes which haven't been deemed unresponsive.
"""
# Construct an iterable of *all* the nodes we know about ordered by
# closeness to the target.
candidates = iter_closest_nodes(
target, network.routing_table, received_node_ids
)
# Remove any unresponsive nodes from that iterable
responsive_candidates = itertools.filterfalse(
lambda node_id: node_id in unresponsive_node_ids, candidates
)
# Grab the closest K
closest_k_candidates = take(
network.routing_table.bucket_size, responsive_candidates,
)
# Filter out any from the closest K that we've already queried or that are in-flight
closest_k_unqueried = itertools.filterfalse(
lambda node_id: node_id in queried_node_ids or node_id in in_flight,
closest_k_candidates,
)
return tuple(take(3, closest_k_unqueried))
def decode_packet(
data: bytes,
local_node_id: NodeID,
) -> AnyPacket:
iv = data[:16]
masking_key = cast(AES128Key, local_node_id[:16])
cipher_text_stream = aesctr_decrypt_stream(masking_key, iv, data[16:])
# Decode the header
header_bytes = bytes(take(HEADER_PACKET_SIZE, cipher_text_stream))
header = Header.from_wire_bytes(header_bytes)
auth_data_bytes = bytes(take(header.auth_data_size, cipher_text_stream))
auth_data: Union[MessagePacket, WhoAreYouPacket, HandshakePacket]
if header.flag == 0:
auth_data = MessagePacket.from_wire_bytes(auth_data_bytes)
elif header.flag == 1:
auth_data = WhoAreYouPacket.from_wire_bytes(auth_data_bytes)
elif header.flag == 2:
auth_data = HandshakePacket.from_wire_bytes(auth_data_bytes)
else:
raise DecodingError(f"Unable to decode datagram: {data.hex()}", data)
message_cipher_text = data[16 + HEADER_PACKET_SIZE +
header.auth_data_size:]
return cast(
AnyPacket,
Packet(iv,
header,
auth_data,
message_cipher_text,
dest_node_id=local_node_id),
)
async def lookup(self, target: NodeID) -> None:
self.logger.debug("Looking up %s", encode_hex(target))
queried_node_ids = set()
unresponsive_node_ids = set()
received_enrs: List[ENRAPI] = []
received_node_ids: List[NodeID] = []
async def lookup_and_store_response(peer: NodeID) -> None:
enrs = await self.lookup_at_peer(peer, target)
queried_node_ids.add(peer)
if enrs is not None:
for enr in enrs:
received_node_ids.append(enr.node_id)
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
received_enrs.append(self.enr_db.get_enr(enr.node_id))
else:
received_enrs.append(enr)
else:
unresponsive_node_ids.add(peer)
for lookup_round_counter in itertools.count():
candidates = iter_closest_nodes(
target, self.routing_table, received_node_ids
)
responsive_candidates = itertools.dropwhile(
lambda node: node in unresponsive_node_ids, candidates
)
closest_k_candidates = take(
self.routing_table.bucket_size, responsive_candidates
)
closest_k_unqueried_candidates = (
candidate
for candidate in closest_k_candidates
if candidate not in queried_node_ids
)
nodes_to_query = tuple(
take(LOOKUP_PARALLELIZATION_FACTOR, closest_k_unqueried_candidates)
)
if nodes_to_query:
self.logger.debug(
"Starting lookup round %d for %s",
lookup_round_counter + 1,
encode_hex(target),
)
async with trio.open_nursery() as nursery:
for peer in nodes_to_query:
nursery.start_soon(lookup_and_store_response, peer)
else:
self.logger.debug(
"Lookup for %s finished in %d rounds",
encode_hex(target),
lookup_round_counter,
)
break
async def connect_to_nodes(self, nodes: Iterator[NodeAPI]) -> None:
# create an generator for the nodes
nodes_iter = iter(nodes)
while True:
if self.is_full or not self.is_operational:
return
# only attempt to connect to up to the maximum number of available
# peer slots that are open.
available_peer_slots = self.max_peers - len(self)
batch_size = clamp(1, 10, available_peer_slots)
batch = tuple(take(batch_size, nodes_iter))
# There are no more *known* nodes to connect to.
if not batch:
return
self.logger.debug(
'Initiating %d peer connection attempts with %d open peer slots',
len(batch),
available_peer_slots,
)
# Try to connect to the peers concurrently.
await asyncio.gather(
*(self.connect_to_node(node) for node in batch),
loop=self.get_event_loop(),
)
def _get_blocks(self,
start_block: BaseBeaconBlock,
max_blocks: int) -> Iterable[BaseBeaconBlock]:
if max_blocks < 0:
raise Exception("Invariant: max blocks cannot be negative")
if max_blocks == 0:
return
yield start_block
blocks_generator = cons(start_block, (
self.db.get_canonical_block_by_slot(slot)
for slot in itertools.count(start_block.slot + 1)
))
max_blocks_generator = take(max_blocks, blocks_generator)
try:
# ensure only a connected chain is returned (breaks might occur if the start block is
# not part of the canonical chain or if the canonical chain changes during execution)
for parent, child in sliding_window(2, max_blocks_generator):
if child.parent_root == parent.hash:
yield child
else:
break
except BlockNotFound:
return
async def purge_distant_ads(self) -> None:
if self.max_advertisement_count is None:
raise Exception("Invalid")
while self.manager.is_running:
# Ensure that the inner loop here doesn't block the event loop
await trio.lowlevel.checkpoint()
# Check if we are over the limit
total_advertisements = self.advertisement_db.count()
if total_advertisements <= self.max_advertisement_count:
break
num_to_purge = total_advertisements - self.max_advertisement_count
# Purge in chunks of 64 to avoid blocking too long
to_purge = tuple(
take(
min(64, num_to_purge),
itertools.filterfalse(
lambda ad: ad.node_id == self._network.local_node_id,
self.advertisement_db.furthest(
self._network.local_node_id),
),
))
self.logger.debug("Purging ads outside radius: count=%d",
len(to_purge))
for advertisement in to_purge:
self.advertisement_db.remove(advertisement)
def merkleize(chunks: Sequence[Hash32]) -> Hash32:
padded_chunks = pad_chunks(chunks)
number_of_layers = int(math.log2(len(padded_chunks))) + 1
layers = take(number_of_layers, iterate(hash_layer, padded_chunks))
root, = last(layers)
return root
def _batch_of_missing_hashes(self) -> Tuple[TrackedRequest, ...]:
"""
Take a batch of missing trie hashes, sized for a single peer request
"""
return tuple(take(
REQUEST_SIZE,
self._missing_trie_hashes(),
))
async def new_sync_headers(
self,
max_batch_size: int = None) -> AsyncIterator[Tuple[BlockHeader, ...]]:
while True:
next_batch = tuple(take(max_batch_size, self._headers_to_emit))
if not next_batch:
self._new_data.clear()
await self._new_data.wait()
continue
yield next_batch
self._headers_to_emit = tuple(drop(max_batch_size, self._headers_to_emit))
async def purge_expired_ads(self) -> None:
while self.manager.is_running:
await trio.lowlevel.checkpoint()
# Purge in chunks of 64 to avoid blocking too long
to_purge = tuple(take(64, self.advertisement_db.expired()))
if not to_purge:
break
self.logger.debug("Purging expired ads: count=%d", len(to_purge))
for advertisement in to_purge:
self.advertisement_db.remove(advertisement)
def test_iterating_operation_pool(sample_attestation_params):
some_attestation_params = zip(itertools.count(), forever(sample_attestation_params))
some_attestations = map(lambda x: mk_attestation(*x), some_attestation_params)
attestation_count = 20
attestations = tuple(take(attestation_count, some_attestations))
pool = AttestationPool()
for a in attestations:
pool.add(a)
for _, a in pool:
assert a in attestations
async def make_raw_request(raw_request: str):
with trio.fail_after(2):
data = raw_request.encode("utf8")
data_iter = iter(data)
while True:
chunk = bytes(take(1024, data_iter))
if chunk:
try:
await socket.send_all(chunk)
except trio.BrokenResourceError:
break
else:
break
return await read_json(socket.socket, buffer)
def get_ancestors(self, limit: int, header: BlockHeaderAPI) -> Tuple[BlockAPI, ...]:
ancestor_count = min(header.block_number, limit)
# We construct a temporary block object
vm_class = self.get_vm_class_for_block_number(header.block_number)
block_class = vm_class.get_block_class()
block = block_class(header=header, uncles=[], transactions=[])
ancestor_generator = iterate(compose(
self.get_block_by_hash,
operator.attrgetter('parent_hash'),
operator.attrgetter('header'),
), block)
# we peel off the first element from the iterator which will be the
# temporary block object we constructed.
next(ancestor_generator)
return tuple(take(ancestor_count, ancestor_generator))
def pack(serialized_values: Sequence[bytes]) -> Tuple[Hash32, ...]:
if len(serialized_values) == 0:
return (EMPTY_CHUNK, )
item_size = len(serialized_values[0])
items_per_chunk = get_items_per_chunk(item_size)
number_of_items = len(serialized_values)
number_of_chunks = (number_of_items +
(items_per_chunk - 1)) // items_per_chunk
chunk_partitions = partition(items_per_chunk, serialized_values, pad=b"")
chunks_unpadded = (b"".join(chunk_partition)
for chunk_partition in chunk_partitions)
full_chunks = tuple(
Hash32(chunk) for chunk in take(number_of_chunks - 1, chunks_unpadded))
last_chunk = first(chunks_unpadded)
if len(tuple(chunks_unpadded)) > 0:
raise Exception("Invariant: all chunks have been taken")
return full_chunks + (Hash32(last_chunk.ljust(CHUNK_SIZE, b"\x00")), )
async def serve_request(self,
request: InboundMessage[LocateMessage]) -> None:
with trio.move_on_after(10):
async with self._concurrency_lock:
content_key = request.message.payload.content_key
advertisements = tuple(
take(
MAX_RESPONSE_ADVERTISEMENTS,
self._source_advertisements(content_key),
))
self.logger.debug(
"Serving request: id=%s content_key=%s num=%d",
request.request_id.hex(),
content_key.hex(),
len(advertisements),
)
await self._client.send_locations(
request.sender_node_id,
request.sender_endpoint,
advertisements=advertisements,
request_id=request.request_id,
)
def set_chunk_in_tree(hash_tree: RawHashTree, index: int,
chunk: Hash32) -> RawHashTree:
hash_tree_with_updated_chunk = hash_tree.transform((0, index), chunk)
parent_layer_indices = drop(1, range(len(hash_tree)))
parent_hash_indices = drop(
1, take(len(hash_tree), iterate(lambda index: index // 2, index)))
update_functions = (partial(recompute_hash_in_tree,
layer_index=layer_index,
hash_index=hash_index)
for layer_index, hash_index in zip(
parent_layer_indices, parent_hash_indices))
hash_tree_with_updated_branch = pipe(hash_tree_with_updated_chunk,
*update_functions)
if len(hash_tree_with_updated_branch[-1]) == 1:
return hash_tree_with_updated_branch
elif len(hash_tree_with_updated_branch[-1]) == 2:
return recompute_hash_in_tree(hash_tree_with_updated_branch,
len(hash_tree), 0)
else:
raise Exception("Unreachable")
async def recursive_find_nodes(self, target: NodeID) -> Tuple[ENRAPI, ...]:
self.logger.debug("Recursive find nodes: %s", humanize_node_id(target))
queried_node_ids = set()
unresponsive_node_ids = set()
received_enrs: List[ENRAPI] = []
received_node_ids: Set[NodeID] = set()
async def do_lookup(node_id: NodeID) -> None:
queried_node_ids.add(node_id)
distance = compute_log_distance(node_id, target)
try:
enrs = await self.find_nodes(node_id, distance)
except trio.TooSlowError:
unresponsive_node_ids.add(node_id)
return
for enr in enrs:
received_node_ids.add(enr.node_id)
try:
self.enr_db.set_enr(enr)
except OldSequenceNumber:
received_enrs.append(self.enr_db.get_enr(enr.node_id))
else:
received_enrs.append(enr)
for lookup_round_counter in itertools.count():
candidates = iter_closest_nodes(target, self.routing_table,
received_node_ids)
responsive_candidates = itertools.dropwhile(
lambda node: node in unresponsive_node_ids, candidates)
closest_k_candidates = take(self.routing_table.bucket_size,
responsive_candidates)
closest_k_unqueried_candidates = (
candidate for candidate in closest_k_candidates
if candidate not in queried_node_ids)
nodes_to_query = tuple(take(3, closest_k_unqueried_candidates))
if nodes_to_query:
self.logger.debug(
"Starting lookup round %d for %s",
lookup_round_counter + 1,
humanize_node_id(target),
)
async with trio.open_nursery() as nursery:
for peer in nodes_to_query:
nursery.start_soon(do_lookup, peer)
else:
self.logger.debug(
"Lookup for %s finished in %d rounds",
humanize_node_id(target),
lookup_round_counter,
)
break
# now sort and return the ENR records in order of closesness to the target.
return tuple(
sorted(
_reduce_enrs(received_enrs),
key=lambda enr: compute_distance(enr.node_id, target),
))
async def _periodically_advertise_content(self) -> None:
await self._network.routing_table_ready()
send_channel, receive_channel = trio.open_memory_channel[ContentKey](
self._concurrency)
for _ in range(self._concurrency):
self.manager.run_daemon_task(self._broadcast_worker,
receive_channel)
async for _ in every(30 * 60):
start_at = trio.current_time()
total_keys = len(self.content_storage)
if not total_keys:
continue
first_key = first(
self.content_storage.iter_closest(
NodeID(secrets.token_bytes(32))))
self.logger.info(
"content-processing-starting: total=%d start=%s",
total_keys,
first_key.hex(),
)
processed_keys = 0
last_key = first_key
has_wrapped_around = False
while self.manager.is_running:
elapsed = trio.current_time() - start_at
content_keys = tuple(
take(
self._concurrency * 2,
self.content_storage.enumerate_keys(
start_key=last_key),
))
# TODO: We need to adjust the
# `ContentStorageAPI.enumerate_keys` to allow a
# non-inclusive left bound so we can query all the keys
# **after** the last key we processed.
if content_keys and content_keys[0] == last_key:
content_keys = content_keys[1:]
if not content_keys:
last_key = None
has_wrapped_around = True
continue
for content_key in content_keys:
await send_channel.send(content_key)
last_key = content_keys[-1]
if has_wrapped_around and last_key >= first_key:
break
processed_keys += len(content_keys)
progress = processed_keys * 100 / total_keys
self.logger.debug(
"content-processing: progress=%0.1f processed=%d "
"total=%d at=%s elapsed=%s",
progress,
processed_keys,
total_keys,
"None" if last_key is None else last_key.hex(),
humanize_seconds(int(elapsed)),
)
self.logger.info(
"content-processing-finished: processed=%d/%d elapsed=%s",
processed_keys,
total_keys,
humanize_seconds(int(elapsed)),
)
async def iterative_lookup(
self,
target_id: NodeID,
filter_self: bool = True,
) -> Tuple[Node, ...]:
self.logger.debug("Starting looking up @ %s",
humanize_node_id(target_id))
# tracks the nodes that have already been queried
queried_node_ids: Set[NodeID] = set()
# keeps track of the nodes that are unresponsive
unresponsive_node_ids: Set[NodeID] = set()
# accumulator of all of the valid responses received
received_nodes: DefaultDict[
NodeID, Set[Endpoint]] = collections.defaultdict(set)
async def do_lookup(peer: Node) -> None:
self.logger.debug(
"Looking up %s via node %s",
humanize_node_id(target_id),
humanize_node_id(peer.node_id),
)
distance = compute_log_distance(peer.node_id, target_id)
try:
with trio.fail_after(FIND_NODES_TIMEOUT):
found_nodes = await self.single_lookup(
peer,
distance=distance,
)
except trio.TooSlowError:
unresponsive_node_ids.add(peer.node_id)
else:
if len(found_nodes) == 0:
unresponsive_node_ids.add(peer.node_id)
else:
received_nodes[peer.node_id].add(peer.endpoint)
for node in found_nodes:
received_nodes[node.node_id].add(node.endpoint)
@to_tuple
def get_endpoints(node_id: NodeID) -> Iterator[Endpoint]:
try:
yield self.endpoint_db.get_endpoint(node_id)
except KeyError:
pass
yield from received_nodes[node_id]
for lookup_round_number in itertools.count():
received_node_ids = tuple(received_nodes.keys())
candidates = iter_closest_nodes(target_id, self.routing_table,
received_node_ids)
responsive_candidates = itertools.dropwhile(
lambda node: node in unresponsive_node_ids,
candidates,
)
closest_k_candidates = take(self.routing_table.bucket_size,
responsive_candidates)
closest_k_unqueried_candidates = (
candidate for candidate in closest_k_candidates
if candidate not in queried_node_ids)
nodes_to_query = tuple(
take(
LOOKUP_CONCURRENCY_FACTOR,
closest_k_unqueried_candidates,
))
if nodes_to_query:
self.logger.debug(
"Starting lookup round %d for %s",
lookup_round_number + 1,
humanize_node_id(target_id),
)
queried_node_ids.update(nodes_to_query)
async with trio.open_nursery() as nursery:
for peer_id in nodes_to_query:
if peer_id == self.client.local_node_id:
continue
for endpoint in get_endpoints(peer_id):
nursery.start_soon(do_lookup,
Node(peer_id, endpoint))
else:
self.logger.debug(
"Lookup for %s finished in %d rounds",
humanize_node_id(target_id),
lookup_round_number,
)
break
found_nodes = tuple(
Node(node_id, endpoint)
for node_id, endpoints in received_nodes.items()
for endpoint in endpoints
if (not filter_self or node_id != self.client.local_node_id))
sorted_found_nodes = tuple(
sorted(
found_nodes,
key=lambda node: compute_distance(self.client.local_node_id,
node.node_id),
))
self.logger.debug(
"Finished looking up %s in %d rounds: Found %d nodes after querying %d nodes",
humanize_node_id(target_id),
lookup_round_number,
len(found_nodes),
len(queried_node_ids),
)
return sorted_found_nodes
MerkleProof,
MerkleTree,
_calc_parent_hash,
_hash_layer,
get_branch_indices,
get_merkle_proof,
get_root,
)
if TYPE_CHECKING:
from typing import Tuple # noqa: F401
TreeDepth = 32
EmptyNodeHashes = tuple(
take(
TreeDepth,
iterate(lambda node_hash: hash_eth2(node_hash + node_hash), b"\x00" * 32),
)
)
def verify_merkle_proof(
root: Hash32, leaf: Hash32, index: int, proof: MerkleProof
) -> bool:
"""
Verify that the given ``item`` is on the merkle branch ``proof``
starting with the given ``root``.
"""
assert len(proof) == TreeDepth
value = leaf
for i in range(TreeDepth):
if index // (2 ** i) % 2:
from eth2._utils.tuple import update_tuple_item
from eth_typing import (
Hash32, )
from .common import ( # noqa: F401
_calc_parent_hash, _hash_layer, get_branch_indices, get_merkle_proof,
get_root, MerkleTree, MerkleProof,
)
if TYPE_CHECKING:
from typing import Tuple # noqa: F401
TreeDepth = 32
EmptyNodeHashes = tuple(
take(
TreeDepth,
iterate(lambda node_hash: hash_eth2(node_hash + node_hash),
b'\x00' * 32)))
def verify_merkle_proof(root: Hash32, leaf: Hash32, index: int,
proof: MerkleProof) -> bool:
"""
Verify that the given ``item`` is on the merkle branch ``proof``
starting with the given ``root``.
"""
assert len(proof) == TreeDepth
value = leaf
for i in range(TreeDepth):
if index // (2**i) % 2:
value = hash_eth2(proof[i] + value)
else:
from eth_typing import Hash32
from eth_utils.toolz import iterate, take
from ssz.hash import hash_eth2
CHUNK_SIZE = 32 # named BYTES_PER_CHUNK in the spec
EMPTY_CHUNK = Hash32(b"\x00" * CHUNK_SIZE)
SIGNATURE_FIELD_NAME = "signature"
# number of bytes for a serialized offset
OFFSET_SIZE = 4
FIELDS_META_ATTR = "fields"
ZERO_BYTES32 = Hash32(b"\x00" * 32)
MAX_ZERO_HASHES_LAYER = 100
ZERO_HASHES = tuple(
take(
MAX_ZERO_HASHES_LAYER,
iterate(lambda child: hash_eth2(child + child), ZERO_BYTES32),
))
BASE_TYPES = (int, bytes, bool)
