def get_unqueried_node_ids() -> Tuple[NodeID, ...]: """ Get the three nodes that are closest to the target such that the node is in the closest `k` nodes which haven't been deemed unresponsive. """ # Construct an iterable of *all* the nodes we know about ordered by # closeness to the target. candidates = iter_closest_nodes( target, network.routing_table, received_node_ids ) # Remove any unresponsive nodes from that iterable responsive_candidates = itertools.filterfalse( lambda node_id: node_id in unresponsive_node_ids, candidates ) # Grab the closest K closest_k_candidates = take( network.routing_table.bucket_size, responsive_candidates, ) # Filter out any from the closest K that we've already queried or that are in-flight closest_k_unqueried = itertools.filterfalse( lambda node_id: node_id in queried_node_ids or node_id in in_flight, closest_k_candidates, ) return tuple(take(3, closest_k_unqueried))
def decode_packet( data: bytes, local_node_id: NodeID, ) -> AnyPacket: iv = data[:16] masking_key = cast(AES128Key, local_node_id[:16]) cipher_text_stream = aesctr_decrypt_stream(masking_key, iv, data[16:]) # Decode the header header_bytes = bytes(take(HEADER_PACKET_SIZE, cipher_text_stream)) header = Header.from_wire_bytes(header_bytes) auth_data_bytes = bytes(take(header.auth_data_size, cipher_text_stream)) auth_data: Union[MessagePacket, WhoAreYouPacket, HandshakePacket] if header.flag == 0: auth_data = MessagePacket.from_wire_bytes(auth_data_bytes) elif header.flag == 1: auth_data = WhoAreYouPacket.from_wire_bytes(auth_data_bytes) elif header.flag == 2: auth_data = HandshakePacket.from_wire_bytes(auth_data_bytes) else: raise DecodingError(f"Unable to decode datagram: {data.hex()}", data) message_cipher_text = data[16 + HEADER_PACKET_SIZE + header.auth_data_size:] return cast( AnyPacket, Packet(iv, header, auth_data, message_cipher_text, dest_node_id=local_node_id), )
async def lookup(self, target: NodeID) -> None: self.logger.debug("Looking up %s", encode_hex(target)) queried_node_ids = set() unresponsive_node_ids = set() received_enrs: List[ENRAPI] = [] received_node_ids: List[NodeID] = [] async def lookup_and_store_response(peer: NodeID) -> None: enrs = await self.lookup_at_peer(peer, target) queried_node_ids.add(peer) if enrs is not None: for enr in enrs: received_node_ids.append(enr.node_id) try: self.enr_db.set_enr(enr) except OldSequenceNumber: received_enrs.append(self.enr_db.get_enr(enr.node_id)) else: received_enrs.append(enr) else: unresponsive_node_ids.add(peer) for lookup_round_counter in itertools.count(): candidates = iter_closest_nodes( target, self.routing_table, received_node_ids ) responsive_candidates = itertools.dropwhile( lambda node: node in unresponsive_node_ids, candidates ) closest_k_candidates = take( self.routing_table.bucket_size, responsive_candidates ) closest_k_unqueried_candidates = ( candidate for candidate in closest_k_candidates if candidate not in queried_node_ids ) nodes_to_query = tuple( take(LOOKUP_PARALLELIZATION_FACTOR, closest_k_unqueried_candidates) ) if nodes_to_query: self.logger.debug( "Starting lookup round %d for %s", lookup_round_counter + 1, encode_hex(target), ) async with trio.open_nursery() as nursery: for peer in nodes_to_query: nursery.start_soon(lookup_and_store_response, peer) else: self.logger.debug( "Lookup for %s finished in %d rounds", encode_hex(target), lookup_round_counter, ) break
async def connect_to_nodes(self, nodes: Iterator[NodeAPI]) -> None: # create an generator for the nodes nodes_iter = iter(nodes) while True: if self.is_full or not self.is_operational: return # only attempt to connect to up to the maximum number of available # peer slots that are open. available_peer_slots = self.max_peers - len(self) batch_size = clamp(1, 10, available_peer_slots) batch = tuple(take(batch_size, nodes_iter)) # There are no more *known* nodes to connect to. if not batch: return self.logger.debug( 'Initiating %d peer connection attempts with %d open peer slots', len(batch), available_peer_slots, ) # Try to connect to the peers concurrently. await asyncio.gather( *(self.connect_to_node(node) for node in batch), loop=self.get_event_loop(), )
def _get_blocks(self, start_block: BaseBeaconBlock, max_blocks: int) -> Iterable[BaseBeaconBlock]: if max_blocks < 0: raise Exception("Invariant: max blocks cannot be negative") if max_blocks == 0: return yield start_block blocks_generator = cons(start_block, ( self.db.get_canonical_block_by_slot(slot) for slot in itertools.count(start_block.slot + 1) )) max_blocks_generator = take(max_blocks, blocks_generator) try: # ensure only a connected chain is returned (breaks might occur if the start block is # not part of the canonical chain or if the canonical chain changes during execution) for parent, child in sliding_window(2, max_blocks_generator): if child.parent_root == parent.hash: yield child else: break except BlockNotFound: return
async def purge_distant_ads(self) -> None: if self.max_advertisement_count is None: raise Exception("Invalid") while self.manager.is_running: # Ensure that the inner loop here doesn't block the event loop await trio.lowlevel.checkpoint() # Check if we are over the limit total_advertisements = self.advertisement_db.count() if total_advertisements <= self.max_advertisement_count: break num_to_purge = total_advertisements - self.max_advertisement_count # Purge in chunks of 64 to avoid blocking too long to_purge = tuple( take( min(64, num_to_purge), itertools.filterfalse( lambda ad: ad.node_id == self._network.local_node_id, self.advertisement_db.furthest( self._network.local_node_id), ), )) self.logger.debug("Purging ads outside radius: count=%d", len(to_purge)) for advertisement in to_purge: self.advertisement_db.remove(advertisement)
def merkleize(chunks: Sequence[Hash32]) -> Hash32: padded_chunks = pad_chunks(chunks) number_of_layers = int(math.log2(len(padded_chunks))) + 1 layers = take(number_of_layers, iterate(hash_layer, padded_chunks)) root, = last(layers) return root
def _batch_of_missing_hashes(self) -> Tuple[TrackedRequest, ...]: """ Take a batch of missing trie hashes, sized for a single peer request """ return tuple(take( REQUEST_SIZE, self._missing_trie_hashes(), ))
async def new_sync_headers( self, max_batch_size: int = None) -> AsyncIterator[Tuple[BlockHeader, ...]]: while True: next_batch = tuple(take(max_batch_size, self._headers_to_emit)) if not next_batch: self._new_data.clear() await self._new_data.wait() continue yield next_batch self._headers_to_emit = tuple(drop(max_batch_size, self._headers_to_emit))
async def purge_expired_ads(self) -> None: while self.manager.is_running: await trio.lowlevel.checkpoint() # Purge in chunks of 64 to avoid blocking too long to_purge = tuple(take(64, self.advertisement_db.expired())) if not to_purge: break self.logger.debug("Purging expired ads: count=%d", len(to_purge)) for advertisement in to_purge: self.advertisement_db.remove(advertisement)
def test_iterating_operation_pool(sample_attestation_params): some_attestation_params = zip(itertools.count(), forever(sample_attestation_params)) some_attestations = map(lambda x: mk_attestation(*x), some_attestation_params) attestation_count = 20 attestations = tuple(take(attestation_count, some_attestations)) pool = AttestationPool() for a in attestations: pool.add(a) for _, a in pool: assert a in attestations
async def make_raw_request(raw_request: str): with trio.fail_after(2): data = raw_request.encode("utf8") data_iter = iter(data) while True: chunk = bytes(take(1024, data_iter)) if chunk: try: await socket.send_all(chunk) except trio.BrokenResourceError: break else: break return await read_json(socket.socket, buffer)
def get_ancestors(self, limit: int, header: BlockHeaderAPI) -> Tuple[BlockAPI, ...]: ancestor_count = min(header.block_number, limit) # We construct a temporary block object vm_class = self.get_vm_class_for_block_number(header.block_number) block_class = vm_class.get_block_class() block = block_class(header=header, uncles=[], transactions=[]) ancestor_generator = iterate(compose( self.get_block_by_hash, operator.attrgetter('parent_hash'), operator.attrgetter('header'), ), block) # we peel off the first element from the iterator which will be the # temporary block object we constructed. next(ancestor_generator) return tuple(take(ancestor_count, ancestor_generator))
def pack(serialized_values: Sequence[bytes]) -> Tuple[Hash32, ...]: if len(serialized_values) == 0: return (EMPTY_CHUNK, ) item_size = len(serialized_values[0]) items_per_chunk = get_items_per_chunk(item_size) number_of_items = len(serialized_values) number_of_chunks = (number_of_items + (items_per_chunk - 1)) // items_per_chunk chunk_partitions = partition(items_per_chunk, serialized_values, pad=b"") chunks_unpadded = (b"".join(chunk_partition) for chunk_partition in chunk_partitions) full_chunks = tuple( Hash32(chunk) for chunk in take(number_of_chunks - 1, chunks_unpadded)) last_chunk = first(chunks_unpadded) if len(tuple(chunks_unpadded)) > 0: raise Exception("Invariant: all chunks have been taken") return full_chunks + (Hash32(last_chunk.ljust(CHUNK_SIZE, b"\x00")), )
async def serve_request(self, request: InboundMessage[LocateMessage]) -> None: with trio.move_on_after(10): async with self._concurrency_lock: content_key = request.message.payload.content_key advertisements = tuple( take( MAX_RESPONSE_ADVERTISEMENTS, self._source_advertisements(content_key), )) self.logger.debug( "Serving request: id=%s content_key=%s num=%d", request.request_id.hex(), content_key.hex(), len(advertisements), ) await self._client.send_locations( request.sender_node_id, request.sender_endpoint, advertisements=advertisements, request_id=request.request_id, )
def set_chunk_in_tree(hash_tree: RawHashTree, index: int, chunk: Hash32) -> RawHashTree: hash_tree_with_updated_chunk = hash_tree.transform((0, index), chunk) parent_layer_indices = drop(1, range(len(hash_tree))) parent_hash_indices = drop( 1, take(len(hash_tree), iterate(lambda index: index // 2, index))) update_functions = (partial(recompute_hash_in_tree, layer_index=layer_index, hash_index=hash_index) for layer_index, hash_index in zip( parent_layer_indices, parent_hash_indices)) hash_tree_with_updated_branch = pipe(hash_tree_with_updated_chunk, *update_functions) if len(hash_tree_with_updated_branch[-1]) == 1: return hash_tree_with_updated_branch elif len(hash_tree_with_updated_branch[-1]) == 2: return recompute_hash_in_tree(hash_tree_with_updated_branch, len(hash_tree), 0) else: raise Exception("Unreachable")
async def recursive_find_nodes(self, target: NodeID) -> Tuple[ENRAPI, ...]: self.logger.debug("Recursive find nodes: %s", humanize_node_id(target)) queried_node_ids = set() unresponsive_node_ids = set() received_enrs: List[ENRAPI] = [] received_node_ids: Set[NodeID] = set() async def do_lookup(node_id: NodeID) -> None: queried_node_ids.add(node_id) distance = compute_log_distance(node_id, target) try: enrs = await self.find_nodes(node_id, distance) except trio.TooSlowError: unresponsive_node_ids.add(node_id) return for enr in enrs: received_node_ids.add(enr.node_id) try: self.enr_db.set_enr(enr) except OldSequenceNumber: received_enrs.append(self.enr_db.get_enr(enr.node_id)) else: received_enrs.append(enr) for lookup_round_counter in itertools.count(): candidates = iter_closest_nodes(target, self.routing_table, received_node_ids) responsive_candidates = itertools.dropwhile( lambda node: node in unresponsive_node_ids, candidates) closest_k_candidates = take(self.routing_table.bucket_size, responsive_candidates) closest_k_unqueried_candidates = ( candidate for candidate in closest_k_candidates if candidate not in queried_node_ids) nodes_to_query = tuple(take(3, closest_k_unqueried_candidates)) if nodes_to_query: self.logger.debug( "Starting lookup round %d for %s", lookup_round_counter + 1, humanize_node_id(target), ) async with trio.open_nursery() as nursery: for peer in nodes_to_query: nursery.start_soon(do_lookup, peer) else: self.logger.debug( "Lookup for %s finished in %d rounds", humanize_node_id(target), lookup_round_counter, ) break # now sort and return the ENR records in order of closesness to the target. return tuple( sorted( _reduce_enrs(received_enrs), key=lambda enr: compute_distance(enr.node_id, target), ))
async def _periodically_advertise_content(self) -> None: await self._network.routing_table_ready() send_channel, receive_channel = trio.open_memory_channel[ContentKey]( self._concurrency) for _ in range(self._concurrency): self.manager.run_daemon_task(self._broadcast_worker, receive_channel) async for _ in every(30 * 60): start_at = trio.current_time() total_keys = len(self.content_storage) if not total_keys: continue first_key = first( self.content_storage.iter_closest( NodeID(secrets.token_bytes(32)))) self.logger.info( "content-processing-starting: total=%d start=%s", total_keys, first_key.hex(), ) processed_keys = 0 last_key = first_key has_wrapped_around = False while self.manager.is_running: elapsed = trio.current_time() - start_at content_keys = tuple( take( self._concurrency * 2, self.content_storage.enumerate_keys( start_key=last_key), )) # TODO: We need to adjust the # `ContentStorageAPI.enumerate_keys` to allow a # non-inclusive left bound so we can query all the keys # **after** the last key we processed. if content_keys and content_keys[0] == last_key: content_keys = content_keys[1:] if not content_keys: last_key = None has_wrapped_around = True continue for content_key in content_keys: await send_channel.send(content_key) last_key = content_keys[-1] if has_wrapped_around and last_key >= first_key: break processed_keys += len(content_keys) progress = processed_keys * 100 / total_keys self.logger.debug( "content-processing: progress=%0.1f processed=%d " "total=%d at=%s elapsed=%s", progress, processed_keys, total_keys, "None" if last_key is None else last_key.hex(), humanize_seconds(int(elapsed)), ) self.logger.info( "content-processing-finished: processed=%d/%d elapsed=%s", processed_keys, total_keys, humanize_seconds(int(elapsed)), )
async def iterative_lookup( self, target_id: NodeID, filter_self: bool = True, ) -> Tuple[Node, ...]: self.logger.debug("Starting looking up @ %s", humanize_node_id(target_id)) # tracks the nodes that have already been queried queried_node_ids: Set[NodeID] = set() # keeps track of the nodes that are unresponsive unresponsive_node_ids: Set[NodeID] = set() # accumulator of all of the valid responses received received_nodes: DefaultDict[ NodeID, Set[Endpoint]] = collections.defaultdict(set) async def do_lookup(peer: Node) -> None: self.logger.debug( "Looking up %s via node %s", humanize_node_id(target_id), humanize_node_id(peer.node_id), ) distance = compute_log_distance(peer.node_id, target_id) try: with trio.fail_after(FIND_NODES_TIMEOUT): found_nodes = await self.single_lookup( peer, distance=distance, ) except trio.TooSlowError: unresponsive_node_ids.add(peer.node_id) else: if len(found_nodes) == 0: unresponsive_node_ids.add(peer.node_id) else: received_nodes[peer.node_id].add(peer.endpoint) for node in found_nodes: received_nodes[node.node_id].add(node.endpoint) @to_tuple def get_endpoints(node_id: NodeID) -> Iterator[Endpoint]: try: yield self.endpoint_db.get_endpoint(node_id) except KeyError: pass yield from received_nodes[node_id] for lookup_round_number in itertools.count(): received_node_ids = tuple(received_nodes.keys()) candidates = iter_closest_nodes(target_id, self.routing_table, received_node_ids) responsive_candidates = itertools.dropwhile( lambda node: node in unresponsive_node_ids, candidates, ) closest_k_candidates = take(self.routing_table.bucket_size, responsive_candidates) closest_k_unqueried_candidates = ( candidate for candidate in closest_k_candidates if candidate not in queried_node_ids) nodes_to_query = tuple( take( LOOKUP_CONCURRENCY_FACTOR, closest_k_unqueried_candidates, )) if nodes_to_query: self.logger.debug( "Starting lookup round %d for %s", lookup_round_number + 1, humanize_node_id(target_id), ) queried_node_ids.update(nodes_to_query) async with trio.open_nursery() as nursery: for peer_id in nodes_to_query: if peer_id == self.client.local_node_id: continue for endpoint in get_endpoints(peer_id): nursery.start_soon(do_lookup, Node(peer_id, endpoint)) else: self.logger.debug( "Lookup for %s finished in %d rounds", humanize_node_id(target_id), lookup_round_number, ) break found_nodes = tuple( Node(node_id, endpoint) for node_id, endpoints in received_nodes.items() for endpoint in endpoints if (not filter_self or node_id != self.client.local_node_id)) sorted_found_nodes = tuple( sorted( found_nodes, key=lambda node: compute_distance(self.client.local_node_id, node.node_id), )) self.logger.debug( "Finished looking up %s in %d rounds: Found %d nodes after querying %d nodes", humanize_node_id(target_id), lookup_round_number, len(found_nodes), len(queried_node_ids), ) return sorted_found_nodes
MerkleProof, MerkleTree, _calc_parent_hash, _hash_layer, get_branch_indices, get_merkle_proof, get_root, ) if TYPE_CHECKING: from typing import Tuple # noqa: F401 TreeDepth = 32 EmptyNodeHashes = tuple( take( TreeDepth, iterate(lambda node_hash: hash_eth2(node_hash + node_hash), b"\x00" * 32), ) ) def verify_merkle_proof( root: Hash32, leaf: Hash32, index: int, proof: MerkleProof ) -> bool: """ Verify that the given ``item`` is on the merkle branch ``proof`` starting with the given ``root``. """ assert len(proof) == TreeDepth value = leaf for i in range(TreeDepth): if index // (2 ** i) % 2:
from eth2._utils.tuple import update_tuple_item from eth_typing import ( Hash32, ) from .common import ( # noqa: F401 _calc_parent_hash, _hash_layer, get_branch_indices, get_merkle_proof, get_root, MerkleTree, MerkleProof, ) if TYPE_CHECKING: from typing import Tuple # noqa: F401 TreeDepth = 32 EmptyNodeHashes = tuple( take( TreeDepth, iterate(lambda node_hash: hash_eth2(node_hash + node_hash), b'\x00' * 32))) def verify_merkle_proof(root: Hash32, leaf: Hash32, index: int, proof: MerkleProof) -> bool: """ Verify that the given ``item`` is on the merkle branch ``proof`` starting with the given ``root``. """ assert len(proof) == TreeDepth value = leaf for i in range(TreeDepth): if index // (2**i) % 2: value = hash_eth2(proof[i] + value) else:
from eth_typing import Hash32 from eth_utils.toolz import iterate, take from ssz.hash import hash_eth2 CHUNK_SIZE = 32 # named BYTES_PER_CHUNK in the spec EMPTY_CHUNK = Hash32(b"\x00" * CHUNK_SIZE) SIGNATURE_FIELD_NAME = "signature" # number of bytes for a serialized offset OFFSET_SIZE = 4 FIELDS_META_ATTR = "fields" ZERO_BYTES32 = Hash32(b"\x00" * 32) MAX_ZERO_HASHES_LAYER = 100 ZERO_HASHES = tuple( take( MAX_ZERO_HASHES_LAYER, iterate(lambda child: hash_eth2(child + child), ZERO_BYTES32), )) BASE_TYPES = (int, bytes, bool)