def test_content_storage_closest_and_furthest_iteration(base_storage): content_keys = tuple(b"key-" + bytes([i]) for i in range(32)) for idx, content_key in enumerate(content_keys): base_storage.set_content(content_key, b"dummy-" + bytes([idx])) target = NodeIDFactory() expected_closest = tuple( sorted( content_keys, key=lambda content_key: compute_content_distance( target, content_key_to_content_id(content_key) ), ) ) expected_furthest = tuple( sorted( content_keys, key=lambda content_key: compute_content_distance( target, content_key_to_content_id(content_key) ), reverse=True, ) ) actual_closest = tuple(base_storage.iter_closest(target)) actual_furthest = tuple(base_storage.iter_furthest(target)) assert actual_closest == expected_closest assert actual_furthest == expected_furthest
def iter_closest(self, target: NodeID) -> Iterable[ContentKey]: yield from sorted( self._db.keys(), key=lambda content_key: compute_content_distance( target, content_key_to_content_id(content_key) ), )
def create( cls, content_key: ContentKey, hash_tree_root: Hash32, private_key: keys.PrivateKey, expires_at: Optional[datetime.datetime] = None, ) -> "Advertisement": if expires_at is None: expires_at = datetime.datetime.utcnow().replace( microsecond=0) + ONE_DAY content_id = content_key_to_content_id(content_key) signature = create_advertisement_signature( content_id, hash_tree_root, expires_at, private_key, ) return cls( content_key=content_key, hash_tree_root=hash_tree_root, expires_at=expires_at, signature_v=signature.v, signature_r=signature.r, signature_s=signature.s, )
class Params: private_key = factory.SubFactory(PrivateKeyFactory) signature = factory.LazyAttribute( lambda o: create_advertisement_signature( content_id=content_key_to_content_id(o.content_key), hash_tree_root=o.hash_tree_root, expires_at=o.expires_at, private_key=o.private_key, ))
def content_radius(self) -> int: if self.is_full: furthest_key = first( self.content_storage.iter_furthest( self._network.local_node_id)) content_id = content_key_to_content_id(furthest_key) return compute_content_distance(self._network.local_node_id, content_id) else: return 2**256 - 1
def insert_content( conn: sqlite3.Connection, content_key: ContentKey, content: bytes, ) -> None: content_id = content_key_to_content_id(content_key) # The high 64 bits of the content id for doing proximate queries short_content_id = int.from_bytes(content_id, "big") >> 193 with conn: conn.execute( STORAGE_INSERT_QUERY, (content_key, short_content_id, content), )
async def process_content(self, content_key: ContentKey, content: bytes) -> None: if self.content_storage.has_content(content_key): local_content = self.content_storage.get_content(content_key) if local_content == content: self.logger.debug( "Ignoring content we already have: content_key=%s", content_key.hex(), ) return self.logger.debug( "Processing content: content_key=%s content=%s", content_key.hex(), content.hex(), ) content_id = content_key_to_content_id(content_key) # TODO: computationally expensive hash_tree_root = ssz.get_hash_tree_root(content, sedes=content_sedes) known_hash_tree_roots = set( self._local_advertisement_db.get_hash_tree_roots_for_content_id( content_id, )) # We should avoid "polution" of our content database with mismatching # roots. This is a stop gap right now because we will need a mechanism # for inserting our own "correct" content into the system even in the # case where the existing "network" content doesn't agree on the hash # tree root. if known_hash_tree_roots and hash_tree_root not in known_hash_tree_roots: known_roots_display = "|".join( (root.hex() for root in known_hash_tree_roots)) raise NotImplementedError( f"Content hash tree root mismatch: " f"content_key={content_key.hex()} root={hash_tree_root.hex()} " f"known={known_roots_display}") self.content_storage.set_content(content_key, content, exists_ok=True) advertisement = self._get_or_create_advertisement( content_key, hash_tree_root) await self._network.broadcast(advertisement) self.logger.debug( "Processed content: content_key=%s content=%s", content_key.hex(), content.hex(), )
def __init__( self, network: AlexandriaNetworkAPI, content_key: ContentKey, hash_tree_root: Hash32, max_node_ids: int = 32, concurrency: int = 3, ) -> None: self._network = network self._concurrency = concurrency self.content_key = content_key self.hash_tree_root = hash_tree_root self.content_id = content_key_to_content_id(content_key) self.node_queue = ResourceQueue(()) self._content_ready = trio.Event()
def __init__( self, network: AlexandriaNetworkAPI, content_key: ContentKey, concurrency: int = 3, ) -> None: self.logger = get_extended_debug_logger("ddht.Seeker") self.concurrency = concurrency self.content_key = content_key self.content_id = content_key_to_content_id(content_key) self._network = network self._content_send, self.content_receive = trio.open_memory_channel[ bytes](0)
async def test_alexandria_network_retrieve_content_api( alice, tester, ): async with AsyncExitStack() as stack: networks = await stack.enter_async_context( tester.alexandria.network_group(8)) all_node_ids = {alice.node_id } | {network.local_node_id for network in networks} # we need a content_key for which "alice" is the furthest from the content for i in range(4096): content_key = b"key-%d" % i content_id = content_key_to_content_id(content_key) furthest_node_id = max( all_node_ids, key=lambda node_id: compute_content_distance( node_id, content_id), ) if furthest_node_id == alice.node_id: break # get the closest node and set the content key in their storage closest_network = min( networks, key=lambda network: compute_content_distance( network.local_node_id, content_id), ) closest_network.storage.set_content(content_key, b"content-value") bootnodes = tuple(network.enr_manager.enr for network in networks) alice_network = await stack.enter_async_context( alice.alexandria.network(bootnodes=bootnodes)) # give alice a little time to connect to the network as well with trio.fail_after(20): for _ in range(1000): await trio.lowlevel.checkpoint() with trio.fail_after(10): content = await alice_network.retrieve_content(content_key) assert content == b"content-value"
def test_advertisement_creation(): private_key = PrivateKeyFactory() content_key = b"\x01testkey" hash_tree_root = b"\x12" * 32 ad = Advertisement.create(content_key, hash_tree_root, private_key) assert ad.content_id == content_key_to_content_id(content_key) signature = ad.signature assert signature.v == ad.signature_v assert signature.r == ad.signature_r assert signature.s == ad.signature_s assert ad.public_key == private_key.public_key assert ad.is_valid ad.verify() expected_node_id = keccak(private_key.public_key.to_bytes()) assert ad.node_id == expected_node_id
def set_content( self, content_key: ContentKey, content: bytes, exists_ok: bool = False ) -> None: """ /content_id.hex()[:2]/content_id.hex()[2:4]/content_id.hex() """ if self.has_content(content_key): if exists_ok: self.delete_content(content_key) else: raise ContentAlreadyExists( f"Content already exists for key: content_key={content_key.hex()}" ) content_id = content_key_to_content_id(content_key) content_id_hex = content_id.hex() # For some content_id: 0xdeadbeef12345... # The directory is: <base-dir>/de/ad/deadbeef1234... content_path = ( self.base_dir / content_id_hex[:2] / content_id_hex[2:4] / content_id_hex ) content_path_rel = content_path.relative_to(self.base_dir) # Lazily create the directory structure content_path.parent.mkdir(parents=True, exist_ok=True) # We have already checked that the file doesn't exist and that the # `content_key` is not present in the database, however, there is the # possibility for a race condition at the filesystem level where the # file appears between the check and writing to it. In the case of any # error we want to avoid our filesystem and database being out-of-sync. try: with content_path.open("wb") as content_file: content_file.write(content) insert_content(self._conn, content_key, len(content), content_path_rel) except Exception: content_path.unlink(missing_ok=True) delete_content(self._conn, content_key) raise
async def _serve_find_content(self) -> None: async with self.client.subscribe(FindContentMessage) as subscription: self._find_content_handler_ready.set() async for request in subscription: # if content in storage, serve it.... # else serve ENR records that we know of which are *closest* content_key = request.message.payload.content_key if self.storage.has_content(content_key): content = self.storage.get_content(content_key) await self.client.send_found_content( request.sender_node_id, request.sender_endpoint, enrs=None, content=content, request_id=request.request_id, ) else: content_id = content_key_to_content_id(content_key) distance = compute_content_distance(self.local_node_id, content_id) try: response_enrs = self._source_nodes((distance,)) except ValidationError as err: self.logger.debug( "Ignoring invalid FindNodesMessage from %s@%s: %s", request.sender_node_id.hex(), request.sender_endpoint, err, ) else: await self.client.send_found_content( request.sender_node_id, request.sender_endpoint, enrs=response_enrs, content=None, request_id=request.request_id, )
def content_id(self) -> ContentID: return content_key_to_content_id(self.content_key)