def to_sg_node(self) -> 'SGNodeAPI': from alexandria.skip_graph import SGNode return SGNode( key=content_key_to_graph_key(self.key), neighbors_left=tuple( content_key_to_graph_key(neighbor) for neighbor in self.neighbors_left ), neighbors_right=tuple( content_key_to_graph_key(neighbor) for neighbor in self.neighbors_right ), )
async def _handle_insert_requests(self) -> None: async def do_graph_insert(key: Key) -> None: try: await self.graph.insert(key) except AlreadyPresent: pass except Exception: self.logger.exception("Error inserting key: %s", key) else: self.logger.info('%s: Inserted %s', self.client.local_node, key) async with trio.open_nursery() as nursery: async with self.client.message_dispatcher.subscribe( GraphInsert) as subscription: async for request in subscription: self.logger.debug("handling request: %s", request) # Acknowledge the request first, then do the linking. await self.client.send_graph_inserted( request.node, request_id=request.payload.request_id, ) if not self._network_graph_ready.is_set(): continue key = content_key_to_graph_key(request.payload.key) nursery.start_soon(do_graph_insert, key)
async def _periodically_advertise_content(self) -> None: async def do_graph_insert(key: Key) -> None: # Insert the key into the skip graph. with trio.move_on_after(INSERT_TIMEOUT): try: sg_node = await self.graph.insert(key) except AlreadyPresent: pass else: self.graph_db.set(key, sg_node) if not self.graph_db.has(key): with trio.move_on_after(TRAVERSAL_TIMEOUT): sg_node = await self.network.get_node(key) traversal_result = await get_traversal_result( self.network, self.graph_db, sg_node, max_traversal_distance=10, ) if traversal_result.is_valid: self.graph_db.set(key, sg_node) else: self.logger.info("GOT INVALID NODE: %s -> %s", sg_node, traversal_result) with trio.move_on_after(2 * INSERT_TIMEOUT): await self.network.insert(key) async def do_announce(key: bytes) -> None: with trio.move_on_after(ANNOUNCE_TIMEOUT): await self.network.announce( key, Node(self.client.local_node_id, self.client.external_endpoint), ) async with trio.open_nursery() as nursery: await self.wait_graph_initialized() while self.manager.is_running: key = await self.advertise_queue.pop_next() if not self.content_manager.has_data(key): self.logger.debug( 'Discarding announcement key missing content: %s', encode_hex(key), ) continue self.logger.debug('Announcing: %s', encode_hex(key)) nursery.start_soon(do_announce, key) nursery.start_soon(do_graph_insert, content_key_to_graph_key(key)) self.advertise_queue.enqueue(key)
async def test_client_send_graph_delete(alice_and_bob_clients): alice, bob = alice_and_bob_clients async with bob.message_dispatcher.subscribe(GraphDelete) as subscription: request_id = await alice.send_graph_delete( bob.local_node, key=content_key_to_graph_key(b'key'), ) with trio.fail_after(1): message = await subscription.receive() assert message.node == alice.local_node payload = message.payload assert isinstance(payload, GraphDelete) assert payload.request_id == request_id assert payload.key == b'key'
async def _handle_get_graph_node_requests(self) -> None: async with self.client.message_dispatcher.subscribe( GraphGetNode) as subscription: async for request in subscription: self.logger.debug("handling request: %s", request) key = content_key_to_graph_key(request.payload.key) sg_node: Optional[SGNodeAPI] try: sg_node = self.graph_db.get(key) except KeyError: sg_node = None await self.client.send_graph_node( request.node, request_id=request.payload.request_id, sg_node=sg_node, )
async def _handle_delete_requests(self) -> None: async with self.client.message_dispatcher.subscribe( GraphDelete) as subscription: async for request in subscription: self.logger.debug("handling request: %s", request) # Acknowledge the request first, then do the linking. await self.client.send_graph_deleted( request.node, request_id=request.payload.request_id, ) if not self._network_graph_ready.is_set(): continue key = content_key_to_graph_key(request.payload.key) try: await self.graph.delete(key) except Exception: self.logger.exception("Error deleteing key: %s", key) continue
async def _initialize_network_graph(self) -> GraphAPI: async def do_get_introduction( node: Node, send_channel: trio.abc.SendChannel[Tuple[ SGNodeAPI, TraversalResults]], # noqa: E501 ) -> None: try: with trio.fail_after(INTRODUCTION_TIMEOUT): candidates = await self.network.get_introduction(node) except trio.TooSlowError: self.logger.debug("Timeout getting introduction from %s", node) return self.logger.debug("Got %d introductions from %s", len(candidates), node) async with send_channel: for candidate in candidates: import time start_at = time.monotonic() try: with trio.fail_after(TRAVERSAL_TIMEOUT): result = await get_traversal_result( self.network, self.graph_db, candidate, max_traversal_distance=10, ) except trio.TooSlowError: self.logger.error( "%s: Traversal timeout: %s", self.client.local_node, candidate, ) return else: end_at = time.monotonic() self.logger.info( "%s: Traversal finished in %s seconds", self.client.local_node, end_at - start_at, ) await send_channel.send(result) while self.manager.is_running: if self.config.can_initialize_network_skip_graph: # Use a probabalistic mechanism here so that multiple # bootnodes coming online at the same time are unlikely to # try and concurrently seed the network with content. if secrets.randbits(256) >= self.client.local_node_id: for key in self.content_manager.iter_content_keys(): seed_node = SGNode(content_key_to_graph_key(key)) break else: continue self.logger.info("%s: Seeding network graph", self.client.local_node) self.graph_db.set(seed_node.key, seed_node) return NetworkGraph(self.graph_db, seed_node, self.network) random_content_id = NodeID(secrets.randbits(256)) candidates = await self.network.iterative_lookup(random_content_id) if not candidates: self.logger.debug("No candidates for introduction") await trio.sleep(5) continue send_channel, receive_channel = trio.open_memory_channel[Tuple[ SGNodeAPI, TraversalResults]](256) # noqa: E501 async with trio.open_nursery() as nursery: async with send_channel: for node in candidates: nursery.start_soon(do_get_introduction, node, send_channel.clone()) async with receive_channel: introduction_results = tuple( [result async for result in receive_channel]) if not introduction_results: self.logger.debug("Received no introductions. sleeping....") await trio.sleep(5) continue best_result = sorted(introduction_results, key=lambda result: result.score, reverse=True)[0] if best_result.score > 0.0: self.logger.info( "%s: Network SkipGraph initialized: node=%s score=%s", self.client.local_node, best_result.node, best_result.score, ) return NetworkGraph(self.graph_db, best_result.node, self.network) else: self.logger.info( "Failed to initialize Skip Graph. All introductions were faulty: %s", tuple(str(result) for result in introduction_results), ) await trio.sleep(5)