async def test_client_send_graph_introduction(alice_and_bob_clients): alice, bob = alice_and_bob_clients async with bob.message_dispatcher.subscribe( GraphIntroduction) as subscription: await alice.send_graph_introduction( bob.local_node, request_id=1234, graph_nodes=(SGNode(5, (3, 2), (7, 9)), SGNode(7, (5, ), (9, ))), ) with trio.fail_after(1): message = await subscription.receive() assert message.node == alice.local_node payload = message.payload assert isinstance(payload, GraphIntroduction) assert payload.request_id == 1234 assert len(payload.nodes) == 2 node_5, node_7 = payload.nodes[0].to_sg_node( ), payload.nodes[1].to_sg_node() assert node_5.key == 5 assert node_5.neighbors[0] == [3, 2] assert node_5.neighbors[1] == [7, 9] assert node_7.key == 7 assert node_7.neighbors[0] == [5] assert node_7.neighbors[1] == [9]
def test_node_get_membership_at_level(): node = SGNode(1234) assert node.get_membership_at_level(0) == 0 assert node.get_membership_at_level(1) == 1 assert node.get_membership_at_level(2) == 3 assert node.get_membership_at_level(3) == 3 assert node.get_membership_at_level(4) == 11
async def do_test_skip_graph_search_fuzz(anchor_key, keys_to_insert, keys_to_search, random_module): anchor = SGNode(anchor_key) graph = LocalGraph(anchor) inserted = {anchor_key} for key in keys_to_insert: graph.cursor = graph.db.get(random.choice(tuple(inserted))) if key in inserted: with pytest.raises(AlreadyPresent): await graph.insert(key) else: node = await graph.insert(key) assert node.key == key inserted.add(key) validate_graph(graph) for key in keys_to_search: graph.cursor = graph.db._db[random.choice(tuple(inserted))] if key == anchor_key or key in keys_to_insert: node = await graph.search(key) assert node.key == key else: with pytest.raises(NotFound): await graph.search(key)
async def test_graph_reverse_iteration(): graph = LocalGraph(SGNode(1)) for key in (3, 5): await graph.insert(key) assert tuple([key async for key in graph.iter_keys(start=10, end=5)]) == () assert tuple([key async for key in graph.iter_keys(start=10, end=4)]) == (5, ) assert tuple([key async for key in graph.iter_keys(start=10, end=3)]) == (5, ) assert tuple([key async for key in graph.iter_keys(start=10, end=2)]) == (5, 3) assert tuple([key async for key in graph.iter_keys(start=5, end=3)]) == (5, ) assert tuple([key async for key in graph.iter_keys(start=5, end=2)]) == (5, 3) assert tuple([key async for key in graph.iter_keys(start=5, end=1)]) == (5, 3) assert tuple([key async for key in graph.iter_keys(start=5, end=0) ]) == (5, 3, 1) assert tuple([key async for key in graph.iter_keys(start=4, end=0)]) == (3, 1) assert tuple([key async for key in graph.iter_keys(start=3, end=0)]) == (3, 1) assert tuple([key async for key in graph.iter_keys(start=2, end=0)]) == (1, ) assert tuple([key async for key in graph.iter_keys(start=1, end=0)]) == (1, )
async def test_search(): anchor = SGNode(0) graph = LocalGraph(anchor) for key in range(5, 100, 5): result = await graph.insert(key) assert result.key == key validate_graph(graph) assert (await graph.search(0)).key == 0 node_5 = await graph.search(5) assert node_5.key == 5 with pytest.raises(NotFound): await graph.search(6) graph.cursor = node_5 with pytest.raises(NotFound): await graph.search(6) with pytest.raises(NotFound): await graph.search(4) node_80 = await graph.search(80) assert node_80.key == 80
async def _handle_introduction(): with trio.fail_after(1): request = await subscription.receive() assert isinstance(request.payload, GraphGetIntroduction) await bob.send_graph_introduction( request.node, request_id=request.payload.request_id, graph_nodes=(SGNode(5, (3, 0), (7, 9)), ), )
async def test_insert_far_left(): anchor = SGNode(1) graph = LocalGraph(anchor) node = await graph.insert(0) assert node.key == 0 assert node.get_neighbor(0, RIGHT) == anchor.key assert node.get_neighbor(0, LEFT) is None validate_graph(graph)
def to_sg_node(self) -> 'SGNodeAPI': from alexandria.skip_graph import SGNode return SGNode( key=content_key_to_graph_key(self.key), neighbors_left=tuple( content_key_to_graph_key(neighbor) for neighbor in self.neighbors_left ), neighbors_right=tuple( content_key_to_graph_key(neighbor) for neighbor in self.neighbors_right ), )
async def _handle_request(): with trio.fail_after(1): request = await subscription.receive() assert isinstance(request.payload, GraphGetNode) assert request.payload.key == b'\x05' await bob.send_graph_node( request.node, request_id=request.payload.request_id, sg_node=SGNode(5, (3, 0), (7, 9)), )
async def test_insert_sequential_to_the_correct_mixed_order(): anchor = SGNode(0) graph = LocalGraph(anchor) node_3, node_1, node_2 = tuple( [await graph.insert(key) for key in (3, 1, 2)]) validate_graph(graph) assert anchor.get_neighbor(0, LEFT) is None assert anchor.get_neighbor(0, RIGHT) == 1 assert node_1.get_neighbor(0, LEFT) == 0 assert node_1.get_neighbor(0, RIGHT) == 2 assert node_2.get_neighbor(0, LEFT) == 1 assert node_2.get_neighbor(0, RIGHT) == 3 assert node_3.get_neighbor(0, LEFT) == 2 assert node_3.get_neighbor(0, RIGHT) is None
async def test_insert_far_right(): anchor = SGNode(0) graph = LocalGraph(anchor) node = await graph.insert(1) assert node.key == 1 assert node.get_neighbor(0, LEFT) == anchor.key assert node.get_neighbor(0, RIGHT) is None assert node.get_neighbor(node.max_level, LEFT) is None assert node.get_neighbor(node.max_level, RIGHT) is None validate_graph(graph)
def test_single_node(): node = SGNode(0) assert node.max_level == 0 for level in range(256): assert node.get_neighbor(level, LEFT) is None assert node.get_neighbor(level, RIGHT) is None assert tuple(node.iter_down_levels(0, LEFT)) == ((0, None), ) assert tuple(node.iter_down_levels(0, RIGHT)) == ((0, None), )
async def test_delete(key_order): anchor = SGNode(4) graph = LocalGraph(anchor) for key in sorted(key_order): await graph.insert(key) validate_graph(graph) assert all(key in graph.db._db for key in key_order) for key in key_order: await graph.search(key) await graph.delete(key) with pytest.raises(NotFound): await graph.search(key) validate_graph(graph)
async def do_test_skip_graph_iteration(raw_keys, start, end): ordered_keys = tuple(sorted(set(raw_keys))) if end is None: left_index = bisect.bisect_left(ordered_keys, start) right_index = None elif end >= start: left_index = bisect.bisect_left(ordered_keys, start) right_index = bisect.bisect_left(ordered_keys, end) elif end < start: left_index = bisect.bisect_right(ordered_keys, end) right_index = bisect.bisect_right(ordered_keys, start) else: raise Exception("Invariant") graph = LocalGraph(SGNode(ordered_keys[0])) for key in set(raw_keys).difference({ordered_keys[0]}): await graph.insert(key) if end is None or end >= start: expected_keys = ordered_keys[left_index:right_index] else: expected_keys = tuple(reversed(ordered_keys[left_index:right_index])) actual_items = tuple([(key, node) async for key, node in graph.iter_items(start, end)]) actual_keys = tuple([key async for key in graph.iter_keys(start, end)]) actual_values = tuple( [node async for node in graph.iter_values(start, end)]) assert len(actual_items) == len(expected_keys) assert len(actual_keys) == len(expected_keys) assert len(actual_values) == len(expected_keys) for key, (actual_key, node) in zip(expected_keys, actual_items): assert actual_key == key assert node.key == key for key, actual_key in zip(expected_keys, actual_keys): assert actual_key == key for key, node in zip(expected_keys, actual_values): assert node.key == key
async def test_client_send_graph_node(alice_and_bob_clients): alice, bob = alice_and_bob_clients async with bob.message_dispatcher.subscribe(GraphNode) as subscription: await alice.send_graph_node( bob.local_node, request_id=1234, sg_node=SGNode(5, (3, 2), (7, 9)), ) with trio.fail_after(1): message = await subscription.receive() assert message.node == alice.local_node payload = message.payload assert isinstance(payload, GraphNode) assert payload.request_id == 1234 node = payload.node.to_sg_node() assert node.key == 5 assert node.neighbors[0] == [3, 2] assert node.neighbors[1] == [7, 9]
async def test_graph_forward_iteration(): graph = LocalGraph(SGNode(1)) for key in (3, 5): await graph.insert(key) assert tuple([key async for key in graph.iter_keys()]) == (1, 3, 5) assert tuple([key async for key in graph.iter_keys(start=1)]) == (1, 3, 5) assert tuple([key async for key in graph.iter_keys(start=2)]) == (3, 5) assert tuple([key async for key in graph.iter_keys(start=3)]) == (3, 5) assert tuple([key async for key in graph.iter_keys(start=4)]) == (5, ) assert tuple([key async for key in graph.iter_keys(start=5)]) == (5, ) assert tuple([key async for key in graph.iter_keys(start=6)]) == () assert tuple([key async for key in graph.iter_keys(end=10)]) == (1, 3, 5) assert tuple([key async for key in graph.iter_keys(start=1, end=10) ]) == (1, 3, 5) assert tuple([key async for key in graph.iter_keys(start=1, end=5)]) == (1, 3) assert tuple([key async for key in graph.iter_keys(start=1, end=4)]) == (1, 3) assert tuple([key async for key in graph.iter_keys(start=1, end=3)]) == (1, ) assert tuple([key async for key in graph.iter_keys(start=2, end=3)]) == ()
async def _initialize_network_graph(self) -> GraphAPI: async def do_get_introduction( node: Node, send_channel: trio.abc.SendChannel[Tuple[ SGNodeAPI, TraversalResults]], # noqa: E501 ) -> None: try: with trio.fail_after(INTRODUCTION_TIMEOUT): candidates = await self.network.get_introduction(node) except trio.TooSlowError: self.logger.debug("Timeout getting introduction from %s", node) return self.logger.debug("Got %d introductions from %s", len(candidates), node) async with send_channel: for candidate in candidates: import time start_at = time.monotonic() try: with trio.fail_after(TRAVERSAL_TIMEOUT): result = await get_traversal_result( self.network, self.graph_db, candidate, max_traversal_distance=10, ) except trio.TooSlowError: self.logger.error( "%s: Traversal timeout: %s", self.client.local_node, candidate, ) return else: end_at = time.monotonic() self.logger.info( "%s: Traversal finished in %s seconds", self.client.local_node, end_at - start_at, ) await send_channel.send(result) while self.manager.is_running: if self.config.can_initialize_network_skip_graph: # Use a probabalistic mechanism here so that multiple # bootnodes coming online at the same time are unlikely to # try and concurrently seed the network with content. if secrets.randbits(256) >= self.client.local_node_id: for key in self.content_manager.iter_content_keys(): seed_node = SGNode(content_key_to_graph_key(key)) break else: continue self.logger.info("%s: Seeding network graph", self.client.local_node) self.graph_db.set(seed_node.key, seed_node) return NetworkGraph(self.graph_db, seed_node, self.network) random_content_id = NodeID(secrets.randbits(256)) candidates = await self.network.iterative_lookup(random_content_id) if not candidates: self.logger.debug("No candidates for introduction") await trio.sleep(5) continue send_channel, receive_channel = trio.open_memory_channel[Tuple[ SGNodeAPI, TraversalResults]](256) # noqa: E501 async with trio.open_nursery() as nursery: async with send_channel: for node in candidates: nursery.start_soon(do_get_introduction, node, send_channel.clone()) async with receive_channel: introduction_results = tuple( [result async for result in receive_channel]) if not introduction_results: self.logger.debug("Received no introductions. sleeping....") await trio.sleep(5) continue best_result = sorted(introduction_results, key=lambda result: result.score, reverse=True)[0] if best_result.score > 0.0: self.logger.info( "%s: Network SkipGraph initialized: node=%s score=%s", self.client.local_node, best_result.node, best_result.score, ) return NetworkGraph(self.graph_db, best_result.node, self.network) else: self.logger.info( "Failed to initialize Skip Graph. All introductions were faulty: %s", tuple(str(result) for result in introduction_results), ) await trio.sleep(5)
def test_single_node_with_only_left_neighbor(): node = SGNode(1, neighbors_left=[0]) assert node.max_level == 1 assert node.get_neighbor(0, LEFT) == 0 assert node.get_neighbor(0, RIGHT) is None for level in range(1, 256): assert node.get_neighbor(level, LEFT) is None assert node.get_neighbor(level, RIGHT) is None assert tuple(node.iter_down_levels(0, LEFT)) == ((0, 0), ) assert tuple(node.iter_down_levels(0, RIGHT)) == ((0, None), ) assert tuple(node.iter_down_levels(1, LEFT)) == ((1, None), (0, 0)) assert tuple(node.iter_down_levels(1, RIGHT)) == ((1, None), (0, None)) assert tuple(node.iter_neighbors()) == ((0, LEFT, 0), )
def test_single_node_with_both_neighbors(): node = SGNode(4, neighbors_left=[2, 0], neighbors_right=[8, 12, 28]) assert node.max_level == 3 assert node.get_neighbor(0, LEFT) == 2 assert node.get_neighbor(0, RIGHT) == 8 assert node.get_neighbor(1, LEFT) == 0 assert node.get_neighbor(1, RIGHT) == 12 assert node.get_neighbor(2, LEFT) is None assert node.get_neighbor(2, RIGHT) == 28 for level in range(3, 256): assert node.get_neighbor(level, LEFT) is None assert node.get_neighbor(level, RIGHT) is None assert tuple(node.iter_down_levels(0, LEFT)) == ((0, 2), ) assert tuple(node.iter_down_levels(0, RIGHT)) == ((0, 8), ) assert tuple(node.iter_down_levels(1, LEFT)) == ((1, 0), (0, 2)) assert tuple(node.iter_down_levels(1, RIGHT)) == ((1, 12), (0, 8)) assert tuple(node.iter_down_levels(2, LEFT)) == ((2, None), (1, 0), (0, 2)) assert tuple(node.iter_down_levels(2, RIGHT)) == ((2, 28), (1, 12), (0, 8)) assert tuple(node.iter_down_levels(3, LEFT)) == ((3, None), (2, None), (1, 0), (0, 2)) assert tuple(node.iter_down_levels(3, RIGHT)) == ((3, None), (2, 28), (1, 12), (0, 8)) assert tuple(node.iter_neighbors()) == ( (0, LEFT, 2), (0, RIGHT, 8), (1, LEFT, 0), (1, RIGHT, 12), (2, RIGHT, 28), )
def test_node_membership_vector(key, level): node = SGNode(key) membership_vector = node.get_membership_at_level(level) assert membership_vector.bit_length() <= level
def test_single_node_neighbor_setting(): node = SGNode(4) assert node.max_level == 0 assert node.get_neighbor(0, LEFT) is None assert node.get_neighbor(0, RIGHT) is None node.set_neighbor(0, LEFT, 2) assert node.max_level == 1 assert node.get_neighbor(0, LEFT) == 2 assert node.get_neighbor(0, RIGHT) is None node.set_neighbor(1, LEFT, 0) assert node.max_level == 2 assert node.get_neighbor(0, LEFT) == 2 assert node.get_neighbor(1, LEFT) == 0 assert node.get_neighbor(0, RIGHT) is None # should be an error to set a neighbor above the topmost non-null neighbor. with pytest.raises(ValidationError): node.set_neighbor(1, RIGHT, 12) with pytest.raises(ValidationError): node.set_neighbor(2, LEFT, 28) node.set_neighbor(0, RIGHT, 8) with pytest.raises(ValidationError): node.set_neighbor(2, RIGHT, 28) assert node.max_level == 2 assert node.get_neighbor(0, RIGHT) == 8 assert node.get_neighbor(1, RIGHT) is None node.set_neighbor(1, RIGHT, 12) node.set_neighbor(2, RIGHT, 28) assert node.max_level == 3 assert node.get_neighbor(0, RIGHT) == 8 assert node.get_neighbor(1, RIGHT) == 12 assert node.get_neighbor(2, RIGHT) == 28 assert node.get_neighbor(3, RIGHT) is None assert node.get_neighbor(4, RIGHT) is None assert node.get_neighbor(5, RIGHT) is None assert node.get_neighbor(6, RIGHT) is None node.set_neighbor(3, RIGHT, None) node.set_neighbor(4, RIGHT, None) node.set_neighbor(5, RIGHT, None) node.set_neighbor(6, RIGHT, None) assert node.max_level == 3 assert node.get_neighbor(0, RIGHT) == 8 assert node.get_neighbor(1, RIGHT) == 12 assert node.get_neighbor(2, RIGHT) == 28 assert node.get_neighbor(3, RIGHT) is None assert node.get_neighbor(4, RIGHT) is None assert node.get_neighbor(5, RIGHT) is None assert node.get_neighbor(6, RIGHT) is None with pytest.raises(ValidationError): node.set_neighbor(1, RIGHT, None) with pytest.raises(ValidationError): node.set_neighbor(0, RIGHT, None) node.set_neighbor(2, RIGHT, None) assert node.max_level == 2 assert node.get_neighbor(0, RIGHT) == 8 assert node.get_neighbor(1, RIGHT) == 12 assert node.get_neighbor(2, RIGHT) is None assert node.get_neighbor(3, RIGHT) is None assert node.get_neighbor(4, RIGHT) is None assert node.get_neighbor(5, RIGHT) is None assert node.get_neighbor(6, RIGHT) is None with pytest.raises(ValidationError): node.set_neighbor(0, RIGHT, None) node.set_neighbor(1, RIGHT, None) assert node.max_level == 2 assert node.get_neighbor(0, RIGHT) == 8 assert node.get_neighbor(1, RIGHT) is None assert node.get_neighbor(2, RIGHT) is None assert node.get_neighbor(3, RIGHT) is None assert node.get_neighbor(4, RIGHT) is None assert node.get_neighbor(5, RIGHT) is None assert node.get_neighbor(6, RIGHT) is None node.set_neighbor(0, RIGHT, 28) assert node.max_level == 2 assert node.get_neighbor(0, RIGHT) == 28 assert node.get_neighbor(1, RIGHT) is None assert node.get_neighbor(2, RIGHT) is None assert node.get_neighbor(3, RIGHT) is None assert node.get_neighbor(4, RIGHT) is None assert node.get_neighbor(5, RIGHT) is None assert node.get_neighbor(6, RIGHT) is None node.set_neighbor(0, RIGHT, None) assert node.max_level == 2 assert node.get_neighbor(0, RIGHT) is None assert node.get_neighbor(1, RIGHT) is None assert node.get_neighbor(2, RIGHT) is None assert node.get_neighbor(3, RIGHT) is None assert node.get_neighbor(4, RIGHT) is None assert node.get_neighbor(5, RIGHT) is None assert node.get_neighbor(6, RIGHT) is None