def test_iteration(self): heap = NodeHeap(mknode(intid=0), 5) nodes = [mknode(intid=x) for x in range(10)] for index, node in enumerate(nodes): heap.push(node) for index, node in enumerate(heap): self.assertEqual(index, node.long_id) self.assertTrue(index < 5)
def test_max_size(self): node = NodeHeap(mknode(intid=0), 3) self.assertEqual(0, len(node)) for digit in range(10): node.push(mknode(intid=digit)) self.assertEqual(3, len(node)) self.assertEqual(3, len(list(node)))
class ValueSpiderCrawl(SpiderCrawl): def __init__(self, protocol, node, peers, ksize, alpha): SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha) # keep track of the single nearest node without value - per # section 2.3 so we can set the key there if found self.nearest_without_value = NodeHeap(self.node, 1) async def find(self): """ Find either the closest nodes or the value requested. """ return await self._find(self.protocol.call_find_value) async def _nodes_found(self, responses): """ Handle the result of an iteration in _find. """ toremove = [] found_values = [] for peerid, response in responses.items(): response = RPCFindResponse(response) if not response.happened(): toremove.append(peerid) elif response.has_value(): found_values.append(response.get_value()) else: peer = self.nearest.get_node(peerid) self.nearest_without_value.push(peer) self.nearest.push(response.get_node_list()) self.nearest.remove(toremove) if found_values: return await self._handle_found_values(found_values) if self.nearest.have_contacted_all(): # not found! return None return await self.find() async def _handle_found_values(self, values): """ We got some values! Exciting. But let's make sure they're all the same or freak out a little bit. Also, make sure we tell the nearest node that *didn't* have the value to store it. """ value_counts = Counter(values) if len(value_counts) != 1: log.warning("Got multiple values for key %i: %s", self.node.long_id, str(values)) value = value_counts.most_common(1)[0][0] peer = self.nearest_without_value.popleft() if peer: await self.protocol.call_store(peer, self.node.id, value) return value
def test_remove(self): heap = NodeHeap(mknode(intid=0), 5) nodes = [mknode(intid=x) for x in range(10)] for node in nodes: heap.push(node) heap.remove([nodes[0].id, nodes[1].id]) self.assertEqual(len(list(heap)), 5) for index, node in enumerate(heap): self.assertEqual(index + 2, node.long_id) self.assertTrue(index < 5)
def __init__(self, protocol, node, peers, ksize, alpha): """ Create a new C{SpiderCrawl}er. Args: protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance. node: A :class:`~kademlia.node.Node` representing the key we're looking for peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network ksize: The value for k based on the paper alpha: The value for alpha based on the paper """ self.protocol = protocol self.ksize = ksize self.alpha = alpha self.node = node self.nearest = NodeHeap(self.node, self.ksize) self.last_ids_crawled = [] log.info("creating spider with peers: %s", peers) self.nearest.push(peers)
class SpiderCrawl: """ Crawl the network and look for given 160-bit keys. """ def __init__(self, protocol, node, peers, ksize, alpha): """ Create a new C{SpiderCrawl}er. Args: protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance. node: A :class:`~kademlia.node.Node` representing the key we're looking for peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network ksize: The value for k based on the paper alpha: The value for alpha based on the paper """ self.protocol = protocol self.ksize = ksize self.alpha = alpha self.node = node self.nearest = NodeHeap(self.node, self.ksize) self.last_ids_crawled = [] log.info("creating spider with peers: %s", peers) self.nearest.push(peers) async def _find(self, rpcmethod): """ Get either a value or list of nodes. Args: rpcmethod: The protocol's callfindValue or call_find_node. The process: 1. calls find_* to current ALPHA nearest not already queried nodes, adding results to current nearest list of k nodes. 2. current nearest list needs to keep track of who has been queried already sort by nearest, keep KSIZE 3. if list is same as last time, next call should be to everyone not yet queried 4. repeat, unless nearest list has all been queried, then ur done """ log.info("crawling network with nearest: %s", str(tuple(self.nearest))) count = self.alpha if self.nearest.get_ids() == self.last_ids_crawled: count = len(self.nearest) self.last_ids_crawled = self.nearest.get_ids() dicts = {} for peer in self.nearest.get_uncontacted()[:count]: dicts[peer.id] = rpcmethod(peer, self.node) self.nearest.mark_contacted(peer) found = await gather_dict(dicts) return await self._nodes_found(found) async def _nodes_found(self, responses): raise NotImplementedError
def __init__(self, protocol, node, peers, ksize, alpha): SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha) # keep track of the single nearest node without value - per # section 2.3 so we can set the key there if found self.nearest_without_value = NodeHeap(self.node, 1)