class SpiderCrawl(object): """ Crawl the network and look for given 160-bit keys. """ def __init__(self, protocol, node, peers, ksize, alpha): """ Create a new C{SpiderCrawl}er. Args: protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance. node: A :class:`~kademlia.node.Node` representing the key we're looking for peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network ksize: The value for k based on the paper alpha: The value for alpha based on the paper """ self.protocol = protocol self.ksize = ksize self.alpha = alpha self.node = node self.nearest = NodeHeap(self.node, self.ksize) self.lastIDsCrawled = [] self.log = Logger(system=self) self.log.info("creating spider with peers: %s" % peers) self.nearest.push(peers) def onError(self, err): self.log.error(repr(err)) return err def _find(self, rpcmethod): """ Get either a value or list of nodes. Args: rpcmethod: The protocol's callfindValue or callFindNode. The process: 1. calls find_* to current ALPHA nearest not already queried nodes, adding results to current nearest list of k nodes. 2. current nearest list needs to keep track of who has been queried already sort by nearest, keep KSIZE 3. if list is same as last time, next call should be to everyone not yet queried 4. repeat, unless nearest list has all been queried, then ur done """ self.log.info("crawling with nearest: %s" % str(tuple(self.nearest))) count = self.alpha if self.nearest.getIDs() == self.lastIDsCrawled: self.log.info("last iteration same as current - checking all in list now") count = len(self.nearest) self.lastIDsCrawled = self.nearest.getIDs() ds = {} for peer in self.nearest.getUncontacted()[:count]: ds[peer.id] = rpcmethod(peer, self.node) self.nearest.markContacted(peer) d = deferredDict(ds) d.addCallback(self._nodesFound) d.addErrback(self.onError) return d
def __init__(self, protocol, node, peers, ksize, alpha): """ Create a new C{SpiderCrawl}er. Args: protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance. node: A :class:`~kademlia.node.Node` representing the key we're looking for peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network ksize: The value for k based on the paper alpha: The value for alpha based on the paper """ self.protocol = protocol self.ksize = ksize self.alpha = alpha self.node = node self.nearest = NodeHeap(self.node, self.ksize) self.lastIDsCrawled = [] self.log = Logger(system=self) self.log.info("creating spider with peers: %s" % peers) self.nearest.push(peers)