def crawl(self): def gather_results(result): for proto in result: n = objects.Node() try: n.ParseFromString(proto) node = Node(n.guid, n.ip, n.port, n.signedPublicKey, n.vendor) self.nodes[(node.ip, node.port)] = node except Exception: pass def start_crawl(results): for node, result in results.items(): if not result[0]: del self.nodes[(node.ip, node.port)] node = Node(digest(random.getrandbits(255))) nearest = self.kserver.protocol.router.findNeighbors(node) spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4) spider.find().addCallback(gather_results) ds = {} for bucket in self.kserver.protocol.router.buckets: for node in bucket.getNodes(): self.nodes[(node.ip, node.port)] = node for node in self.nodes.values(): if node.id != this_node.id: ds[node] = self.kserver.protocol.callPing(node) deferredDict(ds).addCallback(start_crawl)
def bootstrap(self, addrs, deferred=None): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.multiplexer.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) self.log.info("bootstrapping with %s addresses, finding neighbors..." % len(addrs)) if deferred is None: d = defer.Deferred() else: d = deferred def initTable(results): response = False potential_relay_nodes = [] for addr, result in results.items(): if result[0]: response = True n = objects.Node() try: n.ParseFromString(result[1][0]) h = nacl.hash.sha512(n.publicKey) hash_pow = h[40:] if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]: raise Exception('Invalid GUID') node = Node(n.guid, addr[0], addr[1], n.publicKey, None if not n.HasField("relayAddress") else (n.relayAddress.ip, n.relayAddress.port), n.natType, n.vendor) self.protocol.router.addContact(node) if n.natType == objects.FULL_CONE: potential_relay_nodes.append((addr[0], addr[1])) except Exception: self.log.warning("bootstrap node returned invalid GUID") if not response: if self.protocol.multiplexer.testnet: self.bootstrap(self.querySeed(SEEDS_TESTNET), d) else: self.bootstrap(self.querySeed(SEEDS), d) return if len(potential_relay_nodes) > 0 and self.node.nat_type != objects.FULL_CONE: shuffle(potential_relay_nodes) self.node.relay_node = potential_relay_nodes[0] d.callback(True) ds = {} for addr in addrs: if addr != (self.node.ip, self.node.port): ds[addr] = self.protocol.ping(Node(digest("null"), addr[0], addr[1], nat_type=objects.FULL_CONE)) deferredDict(ds).addCallback(initTable) return d
def _find(self, rpcmethod): """ Get either a value or list of nodes. Args: rpcmethod: The protocol's callFindValue or callFindNode. The process: 1. calls find_* to current ALPHA nearest not already queried nodes, adding results to current nearest list of k nodes. 2. current nearest list needs to keep track of who has been queried already sort by nearest, keep KSIZE 3. if list is same as last time, next call should be to everyone not yet queried 4. repeat, unless nearest list has all been queried, then ur done """ self.log.debug("crawling with nearest: %s" % str(tuple(self.nearest))) count = self.alpha if self.nearest.getIDs() == self.lastIDsCrawled: self.log.debug("last iteration same as current - checking all in list now") count = len(self.nearest) self.lastIDsCrawled = self.nearest.getIDs() ds = {} for peer in self.nearest.getUncontacted()[:count]: ds[peer.id] = rpcmethod(peer, self.node) self.nearest.markContacted(peer) return deferredDict(ds).addCallback(self._nodesFound)
def test_defferedDict(self): def checkValues(d): self.assertTrue(type(d) == dict) self.assertTrue(len(d) == 3) def checkEmpty(d): self.assertTrue(type(d) == dict) self.assertTrue(len(d) == 0) ds = {} ds["key1"] = defer.Deferred() ds["key2"] = defer.Deferred() ds["key3"] = defer.Deferred() d = deferredDict(ds).addCallback(checkValues) for v in ds.itervalues(): v.callback("True") d = deferredDict({}).addCallback(checkEmpty)
def test_defferedDict(self): def checkValues(d): self.assertTrue(isinstance(d, dict)) self.assertTrue(len(d) == 3) def checkEmpty(d): self.assertTrue(isinstance(d, dict)) self.assertTrue(len(d) == 0) ds = {} ds["key1"] = defer.Deferred() ds["key2"] = defer.Deferred() ds["key3"] = defer.Deferred() deferredDict(ds).addCallback(checkValues) for v in ds.itervalues(): v.callback("True") deferredDict({}).addCallback(checkEmpty)
def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.multiplexer.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) self.log.info("bootstrapping with %s addresses, finding neighbors..." % len(addrs)) d = defer.Deferred() def initTable(results): for addr, result in results.items(): if result[0]: n = objects.Node() try: n.ParseFromString(result[1][0]) pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(n.signedPublicKey) h = nacl.hash.sha512(n.signedPublicKey) hash_pow = h[64:128] if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]: raise Exception('Invalid GUID') self.protocol.router.addContact(Node(n.guid, addr[0], addr[1], n.signedPublicKey)) except Exception: self.log.warning("bootstrap node returned invalid GUID") d.callback(True) ds = {} for addr in addrs: if addr != (self.node.ip, self.node.port): ds[addr] = self.protocol.ping((addr[0], addr[1])) deferredDict(ds).addCallback(initTable) return d
def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.multiplexer.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) self.log.info("bootstrapping with %s addresses, finding neighbors..." % len(addrs)) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: n = objects.Node() try: n.ParseFromString(result[1][0]) pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(n.signedPublicKey) h = nacl.hash.sha512(n.signedPublicKey) hash_pow = h[64:128] if int(hash_pow[:6], 16) >= 50 or hexlify( n.guid) != h[:40]: raise Exception('Invalid GUID') nodes.append( Node(n.guid, addr[0], addr[1], n.signedPublicKey)) except Exception: self.log.warning( "bootstrap node returned invalid GUID") spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: if addr != (self.node.ip, self.node.port): ds[addr] = self.protocol.ping((addr[0], addr[1])) return deferredDict(ds).addCallback(initTable)
def parse_response(moderators): if moderators is None: return None def parse_profiles(responses): for k, v in responses.items(): if v is None: del responses[k] return responses ds = {} for mod in moderators: try: val = objects.Value() val.ParseFromString(mod) n = objects.Node() n.ParseFromString(val.serializedData) ds[val.serializedData] = self.get_profile(node.Node(n.guid, n.ip, n.port, n.signedPublicKey)) except Exception: pass return deferredDict(ds).addCallback(parse_profiles)
def bootstrap(self, addrs): """ Bootstrap the server by connecting to other known nodes in the network. Args: addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses are acceptable - hostnames will cause an error. """ # if the transport hasn't been initialized yet, wait a second if self.protocol.multiplexer.transport is None: return task.deferLater(reactor, 1, self.bootstrap, addrs) def initTable(results): nodes = [] for addr, result in results.items(): if result[0]: n = objects.Node() try: n.ParseFromString(result[1][0]) pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:] verify_key = nacl.signing.VerifyKey(pubkey) verify_key.verify(n.signedPublicKey) h = nacl.hash.sha512(n.signedPublicKey) pow = h[64:128] if int(pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]: raise Exception('Invalid GUID') nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey)) except: self.log.msg("Bootstrap node returned invalid GUID") spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha) return spider.find() ds = {} for addr in addrs: ds[addr] = self.protocol.ping((addr[0], addr[1])) return deferredDict(ds).addCallback(initTable)