def crawl(self):
def gather_results(result):
for proto in result:
n = objects.Node()
try:
n.ParseFromString(proto)
node = Node(n.guid, n.ip, n.port, n.signedPublicKey,
n.vendor)
self.nodes[(node.ip, node.port)] = node
except Exception:
pass
def start_crawl(results):
for node, result in results.items():
if not result[0]:
del self.nodes[(node.ip, node.port)]
node = Node(digest(random.getrandbits(255)))
nearest = self.kserver.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest,
100, 4)
spider.find().addCallback(gather_results)
ds = {}
for bucket in self.kserver.protocol.router.buckets:
for node in bucket.getNodes():
self.nodes[(node.ip, node.port)] = node
for node in self.nodes.values():
if node.id != this_node.id:
ds[node] = self.kserver.protocol.callPing(node)
deferredDict(ds).addCallback(start_crawl)
def crawl(self):
def gather_results(result):
for proto in result:
n = objects.Node()
try:
n.ParseFromString(proto)
node = Node(n.guid, n.ip, n.port, n.signedPublicKey, n.vendor)
self.nodes[(node.ip, node.port)] = node
except Exception:
pass
def start_crawl(results):
for node, result in results.items():
if not result[0]:
del self.nodes[(node.ip, node.port)]
node = Node(digest(random.getrandbits(255)))
nearest = self.kserver.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4)
spider.find().addCallback(gather_results)
ds = {}
for bucket in self.kserver.protocol.router.buckets:
for node in bucket.getNodes():
self.nodes[(node.ip, node.port)] = node
for node in self.nodes.values():
if node.id != this_node.id:
ds[node] = self.kserver.protocol.callPing(node)
deferredDict(ds).addCallback(start_crawl)
def bootstrap(self, addrs, deferred=None):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
self.log.info("bootstrapping with %s addresses, finding neighbors..." % len(addrs))
if deferred is None:
d = defer.Deferred()
else:
d = deferred
def initTable(results):
response = False
potential_relay_nodes = []
for addr, result in results.items():
if result[0]:
response = True
n = objects.Node()
try:
n.ParseFromString(result[1][0])
h = nacl.hash.sha512(n.publicKey)
hash_pow = h[40:]
if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
node = Node(n.guid, addr[0], addr[1], n.publicKey,
None if not n.HasField("relayAddress") else
(n.relayAddress.ip, n.relayAddress.port),
n.natType,
n.vendor)
self.protocol.router.addContact(node)
if n.natType == objects.FULL_CONE:
potential_relay_nodes.append((addr[0], addr[1]))
except Exception:
self.log.warning("bootstrap node returned invalid GUID")
if not response:
if self.protocol.multiplexer.testnet:
self.bootstrap(self.querySeed(SEEDS_TESTNET), d)
else:
self.bootstrap(self.querySeed(SEEDS), d)
return
if len(potential_relay_nodes) > 0 and self.node.nat_type != objects.FULL_CONE:
shuffle(potential_relay_nodes)
self.node.relay_node = potential_relay_nodes[0]
d.callback(True)
ds = {}
for addr in addrs:
if addr != (self.node.ip, self.node.port):
ds[addr] = self.protocol.ping(Node(digest("null"), addr[0], addr[1], nat_type=objects.FULL_CONE))
deferredDict(ds).addCallback(initTable)
return d
def _find(self, rpcmethod):
"""
Get either a value or list of nodes.
Args:
rpcmethod: The protocol's callFindValue or callFindNode.
The process:
1. calls find_* to current ALPHA nearest not already queried nodes,
adding results to current nearest list of k nodes.
2. current nearest list needs to keep track of who has been queried already
sort by nearest, keep KSIZE
3. if list is same as last time, next call should be to everyone not
yet queried
4. repeat, unless nearest list has all been queried, then ur done
"""
self.log.debug("crawling with nearest: %s" % str(tuple(self.nearest)))
count = self.alpha
if self.nearest.getIDs() == self.lastIDsCrawled:
self.log.debug("last iteration same as current - checking all in list now")
count = len(self.nearest)
self.lastIDsCrawled = self.nearest.getIDs()
ds = {}
for peer in self.nearest.getUncontacted()[:count]:
ds[peer.id] = rpcmethod(peer, self.node)
self.nearest.markContacted(peer)
return deferredDict(ds).addCallback(self._nodesFound)
def _find(self, rpcmethod):
"""
Get either a value or list of nodes.
Args:
rpcmethod: The protocol's callFindValue or callFindNode.
The process:
1. calls find_* to current ALPHA nearest not already queried nodes,
adding results to current nearest list of k nodes.
2. current nearest list needs to keep track of who has been queried already
sort by nearest, keep KSIZE
3. if list is same as last time, next call should be to everyone not
yet queried
4. repeat, unless nearest list has all been queried, then ur done
"""
self.log.debug("crawling with nearest: %s" % str(tuple(self.nearest)))
count = self.alpha
if self.nearest.getIDs() == self.lastIDsCrawled:
self.log.debug("last iteration same as current - checking all in list now")
count = len(self.nearest)
self.lastIDsCrawled = self.nearest.getIDs()
ds = {}
for peer in self.nearest.getUncontacted()[:count]:
ds[peer.id] = rpcmethod(peer, self.node)
self.nearest.markContacted(peer)
return deferredDict(ds).addCallback(self._nodesFound)
def test_defferedDict(self):
def checkValues(d):
self.assertTrue(type(d) == dict)
self.assertTrue(len(d) == 3)
def checkEmpty(d):
self.assertTrue(type(d) == dict)
self.assertTrue(len(d) == 0)
ds = {}
ds["key1"] = defer.Deferred()
ds["key2"] = defer.Deferred()
ds["key3"] = defer.Deferred()
d = deferredDict(ds).addCallback(checkValues)
for v in ds.itervalues():
v.callback("True")
d = deferredDict({}).addCallback(checkEmpty)
def test_defferedDict(self):
def checkValues(d):
self.assertTrue(isinstance(d, dict))
self.assertTrue(len(d) == 3)
def checkEmpty(d):
self.assertTrue(isinstance(d, dict))
self.assertTrue(len(d) == 0)
ds = {}
ds["key1"] = defer.Deferred()
ds["key2"] = defer.Deferred()
ds["key3"] = defer.Deferred()
deferredDict(ds).addCallback(checkValues)
for v in ds.itervalues():
v.callback("True")
deferredDict({}).addCallback(checkEmpty)
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
self.log.info("bootstrapping with %s addresses, finding neighbors..." % len(addrs))
d = defer.Deferred()
def initTable(results):
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
hash_pow = h[64:128]
if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
self.protocol.router.addContact(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except Exception:
self.log.warning("bootstrap node returned invalid GUID")
d.callback(True)
ds = {}
for addr in addrs:
if addr != (self.node.ip, self.node.port):
ds[addr] = self.protocol.ping((addr[0], addr[1]))
deferredDict(ds).addCallback(initTable)
return d
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
self.log.info("bootstrapping with %s addresses, finding neighbors..." %
len(addrs))
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) -
32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
hash_pow = h[64:128]
if int(hash_pow[:6], 16) >= 50 or hexlify(
n.guid) != h[:40]:
raise Exception('Invalid GUID')
nodes.append(
Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except Exception:
self.log.warning(
"bootstrap node returned invalid GUID")
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
return spider.find()
ds = {}
for addr in addrs:
if addr != (self.node.ip, self.node.port):
ds[addr] = self.protocol.ping((addr[0], addr[1]))
return deferredDict(ds).addCallback(initTable)
def parse_response(moderators):
if moderators is None:
return None
def parse_profiles(responses):
for k, v in responses.items():
if v is None:
del responses[k]
return responses
ds = {}
for mod in moderators:
try:
val = objects.Value()
val.ParseFromString(mod)
n = objects.Node()
n.ParseFromString(val.serializedData)
ds[val.serializedData] = self.get_profile(node.Node(n.guid, n.ip, n.port, n.signedPublicKey))
except Exception:
pass
return deferredDict(ds).addCallback(parse_profiles)
def bootstrap(self, addrs):
"""
Bootstrap the server by connecting to other known nodes in the network.
Args:
addrs: A `list` of (ip, port) `tuple` pairs. Note that only IP addresses
are acceptable - hostnames will cause an error.
"""
# if the transport hasn't been initialized yet, wait a second
if self.protocol.multiplexer.transport is None:
return task.deferLater(reactor, 1, self.bootstrap, addrs)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
pow = h[64:128]
if int(pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except:
self.log.msg("Bootstrap node returned invalid GUID")
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
ds = {}
for addr in addrs:
ds[addr] = self.protocol.ping((addr[0], addr[1]))
return deferredDict(ds).addCallback(initTable)
