def delete(self, keyword, key, signature):
"""
Delete the given key/value pair from the keyword dictionary on the network.
To delete you must provide a signature covering the key that you wish to
delete. It will be verified against the public key stored in the value. We
use our ksize as alpha to make sure we reach as many nodes storing our value
as possible.
Args:
keyword: the `string` keyword where the data being deleted is stored.
key: the 20 byte hash of the data.
signature: a signature covering the key.
"""
self.log.debug("deleting '%s':'%s' from the network" % (keyword.encode("hex"), key.encode("hex")))
dkey = digest(keyword)
def delete(nodes):
self.log.debug("deleting '%s' on %s" % (key.encode("hex"), [str(i) for i in nodes]))
ds = [self.protocol.callDelete(node, dkey, key, signature) for node in nodes]
if self.storage.getSpecific(dkey, key) is not None:
self.storage.delete(dkey, key)
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("there are no known neighbors to delete key %s" % key.encode("hex"))
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.ksize)
return spider.find().addCallback(delete)
def resolve(self, guid):
"""
Given a guid return a `Node` object containing its ip and port or none if it's
not found.
Args:
guid: the 20 raw bytes representing the guid.
"""
node_to_find = Node(guid)
def check_for_node(nodes):
for node in nodes:
if node.id == node_to_find.id:
return node
return None
index = self.protocol.router.getBucketFor(node_to_find)
nodes = self.protocol.router.buckets[index].getNodes()
for node in nodes:
if node.id == node_to_find.id:
return defer.succeed(node)
nearest = self.protocol.router.findNeighbors(node_to_find)
if len(nearest) == 0:
self.log.warning("there are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
return defer.succeed(None)
spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
return spider.find().addCallback(check_for_node)
def start_crawl(results):
for node, result in results.items():
if not result[0]:
del self.nodes[(node.ip, node.port)]
node = Node(digest(random.getrandbits(255)))
nearest = self.kserver.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.kserver.protocol, node, nearest, 100, 4)
spider.find().addCallback(gather_results)
def test_find(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
spider.find()
self.clock.advance(constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for rid in self.protocol.getRefreshIDs():
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
ds.append(spider.find())
def republishKeys(_):
for bucket in self.protocol.router.buckets:
for node in bucket.nodes.values():
self.protocol.transferKeyValues(node)
return defer.gatherResults(ds).addCallback(republishKeys)
def test_nodesFound(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
response = (True, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node1.getProto() in node_protos)
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
response = (False, (self.node1.getProto().SerializeToString(), self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
def get_node(self, guid):
node_to_find = Node(guid)
def check_for_node(nodes):
for node in nodes:
if node.id == node_to_find.id:
return node
return None
index = self.protocol.router.getBucketFor(node_to_find)
nodes = self.protocol.router.buckets[index].getNodes()
for node in nodes:
if node.id == node_to_find.id:
return node
nearest = self.protocol.router.findNeighbors(node_to_find)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
return defer.succeed(None)
spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha)
spider.find().addCallback(check_for_node)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) - 32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
hash_pow = h[64:128]
if int(hash_pow[:6], 16) >= 50 or hexlify(n.guid) != h[:40]:
raise Exception('Invalid GUID')
nodes.append(Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except Exception:
self.log.warning("bootstrap node returned invalid GUID")
spider = NodeSpiderCrawl(self.protocol, self.node, nodes, self.ksize, self.alpha)
return spider.find()
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for id in self.protocol.getRefreshIDs():
node = Node(id)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for keyword in self.storage.iterkeys():
for k, v in self.storage.iteritems(keyword):
if self.storage.get_ttl(keyword, k) < 601200:
ds.append(self.set(keyword, k, v))
return defer.gatherResults(ds).addCallback(republishKeys)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
refresh_ids = self.protocol.getRefreshIDs()
refresh_ids.append(digest(random.getrandbits(255))) # random node so we get more diversity
for rid in refresh_ids:
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
ds.append(spider.find())
def republishKeys(_):
self.log.debug("Republishing key/values...")
neighbors = self.protocol.router.findNeighbors(self.node, exclude=self.node)
for node in neighbors:
self.protocol.transferKeyValues(node)
return defer.gatherResults(ds).addCallback(republishKeys)
def set(self, keyword, key, value, ttl=604800):
"""
Set the given key/value tuple at the hash of the given keyword.
All values stored in the DHT are stored as dictionaries of key/value
pairs. If a value already exists for a given keyword, the new key/value
pair will be appended to the dictionary.
Args:
keyword: The keyword to use. Should be hashed with hash160 before
passing it in here.
key: the 20 byte hash of the data.
value: a serialized `protos.objects.Node` object which serves as a
pointer to the node storing the data.
Return: True if at least one peer responded. False if the store rpc
completely failed.
"""
if len(keyword) != 20:
return defer.succeed(False)
self.log.debug("setting '%s' on network" % keyword.encode("hex"))
def store(nodes):
self.log.debug("setting '%s' on %s" % (keyword.encode("hex"), [str(i) for i in nodes]))
ds = [self.protocol.callStore(node, keyword, key, value, ttl) for node in nodes]
keynode = Node(keyword)
if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
self.storage[keyword] = (key, value, ttl)
self.log.debug("got a store request from %s, storing value" % str(self.node))
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(keyword)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("there are no known neighbors to set keyword %s" % keyword.encode("hex"))
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def delete(self, keyword, key, signature):
"""
Delete the given key/value pair from the keyword dictionary on the network.
To delete you must provide a signature covering the key that you wish to
delete. It will be verified against the public key stored in the value. We
use our ksize as alpha to make sure we reach as many nodes storing our value
as possible.
Args:
keyword: the `string` keyword where the data being deleted is stored.
key: the 20 byte hash of the data.
signature: a signature covering the key.
"""
self.log.debug("deleting '%s':'%s' from the network" %
(keyword.encode("hex"), key.encode("hex")))
dkey = digest(keyword)
def delete(nodes):
self.log.debug("deleting '%s' on %s" %
(key.encode("hex"), [str(i) for i in nodes]))
ds = [
self.protocol.callDelete(node, dkey, key, signature)
for node in nodes
]
if self.storage.getSpecific(dkey, key) is not None:
self.storage.delete(dkey, key)
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("there are no known neighbors to delete key %s" %
key.encode("hex"))
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.ksize)
return spider.find().addCallback(delete)
def set(self, keyword, key, value, ttl=604800):
"""
Set the given key/value tuple at the hash of the given keyword.
All values stored in the DHT are stored as dictionaries of key/value
pairs. If a value already exists for a given keyword, the new key/value
pair will be appended to the dictionary.
Args:
keyword: The keyword to use. Should be hashed with hash160 before
passing it in here.
key: the 20 byte hash of the data.
value: a serialized `protos.objects.Node` object which serves as a
pointer to the node storing the data.
Return: True if at least one peer responded. False if the store rpc
completely failed.
"""
if len(keyword) != 20:
return defer.succeed(False)
self.log.debug("setting '%s' on network" % keyword.encode("hex"))
def store(nodes):
self.log.debug("setting '%s' on %s" % (keyword.encode("hex"), [str(i) for i in nodes]))
ds = [self.protocol.callStore(node, keyword, key, value, ttl) for node in nodes]
keynode = Node(keyword)
if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
self.storage[keyword] = (key, value, ttl)
self.log.debug("got a store request from %s, storing value" % str(self.node))
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(keyword)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("there are no known neighbors to set keyword %s" % keyword.encode("hex"))
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
refresh_ids = self.protocol.getRefreshIDs()
refresh_ids.append(digest(
random.getrandbits(255))) # random node so we get more diversity
for rid in refresh_ids:
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
ds.append(spider.find())
def republishKeys(_):
for bucket in self.protocol.router.buckets:
for node in bucket.nodes.values():
self.protocol.transferKeyValues(node)
return defer.gatherResults(ds).addCallback(republishKeys)
def refreshTable(self):
"""
Refresh buckets that haven't had any lookups in the last hour
(per section 2.3 of the paper).
"""
ds = []
for rid in self.protocol.getRefreshIDs():
node = Node(rid)
nearest = self.protocol.router.findNeighbors(node, self.alpha)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize,
self.alpha)
ds.append(spider.find())
def republishKeys(_):
ds = []
# Republish keys older than one hour
for keyword in self.storage.iterkeys():
for k, v in self.storage.iteritems(keyword):
if self.storage.get_ttl(keyword, k) < 601200:
ds.append(self.set(keyword, k, v))
return defer.gatherResults(ds).addCallback(republishKeys)
def set(self, keyword, key, value):
"""
Set the given key/value tuple at the hash of the given keyword.
All values stored in the DHT are stored as dictionaries of key/value
pairs. If a value already exists for a given keyword, the new key/value
pair will be appended to the dictionary.
Args:
keyword: a `string` keyword. The SHA1 hash of which will be used as
the key when inserting in the DHT.
key: the 20 byte hash of the data.
value: a serialized `protos.objects.Node` object which serves as a
pointer to the node storing the data.
Return: True if at least one peer responded. False if the store rpc
completely failed.
"""
self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
dkey = digest(keyword)
def store(nodes):
self.log.info("setting '%s' on %s" % (keyword, [str(i) for i in nodes]))
ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]
keynode = Node(dkey)
if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
self.storage[dkey] = (key, value)
self.log.debug("got a store request from %s, storing value" % str(self.node))
return defer.DeferredList(ds).addCallback(_anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def set(self, keyword, key, value):
"""
Set the given key/value tuple at the hash of the given keyword.
All values stored in the DHT are stored as dictionaries of key/value
pairs. If a value already exists for a given keyword, the new key/value
pair will be appended to the dictionary.
Args:
keyword: a `string` keyword. The SHA1 hash of which will be used as
the key when inserting in the DHT.
key: the 20 byte hash of the data.
value: a serialized `protos.objects.Node` object which serves as a
pointer to the node storing the data.
Return: True if at least one peer responded. False if the store rpc
completely failed.
"""
self.log.debug("setting '%s' = '%s':'%s' on network" % (keyword, hexlify(key), hexlify(value)))
dkey = digest(keyword)
def store(nodes):
self.log.info("setting '%s' on %s" % (keyword, map(str, nodes)))
ds = [self.protocol.callStore(node, dkey, key, value) for node in nodes]
keynode = Node(dkey)
if self.node.distanceTo(keynode) < max([n.distanceTo(keynode) for n in nodes]):
self.storage[dkey] = (key, value)
self.log.debug("got a store request from %s, storing value" % str(self.node))
return defer.DeferredList(ds).addCallback(self._anyRespondSuccess)
node = Node(dkey)
nearest = self.protocol.router.findNeighbors(node)
if len(nearest) == 0:
self.log.warning("There are no known neighbors to set key %s" % key)
return defer.succeed(False)
spider = NodeSpiderCrawl(self.protocol, node, nearest, self.ksize, self.alpha)
return spider.find().addCallback(store)
def resolve(self, guid):
"""
Given a guid return a `Node` object containing its ip and port or none if it's
not found.
Args:
guid: the 20 raw bytes representing the guid.
"""
self.log.debug("crawling dht to find IP for %s" % guid.encode("hex"))
node_to_find = Node(guid)
for connection in self.protocol.multiplexer.values():
if connection.handler.node is not None and connection.handler.node.id == node_to_find.id:
self.log.debug("%s successfully resolved as %s" % (guid.encode("hex"), connection.handler.node))
return defer.succeed(connection.handler.node)
def check_for_node(nodes):
for node in nodes:
if node.id == node_to_find.id:
self.log.debug("%s successfully resolved as %s" % (guid.encode("hex"), node))
return node
self.log.debug("%s was not found in the dht" % guid.encode("hex"))
return None
index = self.protocol.router.getBucketFor(node_to_find)
nodes = self.protocol.router.buckets[index].getNodes()
for node in nodes:
if node.id == node_to_find.id:
self.log.debug("%s successfully resolved as %s" % (guid.encode("hex"), node))
return defer.succeed(node)
nearest = self.protocol.router.findNeighbors(node_to_find)
if len(nearest) == 0:
self.log.warning("there are no known neighbors to find node %s" % node_to_find.id.encode("hex"))
return defer.succeed(None)
spider = NodeSpiderCrawl(self.protocol, node_to_find, nearest, self.ksize, self.alpha, True)
return spider.find().addCallback(check_for_node)
def initTable(results):
nodes = []
for addr, result in results.items():
if result[0]:
n = objects.Node()
try:
n.ParseFromString(result[1][0])
pubkey = n.signedPublicKey[len(n.signedPublicKey) -
32:]
verify_key = nacl.signing.VerifyKey(pubkey)
verify_key.verify(n.signedPublicKey)
h = nacl.hash.sha512(n.signedPublicKey)
hash_pow = h[64:128]
if int(hash_pow[:6], 16) >= 50 or hexlify(
n.guid) != h[:40]:
raise Exception('Invalid GUID')
nodes.append(
Node(n.guid, addr[0], addr[1], n.signedPublicKey))
except Exception:
self.log.msg("Bootstrap node returned invalid GUID")
spider = NodeSpiderCrawl(self.protocol, self.node, nodes,
self.ksize, self.alpha)
return spider.find()
def test_nodesFound(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = NodeSpiderCrawl(self.protocol, node, nearest, 20, 3)
response = (True, (self.node1.getProto().SerializeToString(),
self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
response = (True, (self.node1.getProto().SerializeToString(),
self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node1.getProto() in node_protos)
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)
response = (False, (self.node1.getProto().SerializeToString(),
self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
nodes = spider._nodesFound(responses)
node_protos = []
for n in nodes:
node_protos.append(n.getProto())
self.assertTrue(self.node2.getProto() in node_protos)
self.assertTrue(self.node3.getProto() in node_protos)