def test_iteration(self):
heap = NodeHeap(mknode(intid=0), 5)
nodes = [mknode(intid=x) for x in range(10)]
for index, node in enumerate(nodes):
heap.push(node)
for index, node in enumerate(heap):
self.assertEqual(index, node.long_id)
self.assertTrue(index < 5)
def test_iteration(self):
heap = NodeHeap(mknode(intid=0), 5)
nodes = [mknode(intid=x) for x in range(10)]
for index, node in enumerate(nodes):
heap.push(node)
for index, node in enumerate(heap):
self.assertEqual(index, node.long_id)
self.assertTrue(index < 5)
def test_maxSize(self):
n = NodeHeap(mknode(intid=0), 3)
self.assertEqual(0, len(n))
for d in range(10):
n.push(mknode(intid=d))
self.assertEqual(3, len(n))
self.assertEqual(3, len(list(n)))
def test_maxSize(self):
n = NodeHeap(mknode(intid=0), 3)
self.assertEqual(0, len(n))
for d in range(10):
n.push(mknode(intid=d))
self.assertEqual(3, len(n))
self.assertEqual(3, len(list(n)))
def test_remove(self):
heap = NodeHeap(mknode(intid=0), 5)
nodes = [mknode(intid=x) for x in range(10)]
for node in nodes:
heap.push(node)
heap.remove([nodes[0].id, nodes[1].id])
self.assertEqual(len(list(heap)), 5)
for index, node in enumerate(heap):
self.assertEqual(index + 2, node.long_id)
self.assertTrue(index < 5)
def __init__(self,
protocol,
node,
peers,
ksize,
alpha,
save_at_nearest=True):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
self.saveToNearestWitoutValue = save_at_nearest
def test_nodesFound(self):
self._connecting_to_connected()
self.wire_protocol[self.addr1] = self.con
self.wire_protocol[self.addr2] = self.con
self.wire_protocol[self.addr3] = self.con
self.protocol.router.addContact(self.node1)
self.protocol.router.addContact(self.node2)
self.protocol.router.addContact(self.node3)
# test resonse with uncontacted nodes
node = Node(digest("s"))
nearest = self.protocol.router.findNeighbors(node)
spider = ValueSpiderCrawl(self.protocol, node, nearest,
dht.constants.KSIZE, dht.constants.ALPHA)
response = (True, (self.node1.getProto().SerializeToString(),
self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.clock.advance(100 * constants.PACKET_TIMEOUT)
connection.REACTOR.runUntilCurrent()
self.assertEqual(len(self.proto_mock.send_datagram.call_args_list), 4)
# test all been contacted
spider = ValueSpiderCrawl(self.protocol, node, nearest,
dht.constants.KSIZE, dht.constants.ALPHA)
for peer in spider.nearest.getUncontacted():
spider.nearest.markContacted(peer)
response = (True, (self.node1.getProto().SerializeToString(),
self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node2.id: response}
resp = spider._nodesFound(responses)
self.assertTrue(resp is None)
# test didn't happen
spider = ValueSpiderCrawl(self.protocol, node, nearest,
dht.constants.KSIZE, dht.constants.ALPHA)
response = (False, (self.node1.getProto().SerializeToString(),
self.node2.getProto().SerializeToString(),
self.node3.getProto().SerializeToString()))
responses = {self.node1.id: response}
spider._nodesFound(responses)
self.assertTrue(len(spider.nearest) == 2)
# test got value
val = Value()
val.valueKey = digest("contractID")
val.serializedData = self.protocol.sourceNode.getProto(
).SerializeToString()
response = (True, ("value", val.SerializeToString()))
responses = {self.node3.id: response}
spider.nearestWithoutValue = NodeHeap(node, 1)
value = spider._nodesFound(responses)
self.assertEqual(value[0], val.SerializeToString())
def __init__(self, protocol, node, peers, ksize, alpha):
"""
Create a new C{SpiderCrawl}er.
Args:
protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance.
node: A :class:`~kademlia.node.Node` representing the key we're looking for
peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network
ksize: The value for k based on the paper
alpha: The value for alpha based on the paper
"""
self.protocol = protocol
self.ksize = ksize
self.alpha = alpha
self.node = node
self.nearest = NodeHeap(self.node, self.ksize)
self.lastIDsCrawled = []
self.log = Logger(system=self)
self.log.debug("creating spider with peers: %s" % peers)
self.nearest.push(peers)
class SpiderCrawl(object):
"""
Crawl the network and look for given 160-bit keys.
"""
def __init__(self, protocol, node, peers, ksize, alpha):
"""
Create a new C{SpiderCrawl}er.
Args:
protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance.
node: A :class:`~kademlia.node.Node` representing the key we're looking for
peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network
ksize: The value for k based on the paper
alpha: The value for alpha based on the paper
"""
self.protocol = protocol
self.ksize = ksize
self.alpha = alpha
self.node = node
self.nearest = NodeHeap(self.node, self.ksize)
self.lastIDsCrawled = []
self.log = Logger(system=self)
self.log.debug("creating spider with peers: %s" % peers)
self.nearest.push(peers)
def _find(self, rpcmethod):
"""
Get either a value or list of nodes.
Args:
rpcmethod: The protocol's callFindValue or callFindNode.
The process:
1. calls find_* to current ALPHA nearest not already queried nodes,
adding results to current nearest list of k nodes.
2. current nearest list needs to keep track of who has been queried already
sort by nearest, keep KSIZE
3. if list is same as last time, next call should be to everyone not
yet queried
4. repeat, unless nearest list has all been queried, then ur done
"""
self.log.debug("crawling with nearest: %s" % str(tuple(self.nearest)))
count = self.alpha
if self.nearest.getIDs() == self.lastIDsCrawled:
self.log.debug("last iteration same as current - checking all in list now")
count = len(self.nearest)
self.lastIDsCrawled = self.nearest.getIDs()
ds = {}
for peer in self.nearest.getUncontacted()[:count]:
ds[peer.id] = rpcmethod(peer, self.node)
self.nearest.markContacted(peer)
return deferredDict(ds).addCallback(self._nodesFound)
class SpiderCrawl(object):
"""
Crawl the network and look for given 160-bit keys.
"""
def __init__(self, protocol, node, peers, ksize, alpha):
"""
Create a new C{SpiderCrawl}er.
Args:
protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance.
node: A :class:`~kademlia.node.Node` representing the key we're looking for
peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network
ksize: The value for k based on the paper
alpha: The value for alpha based on the paper
"""
self.protocol = protocol
self.ksize = ksize
self.alpha = alpha
self.node = node
self.nearest = NodeHeap(self.node, self.ksize)
self.lastIDsCrawled = []
self.log = Logger(system=self)
self.log.debug("creating spider with peers: %s" % peers)
self.nearest.push(peers)
def _find(self, rpcmethod):
"""
Get either a value or list of nodes.
Args:
rpcmethod: The protocol's callFindValue or callFindNode.
The process:
1. calls find_* to current ALPHA nearest not already queried nodes,
adding results to current nearest list of k nodes.
2. current nearest list needs to keep track of who has been queried already
sort by nearest, keep KSIZE
3. if list is same as last time, next call should be to everyone not
yet queried
4. repeat, unless nearest list has all been queried, then ur done
"""
self.log.debug("crawling with nearest: %s" % str(tuple(self.nearest)))
count = self.alpha
if self.nearest.getIDs() == self.lastIDsCrawled:
self.log.debug("last iteration same as current - checking all in list now")
count = len(self.nearest)
self.lastIDsCrawled = self.nearest.getIDs()
ds = {}
for peer in self.nearest.getUncontacted()[:count]:
ds[peer.id] = rpcmethod(peer, self.node)
self.nearest.markContacted(peer)
return deferredDict(ds).addCallback(self._nodesFound)
def test_remove(self):
heap = NodeHeap(mknode(intid=0), 5)
nodes = [mknode(intid=x) for x in range(10)]
for node in nodes:
heap.push(node)
heap.remove([nodes[0].id, nodes[1].id])
self.assertEqual(len(list(heap)), 5)
for index, node in enumerate(heap):
self.assertEqual(index + 2, node.long_id)
self.assertTrue(index < 5)
def __init__(self, protocol, node, peers, ksize, alpha):
"""
Create a new C{SpiderCrawl}er.
Args:
protocol: A :class:`~kademlia.protocol.KademliaProtocol` instance.
node: A :class:`~kademlia.node.Node` representing the key we're looking for
peers: A list of :class:`~kademlia.node.Node` instances that provide the entry point for the network
ksize: The value for k based on the paper
alpha: The value for alpha based on the paper
"""
self.protocol = protocol
self.ksize = ksize
self.alpha = alpha
self.node = node
self.nearest = NodeHeap(self.node, self.ksize)
self.lastIDsCrawled = []
self.log = Logger(system=self)
self.log.debug("creating spider with peers: %s" % peers)
self.nearest.push(peers)
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
foundValues.append(response.getValue())
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
valueCounts = Counter(values)
if len(valueCounts) != 1:
args = (self.node.long_id, str(values))
self.log.warning("Got multiple values for key %i: %s" % args)
value = valueCounts.most_common(1)[0][0]
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(self.protocol.callStore(peerToSaveTo, self.node.id, val.valueKey, val.serializedData))
except:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha, save_at_nearest=True):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
self.saveToNearestWitoutValue = save_at_nearest
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
# since we get back a list of values, we will just extend foundValues (excluding duplicates)
foundValues = list(set(foundValues) | set(response.getValue()))
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
value_dict = defaultdict(list)
ttl_dict = defaultdict(list)
for v in values:
try:
d = objects.Value()
d.ParseFromString(v)
value_dict[d.valueKey].append(d.serializedData)
ttl_dict[d.valueKey].append(d.ttl)
except Exception:
pass
value = []
for k, v in value_dict.items():
ttl = ttl_dict[k]
if len(v) > 1:
valueCounts = Counter(v)
v = [valueCounts.most_common(1)[0][0]]
ttlCounts = Counter(ttl_dict[k])
ttl = [ttlCounts.most_common(1)[0][0]]
val = objects.Value()
val.valueKey = k
val.serializedData = v[0]
val.ttl = ttl[0]
value.append(val.SerializeToString())
if self.saveToNearestWitoutValue:
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(self.protocol.callStore(peerToSaveTo, self.node.id, val.valueKey,
val.serializedData, val.ttl))
except Exception:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha, save_at_nearest=True):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
self.saveToNearestWitoutValue = save_at_nearest
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
# since we get back a list of values, we will just extend foundValues (excluding duplicates)
foundValues = list(set(foundValues) | set(response.getValue()))
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
value_dict = defaultdict(list)
ttl_dict = defaultdict(list)
for v in values:
try:
d = objects.Value()
d.ParseFromString(v)
value_dict[d.valueKey].append(d.serializedData)
ttl_dict[d.valueKey].append(d.ttl)
except Exception:
pass
value = []
for k, v in value_dict.items():
ttl = ttl_dict[k]
if len(v) > 1:
valueCounts = Counter(v)
v = [valueCounts.most_common(1)[0][0]]
ttlCounts = Counter(ttl_dict[k])
ttl = [ttlCounts.most_common(1)[0][0]]
val = objects.Value()
val.valueKey = k
val.serializedData = v[0]
val.ttl = ttl[0]
value.append(val.SerializeToString())
if self.saveToNearestWitoutValue:
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(self.protocol.callStore(peerToSaveTo, self.node.id, val.valueKey,
val.serializedData, val.ttl))
except Exception:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
def test_getNoneNodeById(self):
n = Node('127.0.0.1', 0, 'testkey')
nh = NodeHeap(n, 5)
val = nh.getNodeById('')
self.assertIsNone(val)
def test_getNoneNodeById(self):
n = Node("127.0.0.1", 0, "testkey")
nh = NodeHeap(n, 5)
val = nh.getNodeById("")
self.assertIsNone(val)
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
foundValues.append(response.getValue())
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
valueCounts = Counter(values)
if len(valueCounts) != 1:
args = (self.node.long_id, str(values))
self.log.warning("got multiple values for key %i: %s" % args)
value = valueCounts.most_common(1)[0][0]
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(
self.protocol.callStore(peerToSaveTo, self.node.id,
val.valueKey,
val.serializedData))
except Exception:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
def __init__(self, protocol, node, peers, ksize, alpha, save_at_nearest=True):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
self.saveToNearestWitoutValue = save_at_nearest
def test_getNoneNodeById(self):
n = Node('127.0.0.1', 0, 'testkey')
nh = NodeHeap(n, 5)
val = nh.getNodeById('')
self.assertIsNone(val)
