class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha, save_at_nearest=True):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
self.saveToNearestWitoutValue = save_at_nearest
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
# since we get back a list of values, we will just extend foundValues (excluding duplicates)
foundValues = list(set(foundValues) | set(response.getValue()))
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
value_dict = defaultdict(list)
ttl_dict = defaultdict(list)
for v in values:
try:
d = objects.Value()
d.ParseFromString(v)
value_dict[d.valueKey].append(d.serializedData)
ttl_dict[d.valueKey].append(d.ttl)
except Exception:
pass
value = []
for k, v in value_dict.items():
ttl = ttl_dict[k]
if len(v) > 1:
valueCounts = Counter(v)
v = [valueCounts.most_common(1)[0][0]]
ttlCounts = Counter(ttl_dict[k])
ttl = [ttlCounts.most_common(1)[0][0]]
val = objects.Value()
val.valueKey = k
val.serializedData = v[0]
val.ttl = ttl[0]
value.append(val.SerializeToString())
if self.saveToNearestWitoutValue:
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(self.protocol.callStore(peerToSaveTo, self.node.id, val.valueKey,
val.serializedData, val.ttl))
except Exception:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha, save_at_nearest=True):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
self.saveToNearestWitoutValue = save_at_nearest
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
# since we get back a list of values, we will just extend foundValues (excluding duplicates)
foundValues = list(set(foundValues) | set(response.getValue()))
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
value_dict = defaultdict(list)
ttl_dict = defaultdict(list)
for v in values:
try:
d = objects.Value()
d.ParseFromString(v)
value_dict[d.valueKey].append(d.serializedData)
ttl_dict[d.valueKey].append(d.ttl)
except Exception:
pass
value = []
for k, v in value_dict.items():
ttl = ttl_dict[k]
if len(v) > 1:
valueCounts = Counter(v)
v = [valueCounts.most_common(1)[0][0]]
ttlCounts = Counter(ttl_dict[k])
ttl = [ttlCounts.most_common(1)[0][0]]
val = objects.Value()
val.valueKey = k
val.serializedData = v[0]
val.ttl = ttl[0]
value.append(val.SerializeToString())
if self.saveToNearestWitoutValue:
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(self.protocol.callStore(peerToSaveTo, self.node.id, val.valueKey,
val.serializedData, val.ttl))
except Exception:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
foundValues.append(response.getValue())
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
valueCounts = Counter(values)
if len(valueCounts) != 1:
args = (self.node.long_id, str(values))
self.log.warning("Got multiple values for key %i: %s" % args)
value = valueCounts.most_common(1)[0][0]
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(self.protocol.callStore(peerToSaveTo, self.node.id, val.valueKey, val.serializedData))
except:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
class ValueSpiderCrawl(SpiderCrawl):
def __init__(self, protocol, node, peers, ksize, alpha):
SpiderCrawl.__init__(self, protocol, node, peers, ksize, alpha)
# keep track of the single nearest node without value - per
# section 2.3 so we can set the key there if found
self.nearestWithoutValue = NodeHeap(self.node, 1)
def find(self):
"""
Find either the closest nodes or the value requested.
"""
return self._find(self.protocol.callFindValue)
def _nodesFound(self, responses):
"""
Handle the result of an iteration in _find.
"""
toremove = []
foundValues = []
for peerid, response in responses.items():
response = RPCFindResponse(response)
if not response.happened():
toremove.append(peerid)
elif response.hasValue():
foundValues.append(response.getValue())
else:
peer = self.nearest.getNodeById(peerid)
self.nearestWithoutValue.push(peer)
self.nearest.push(response.getNodeList())
self.nearest.remove(toremove)
if len(foundValues) > 0:
return self._handleFoundValues(foundValues)
if self.nearest.allBeenContacted():
# not found!
return None
return self.find()
def _handleFoundValues(self, values):
"""
We got some values! Exciting. But let's make sure
they're all the same or freak out a little bit. Also,
make sure we tell the nearest node that *didn't* have
the value to store it.
"""
valueCounts = Counter(values)
if len(valueCounts) != 1:
args = (self.node.long_id, str(values))
self.log.warning("got multiple values for key %i: %s" % args)
value = valueCounts.most_common(1)[0][0]
ds = []
peerToSaveTo = self.nearestWithoutValue.popleft()
if peerToSaveTo is not None:
for v in value:
try:
val = objects.Value()
val.ParseFromString(v)
ds.append(
self.protocol.callStore(peerToSaveTo, self.node.id,
val.valueKey,
val.serializedData))
except Exception:
pass
return defer.gatherResults(ds).addCallback(lambda _: value)
return value
