def test_josn_support(self):
'''It should convert to json and back'''
node_list = NodeList([Node(KeyBytes(), ('127.0.0.1', 12345))])
self.assertEqual(node_list,
NodeList.from_json_loadable(node_list.to_json_dumpable()))
def sorted_nodes(self):
'''The nodes sorted by distance.
The first node is the closest.
:rtype: ``list``
'''
node_list = NodeList(self._nodes)
node_list.sort_distance(self._key_obj)
return node_list
def run(self, controller, key, index):
kvpid = KVPID(key, index)
kvp_record = controller._kvp_table.record(kvpid)
task = controller.find_value_shortlist(key, index)
self.hook_task(task)
shortlist = task.result()
node_list = NodeList(list(shortlist.nodes - shortlist.useful_nodes))
node_list.sort_distance(controller.key)
nodes = collections.deque(node_list)
kvp_record.last_update = time.time()
_logger.debug('Uploading value %s', kvpid)
store_count = 0
if not nodes and not shortlist.useful_nodes:
_logger.warning('No destination nodes for publication')
_logger.debug('%s', controller._routing_table)
while len(nodes):
tasks = []
for dummy in range(DHTNetwork.NETWORK_PARALLELISM):
try:
node = nodes.popleft()
except IndexError:
break
value = controller._kvp_table[kvpid]
task = controller.store_to_node(node, key, index, value,
kvp_record.timestamp)
self.hook_task(task)
self._store_to_node_task_observer(True, task)
tasks.append(task)
for task in tasks:
bytes_sent = task.result()
self._store_to_node_task_observer(False, task)
if bytes_sent:
store_count += 1
return store_count
def run(self, controller, node, key):
d = controller._template_dict()
d[JSONKeys.RPC] = JSONKeys.RPCs.FIND_NODE
d[JSONKeys.KEY] = key.base64
task = controller._network.send(node.address, d, timeout=True)
self.hook_task(task)
data_packet = task.result()
if not data_packet:
_logger.debug('Find node timeout %s←%s', controller.node, node)
return
controller._update_routing_table_from_data_packet(data_packet)
dict_obj = data_packet.dict_obj
node_list = dict_obj.get(JSONKeys.NODES)
if node_list:
try:
nodes = NodeList.from_json_loadable(node_list)
except ValueError as e:
_logger.debug('Find node invalid %s←%s err=%s',
controller.node, node, e)
else:
_logger.debug('Find node nodes %s←%s len=%d',
controller.node, node, len(nodes))
return nodes
_logger.debug('Find node invalid %s←%s', controller.node, node)
def run(self, controller, key, index):
_logger.info('Downloading %s:%s', key.base16, index.base16)
self._controller = controller
self._key = key
self._index = index
find_value_task = controller.find_value_shortlist(key, index)
self.hook_task(find_value_task)
self._shortlist = find_value_task.result()
self._useful_node_list = NodeList(self._shortlist.useful_nodes)
if not self._useful_node_list:
return None
self._useful_node_list.sort_distance(key)
self._kvp_exchange_info = self._shortlist.get_common_kvp_exchange_info(
key, index)
for dummy in range(3):
self._file = io.BytesIO()
if self._download_round():
break
else:
self._file = None
if self.is_running:
return
if self.file:
self._replicate_value()
return self._file.value()
def run(self, controller, node, key, index):
d = controller._template_dict()
d[JSONKeys.RPC] = JSONKeys.RPCs.FIND_VALUE
d[JSONKeys.KEY] = key.base64
if index:
d[JSONKeys.INDEX] = index.base64
future = controller._network.send(node.address, d, timeout=True)
data_packet = future.result()
if not data_packet:
_logger.debug('Find value timeout %s←%s', controller.node, node)
return
controller._update_routing_table_from_data_packet(data_packet)
dict_obj = data_packet.dict_obj
if JSONKeys.VALUES in dict_obj:
kvp_info_list = KVPExchangeInfoList.from_json_loadable(
dict_obj[JSONKeys.VALUES])
_logger.debug('Find value %s←%s dictlen=%d',
controller.node, node, len(kvp_info_list))
return FindValueFromNodeResult(kvp_info_list, None)
elif JSONKeys.NODES in dict_obj:
try:
nodes = NodeList.from_json_loadable(dict_obj[JSONKeys.NODES])
except ValueError as e:
_logger.debug('Find value node invalid %s←%s err=%s',
controller.node, node, e)
else:
_logger.debug('Find value nodes %s←%s len=%d',
controller.node, node, len(nodes))
return FindValueFromNodeResult(None, nodes)
class GetValueTask(Task):
'''Returns the ``bytes``'''
def run(self, controller, key, index):
_logger.info('Downloading %s:%s', key.base16, index.base16)
self._controller = controller
self._key = key
self._index = index
find_value_task = controller.find_value_shortlist(key, index)
self.hook_task(find_value_task)
self._shortlist = find_value_task.result()
self._useful_node_list = NodeList(self._shortlist.useful_nodes)
if not self._useful_node_list:
return None
self._useful_node_list.sort_distance(key)
self._kvp_exchange_info = self._shortlist.get_common_kvp_exchange_info(
key, index)
for dummy in range(3):
self._file = io.BytesIO()
if self._download_round():
break
else:
self._file = None
if self.is_running:
return
if self.file:
self._replicate_value()
return self._file.value()
def _download_round(self):
_logger.debug('Download round')
for node in self._useful_node_list:
download_task = self._controller.get_value_from_node(node,
self._key, self._index, offset=self._file.tell())
self.hook_task(download_task)
transfered_file = download_task.result()
data = transfered_file.read()
self._file.write(data)
if self._file.tell() >= self._data_size:
break
return self._index.validate_value(data)
def _replicate_value(self):
node_list = self._shortlist.sorted_nodes
if node_list:
node = node_list[0]
if node not in self._shortlist.useful_nodes:
_logger.debug('Replicating value')
self._controller.store_to_node(node, self._key, self._index,
self._file.value(), self._kvp_exchange_info.timestamp)
