def test_josn_support(self): '''It should convert to json and back''' node_list = NodeList([Node(KeyBytes(), ('127.0.0.1', 12345))]) self.assertEqual(node_list, NodeList.from_json_loadable(node_list.to_json_dumpable()))
def sorted_nodes(self): '''The nodes sorted by distance. The first node is the closest. :rtype: ``list`` ''' node_list = NodeList(self._nodes) node_list.sort_distance(self._key_obj) return node_list
def run(self, controller, key, index): kvpid = KVPID(key, index) kvp_record = controller._kvp_table.record(kvpid) task = controller.find_value_shortlist(key, index) self.hook_task(task) shortlist = task.result() node_list = NodeList(list(shortlist.nodes - shortlist.useful_nodes)) node_list.sort_distance(controller.key) nodes = collections.deque(node_list) kvp_record.last_update = time.time() _logger.debug('Uploading value %s', kvpid) store_count = 0 if not nodes and not shortlist.useful_nodes: _logger.warning('No destination nodes for publication') _logger.debug('%s', controller._routing_table) while len(nodes): tasks = [] for dummy in range(DHTNetwork.NETWORK_PARALLELISM): try: node = nodes.popleft() except IndexError: break value = controller._kvp_table[kvpid] task = controller.store_to_node(node, key, index, value, kvp_record.timestamp) self.hook_task(task) self._store_to_node_task_observer(True, task) tasks.append(task) for task in tasks: bytes_sent = task.result() self._store_to_node_task_observer(False, task) if bytes_sent: store_count += 1 return store_count
def run(self, controller, node, key): d = controller._template_dict() d[JSONKeys.RPC] = JSONKeys.RPCs.FIND_NODE d[JSONKeys.KEY] = key.base64 task = controller._network.send(node.address, d, timeout=True) self.hook_task(task) data_packet = task.result() if not data_packet: _logger.debug('Find node timeout %s←%s', controller.node, node) return controller._update_routing_table_from_data_packet(data_packet) dict_obj = data_packet.dict_obj node_list = dict_obj.get(JSONKeys.NODES) if node_list: try: nodes = NodeList.from_json_loadable(node_list) except ValueError as e: _logger.debug('Find node invalid %s←%s err=%s', controller.node, node, e) else: _logger.debug('Find node nodes %s←%s len=%d', controller.node, node, len(nodes)) return nodes _logger.debug('Find node invalid %s←%s', controller.node, node)
def run(self, controller, key, index): _logger.info('Downloading %s:%s', key.base16, index.base16) self._controller = controller self._key = key self._index = index find_value_task = controller.find_value_shortlist(key, index) self.hook_task(find_value_task) self._shortlist = find_value_task.result() self._useful_node_list = NodeList(self._shortlist.useful_nodes) if not self._useful_node_list: return None self._useful_node_list.sort_distance(key) self._kvp_exchange_info = self._shortlist.get_common_kvp_exchange_info( key, index) for dummy in range(3): self._file = io.BytesIO() if self._download_round(): break else: self._file = None if self.is_running: return if self.file: self._replicate_value() return self._file.value()
def run(self, controller, node, key, index): d = controller._template_dict() d[JSONKeys.RPC] = JSONKeys.RPCs.FIND_VALUE d[JSONKeys.KEY] = key.base64 if index: d[JSONKeys.INDEX] = index.base64 future = controller._network.send(node.address, d, timeout=True) data_packet = future.result() if not data_packet: _logger.debug('Find value timeout %s←%s', controller.node, node) return controller._update_routing_table_from_data_packet(data_packet) dict_obj = data_packet.dict_obj if JSONKeys.VALUES in dict_obj: kvp_info_list = KVPExchangeInfoList.from_json_loadable( dict_obj[JSONKeys.VALUES]) _logger.debug('Find value %s←%s dictlen=%d', controller.node, node, len(kvp_info_list)) return FindValueFromNodeResult(kvp_info_list, None) elif JSONKeys.NODES in dict_obj: try: nodes = NodeList.from_json_loadable(dict_obj[JSONKeys.NODES]) except ValueError as e: _logger.debug('Find value node invalid %s←%s err=%s', controller.node, node, e) else: _logger.debug('Find value nodes %s←%s len=%d', controller.node, node, len(nodes)) return FindValueFromNodeResult(None, nodes)
class GetValueTask(Task): '''Returns the ``bytes``''' def run(self, controller, key, index): _logger.info('Downloading %s:%s', key.base16, index.base16) self._controller = controller self._key = key self._index = index find_value_task = controller.find_value_shortlist(key, index) self.hook_task(find_value_task) self._shortlist = find_value_task.result() self._useful_node_list = NodeList(self._shortlist.useful_nodes) if not self._useful_node_list: return None self._useful_node_list.sort_distance(key) self._kvp_exchange_info = self._shortlist.get_common_kvp_exchange_info( key, index) for dummy in range(3): self._file = io.BytesIO() if self._download_round(): break else: self._file = None if self.is_running: return if self.file: self._replicate_value() return self._file.value() def _download_round(self): _logger.debug('Download round') for node in self._useful_node_list: download_task = self._controller.get_value_from_node(node, self._key, self._index, offset=self._file.tell()) self.hook_task(download_task) transfered_file = download_task.result() data = transfered_file.read() self._file.write(data) if self._file.tell() >= self._data_size: break return self._index.validate_value(data) def _replicate_value(self): node_list = self._shortlist.sorted_nodes if node_list: node = node_list[0] if node not in self._shortlist.useful_nodes: _logger.debug('Replicating value') self._controller.store_to_node(node, self._key, self._index, self._file.value(), self._kvp_exchange_info.timestamp)