def callback(self, packet, sender=None): """In this method should be implemented logic of processing response packet from requested node @param packet - object of FabnetPacketResponse class @param sender - address of sender node. If sender == None then current node is operation initiator @return object of FabnetPacketResponse that should be resended to current node requestor or None for disabling packet resending """ if self.operator.is_stopped(): return if packet.ret_code != RC_OK: logger.warning('ManageNeighbour: neighbour does not process request...') self.operator.rebalance_remove() self.operator.rebalance_append({'neighbour_type': NT_SUPERIOR}) self.operator.rebalance_append({'neighbour_type': NT_UPPER}) return n_type, operation, node_address, op_type = self._valiadate_packet(packet.ret_parameters) dont_append = packet.ret_parameters.get('dont_append', False) dont_remove = packet.ret_parameters.get('dont_remove', False) self.operator.callback_manage_neighbours(n_type, operation, node_address, \ op_type, dont_append, dont_remove) self.operator.rebalance_remove() self.operator.rebalance_append(packet.ret_parameters)
def run(self): logger.info('Thread started!') while not self.stopped.is_set(): dt = 0 try: t0 = datetime.now() logger.debug('Collecting %s nodes statistic...'%self.check_status) nodeaddrs = self.operator.get_nodes_list(self.check_status) for nodeaddr in nodeaddrs: logger.debug('Get statistic from %s'%nodeaddr) packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True) ret_packet = self.client.call_sync(nodeaddr, packet_obj) if self.check_status == UP and ret_packet.ret_code: logger.warning('Node with address %s does not response... Details: %s'%(nodeaddr, ret_packet)) self.operator.change_node_status(nodeaddr, DOWN) else: stat = json.dumps(ret_packet.ret_parameters) self.operator.update_node_stat(nodeaddr, stat) dt = total_seconds(datetime.now() - t0) logger.info('Nodes (with status=%s) stat is collected. Processed secs: %s'%(self.check_status, dt)) except Exception, err: logger.error(str(err)) finally:
def _check_range_free_size(self): dht_range = self.operator.get_dht_range() free_percents = dht_range.get_free_size_percents() percents = 100 - free_percents if percents >= Config.MAX_USED_SIZE_PERCENTS: if free_percents < Config.CRITICAL_FREE_SPACE_PERCENT: logger.warning('Critical free disk space! Blocking range for write!') dht_range.block_for_write() logger.warning('Few free size for data range. Trying pull part of range to network') if not self._pull_subrange(dht_range): self._pull_subrange(dht_range) elif percents >= Config.DANGER_USED_SIZE_PERCENTS: if self.__notification_flag: return message = '%s percents'%percents params = {'event_type': ET_ALERT, 'event_message': message,\ 'event_topic': 'HDD usage', 'event_provider': self.operator.self_address} packet = FabnetPacketRequest(method='NotifyOperation', parameters=params, sender=self.operator.self_address) self.operator.call_network(packet) self.__notification_flag = True else: self.__notification_flag = False
def check_dht_range(self, reinit=True): if self.status == DS_INITIALIZE: return dht_range = self.get_dht_range() if dht_range.get_subranges(): return start = dht_range.get_start() end = dht_range.get_end() range_obj = self.ranges_table.find(start) if not range_obj: range_obj = self.ranges_table.find(end) if not range_obj or range_obj.start != start or range_obj.end != end or range_obj.node_address != self.self_address: msg = 'Invalid self range!' if range_obj: msg += ' hash table range - [%040x-%040x]%s... my range - [%040x-%040x]%s'%\ (range_obj.start, range_obj.end, range_obj.node_address, start, end, self.self_address) else: msg += 'Not found in hash table' logger.info(msg) if (not range_obj) or reinit: logger.warning('DHT range on this node is not found in ranges_table') if range_obj: logger.info('Self range: %040x-%040x, In hash table: %040x-%040x(%s)'%\ (start, end, range_obj.start, range_obj.end, range_obj.node_address)) logger.info('Trying reinit node as DHT member...') self.start_as_dht_member() return True
def process(self, packet): """In this method should be implemented logic of processing reuqest packet from sender node @param packet - object of FabnetPacketRequest class @return object of FabnetPacketResponse or None for disabling packet response to sender """ try: event_type = packet.parameters.get('event_type', None) event_provider = packet.parameters.get('event_provider', None) event_topic = packet.parameters.get('event_topic', None) if event_provider is None: raise Exception('event_provider does not found!') event_message = packet.parameters.get('event_message', None) if packet.sender is None: #this is sender if event_type == ET_ALERT: logger.warning('[ALERT][%s] *%s* %s'%(event_provider, event_topic, event_message)) elif event_type == ET_INFO: logger.info('[INFORMATION][%s] *%s* %s'%(event_provider, event_topic, event_message)) else: logger.info('[NOTIFICATION.%s][%s] *%s* %s'%(event_type, event_provider, event_topic, event_message)) self.on_network_notify(event_type, event_provider, event_topic, event_message) except Exception, err: logger.error('[NotifyOperation] %s'%err)
def _check_neighbours_count(self, n_type, neighbours, other_n_type, other_neighbours, ret_parameters, reinit=False): if len(neighbours) >= ONE_DIRECT_NEIGHBOURS_COUNT: self.__discovered_nodes[n_type] = [] return new_node = None for node in ret_parameters.get('neighbours', []): #trying find new node... if (node in self.__discovered_nodes[n_type]) or (node in neighbours) \ or (node in other_neighbours) or (node == self.operator.self_address): continue new_node = node break d_nodes = self.operator.get_discovered_nodes() if reinit: self.__discovered_nodes[n_type] = [] for node_addr, node_info in d_nodes.items(): if (node_addr in self.__discovered_nodes[n_type]) or (node_addr in neighbours) \ or (node_addr in other_neighbours) or (node_addr == self.operator.self_address): continue new_node = node_addr if len(set(node_info.get('uppers', [])) & set(node_info.get('superiors', []))): #node with interset neighbours break if new_node == None: for node in neighbours: if node not in self.__discovered_nodes[n_type]: new_node = node break if new_node == None: for node in other_neighbours: if node not in self.__discovered_nodes[n_type]: new_node = node break if new_node is None: return self.__discovered_nodes[n_type].append(new_node) if not self.operator.is_node_alive(new_node): #node is not alive... logger.warning('trying communacate with %s node but it is not responding...'%new_node) return self._check_neighbours_count(n_type, neighbours, other_n_type, other_neighbours, ret_parameters, reinit=False) parameters = { 'neighbour_type': other_n_type, 'operation': MNO_APPEND, 'node_address': self.operator.self_address, 'operator_type': self.operator.OPTYPE } self.operator.async_remote_call(new_node, 'ManageNeighbour', parameters)
def process(self, packet): """In this method should be implemented logic of processing reuqest packet from sender node @param packet - object of FabnetPacketRequest class @return object of FabnetPacketResponse or None for disabling packet response to sender """ key = packet.parameters.get('key', None) replica_count = packet.parameters.get('replica_count', None) if key is None: return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Key is not found in request packet!') if replica_count is None: return FabnetPacketResponse(ret_code=RC_ERROR, ret_message='Replica count should be passed to GetData operation') self._validate_key(key) keys = KeyUtils.get_all_keys(key, replica_count) data = None checksum = None is_replica = False for key in keys: long_key = self._validate_key(key) range_obj = self.operator.find_range(long_key) if not range_obj: logger.warning('[ClientGetOperation] Internal error: No hash range found for key=%s!'%key) else: _, _, node_address = range_obj resp = self._init_operation(node_address, 'GetDataBlock', {'key': key, 'is_replica': is_replica, \ 'user_id': packet.session_id}, sync=True) if resp.ret_code == RC_PERMISSION_DENIED: return FabnetPacketResponse(ret_code=RC_PERMISSION_DENIED, ret_message=resp.ret_message) elif resp.ret_code: logger.warning('[ClientGetOperation] GetDataBlock error from %s: %s'%(node_address, resp.ret_message)) else: data = resp.binary_data checksum = resp.ret_parameters['checksum'] break is_replica = True if data is None: return FabnetPacketResponse(ret_code=RC_NO_DATA, ret_message='No data found!') return FabnetPacketResponse(binary_data=data, ret_parameters={'checksum': checksum})
if n_type.lower() != optype: continue if type(urls) not in (list, tuple): urls = [urls] try: self.__upgrade_node(urls, packet.parameters.get('force', False)) except Exception, err: self._throw_event(ET_ALERT, 'UpgradeNodeOperation failed', err) logger.error('[UpgradeNodeOperation] %s'%err) return FabnetPacketResponse(ret_code=RC_UPGRADE_ERROR, ret_message=err) return FabnetPacketResponse() else: logger.warning('UpgradeNodeOperation: release URL does not specified for "%s" node type'%optype) def callback(self, packet, sender): """In this method should be implemented logic of processing response packet from requested node @param packet - object of FabnetPacketResponse class @param sender - address of sender node. If sender == None then current node is operation initiator @return object of FabnetPacketResponse that should be resended to current node requestor or None for disabling packet resending """ pass