def callback(self, packet, sender=None):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
if self.operator.is_stopped():
return
if packet.ret_code != RC_OK:
logger.warning('ManageNeighbour: neighbour does not process request...')
self.operator.rebalance_remove()
self.operator.rebalance_append({'neighbour_type': NT_SUPERIOR})
self.operator.rebalance_append({'neighbour_type': NT_UPPER})
return
n_type, operation, node_address, op_type = self._valiadate_packet(packet.ret_parameters)
dont_append = packet.ret_parameters.get('dont_append', False)
dont_remove = packet.ret_parameters.get('dont_remove', False)
self.operator.callback_manage_neighbours(n_type, operation, node_address, \
op_type, dont_append, dont_remove)
self.operator.rebalance_remove()
self.operator.rebalance_append(packet.ret_parameters)
def run(self):
logger.info('Thread started!')
while not self.stopped.is_set():
dt = 0
try:
t0 = datetime.now()
logger.debug('Collecting %s nodes statistic...'%self.check_status)
nodeaddrs = self.operator.get_nodes_list(self.check_status)
for nodeaddr in nodeaddrs:
logger.debug('Get statistic from %s'%nodeaddr)
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = self.client.call_sync(nodeaddr, packet_obj)
if self.check_status == UP and ret_packet.ret_code:
logger.warning('Node with address %s does not response... Details: %s'%(nodeaddr, ret_packet))
self.operator.change_node_status(nodeaddr, DOWN)
else:
stat = json.dumps(ret_packet.ret_parameters)
self.operator.update_node_stat(nodeaddr, stat)
dt = total_seconds(datetime.now() - t0)
logger.info('Nodes (with status=%s) stat is collected. Processed secs: %s'%(self.check_status, dt))
except Exception, err:
logger.error(str(err))
finally:
def _check_range_free_size(self):
dht_range = self.operator.get_dht_range()
free_percents = dht_range.get_free_size_percents()
percents = 100 - free_percents
if percents >= Config.MAX_USED_SIZE_PERCENTS:
if free_percents < Config.CRITICAL_FREE_SPACE_PERCENT:
logger.warning('Critical free disk space! Blocking range for write!')
dht_range.block_for_write()
logger.warning('Few free size for data range. Trying pull part of range to network')
if not self._pull_subrange(dht_range):
self._pull_subrange(dht_range)
elif percents >= Config.DANGER_USED_SIZE_PERCENTS:
if self.__notification_flag:
return
message = '%s percents'%percents
params = {'event_type': ET_ALERT, 'event_message': message,\
'event_topic': 'HDD usage', 'event_provider': self.operator.self_address}
packet = FabnetPacketRequest(method='NotifyOperation', parameters=params, sender=self.operator.self_address)
self.operator.call_network(packet)
self.__notification_flag = True
else:
self.__notification_flag = False
def check_dht_range(self, reinit=True):
if self.status == DS_INITIALIZE:
return
dht_range = self.get_dht_range()
if dht_range.get_subranges():
return
start = dht_range.get_start()
end = dht_range.get_end()
range_obj = self.ranges_table.find(start)
if not range_obj:
range_obj = self.ranges_table.find(end)
if not range_obj or range_obj.start != start or range_obj.end != end or range_obj.node_address != self.self_address:
msg = 'Invalid self range!'
if range_obj:
msg += ' hash table range - [%040x-%040x]%s... my range - [%040x-%040x]%s'%\
(range_obj.start, range_obj.end, range_obj.node_address, start, end, self.self_address)
else:
msg += 'Not found in hash table'
logger.info(msg)
if (not range_obj) or reinit:
logger.warning('DHT range on this node is not found in ranges_table')
if range_obj:
logger.info('Self range: %040x-%040x, In hash table: %040x-%040x(%s)'%\
(start, end, range_obj.start, range_obj.end, range_obj.node_address))
logger.info('Trying reinit node as DHT member...')
self.start_as_dht_member()
return True
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
try:
event_type = packet.parameters.get('event_type', None)
event_provider = packet.parameters.get('event_provider', None)
event_topic = packet.parameters.get('event_topic', None)
if event_provider is None:
raise Exception('event_provider does not found!')
event_message = packet.parameters.get('event_message', None)
if packet.sender is None: #this is sender
if event_type == ET_ALERT:
logger.warning('[ALERT][%s] *%s* %s'%(event_provider, event_topic, event_message))
elif event_type == ET_INFO:
logger.info('[INFORMATION][%s] *%s* %s'%(event_provider, event_topic, event_message))
else:
logger.info('[NOTIFICATION.%s][%s] *%s* %s'%(event_type, event_provider, event_topic, event_message))
self.on_network_notify(event_type, event_provider, event_topic, event_message)
except Exception, err:
logger.error('[NotifyOperation] %s'%err)
def _check_neighbours_count(self, n_type, neighbours, other_n_type, other_neighbours, ret_parameters, reinit=False):
if len(neighbours) >= ONE_DIRECT_NEIGHBOURS_COUNT:
self.__discovered_nodes[n_type] = []
return
new_node = None
for node in ret_parameters.get('neighbours', []):
#trying find new node...
if (node in self.__discovered_nodes[n_type]) or (node in neighbours) \
or (node in other_neighbours) or (node == self.operator.self_address):
continue
new_node = node
break
d_nodes = self.operator.get_discovered_nodes()
if reinit:
self.__discovered_nodes[n_type] = []
for node_addr, node_info in d_nodes.items():
if (node_addr in self.__discovered_nodes[n_type]) or (node_addr in neighbours) \
or (node_addr in other_neighbours) or (node_addr == self.operator.self_address):
continue
new_node = node_addr
if len(set(node_info.get('uppers', [])) & set(node_info.get('superiors', []))):
#node with interset neighbours
break
if new_node == None:
for node in neighbours:
if node not in self.__discovered_nodes[n_type]:
new_node = node
break
if new_node == None:
for node in other_neighbours:
if node not in self.__discovered_nodes[n_type]:
new_node = node
break
if new_node is None:
return
self.__discovered_nodes[n_type].append(new_node)
if not self.operator.is_node_alive(new_node):
#node is not alive...
logger.warning('trying communacate with %s node but it is not responding...'%new_node)
return self._check_neighbours_count(n_type, neighbours, other_n_type, other_neighbours, ret_parameters, reinit=False)
parameters = { 'neighbour_type': other_n_type, 'operation': MNO_APPEND,
'node_address': self.operator.self_address, 'operator_type': self.operator.OPTYPE }
self.operator.async_remote_call(new_node, 'ManageNeighbour', parameters)
def process(self, packet):
"""In this method should be implemented logic of processing
reuqest packet from sender node
@param packet - object of FabnetPacketRequest class
@return object of FabnetPacketResponse
or None for disabling packet response to sender
"""
key = packet.parameters.get('key', None)
replica_count = packet.parameters.get('replica_count', None)
if key is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Key is not found in request packet!')
if replica_count is None:
return FabnetPacketResponse(ret_code=RC_ERROR,
ret_message='Replica count should be passed to GetData operation')
self._validate_key(key)
keys = KeyUtils.get_all_keys(key, replica_count)
data = None
checksum = None
is_replica = False
for key in keys:
long_key = self._validate_key(key)
range_obj = self.operator.find_range(long_key)
if not range_obj:
logger.warning('[ClientGetOperation] Internal error: No hash range found for key=%s!'%key)
else:
_, _, node_address = range_obj
resp = self._init_operation(node_address, 'GetDataBlock', {'key': key, 'is_replica': is_replica, \
'user_id': packet.session_id}, sync=True)
if resp.ret_code == RC_PERMISSION_DENIED:
return FabnetPacketResponse(ret_code=RC_PERMISSION_DENIED, ret_message=resp.ret_message)
elif resp.ret_code:
logger.warning('[ClientGetOperation] GetDataBlock error from %s: %s'%(node_address, resp.ret_message))
else:
data = resp.binary_data
checksum = resp.ret_parameters['checksum']
break
is_replica = True
if data is None:
return FabnetPacketResponse(ret_code=RC_NO_DATA, ret_message='No data found!')
return FabnetPacketResponse(binary_data=data, ret_parameters={'checksum': checksum})
if n_type.lower() != optype:
continue
if type(urls) not in (list, tuple):
urls = [urls]
try:
self.__upgrade_node(urls, packet.parameters.get('force', False))
except Exception, err:
self._throw_event(ET_ALERT, 'UpgradeNodeOperation failed', err)
logger.error('[UpgradeNodeOperation] %s'%err)
return FabnetPacketResponse(ret_code=RC_UPGRADE_ERROR, ret_message=err)
return FabnetPacketResponse()
else:
logger.warning('UpgradeNodeOperation: release URL does not specified for "%s" node type'%optype)
def callback(self, packet, sender):
"""In this method should be implemented logic of processing
response packet from requested node
@param packet - object of FabnetPacketResponse class
@param sender - address of sender node.
If sender == None then current node is operation initiator
@return object of FabnetPacketResponse
that should be resended to current node requestor
or None for disabling packet resending
"""
pass
