def run(self):
logger.info('Thread started!')
while not self.stopped.is_set():
dt = 0
try:
t0 = datetime.now()
logger.debug('Collecting %s nodes statistic...'%self.check_status)
nodeaddrs = self.operator.get_nodes_list(self.check_status)
for nodeaddr in nodeaddrs:
logger.debug('Get statistic from %s'%nodeaddr)
packet_obj = FabnetPacketRequest(method='NodeStatistic', sync=True)
ret_packet = self.client.call_sync(nodeaddr, packet_obj)
if self.check_status == UP and ret_packet.ret_code:
logger.warning('Node with address %s does not response... Details: %s'%(nodeaddr, ret_packet))
self.operator.change_node_status(nodeaddr, DOWN)
else:
stat = json.dumps(ret_packet.ret_parameters)
self.operator.update_node_stat(nodeaddr, stat)
dt = total_seconds(datetime.now() - t0)
logger.info('Nodes (with status=%s) stat is collected. Processed secs: %s'%(self.check_status, dt))
except Exception, err:
logger.error(str(err))
finally:
def check_neighbours(self):
ka_packet = FabnetPacketRequest(method=KEEP_ALIVE_METHOD, sender=self.self_address, sync=True)
superiors = self.get_neighbours(NT_SUPERIOR)
remove_nodes = []
for superior in superiors:
resp = self.fri_client.call_sync(superior, ka_packet)
cnt = 0
self._lock()
try:
if self.__superior_keep_alives.get(superior, None) is None:
self.__superior_keep_alives[superior] = 0
if resp.ret_code == RC_OK:
self.__superior_keep_alives[superior] = 0
elif resp.ret_code == RC_NOT_MY_NEIGHBOUR:
remove_nodes.append((NT_SUPERIOR, superior, False))
continue
else:
self.__superior_keep_alives[superior] += 1
cnt = self.__superior_keep_alives[superior]
finally:
self._unlock()
if cnt == KEEP_ALIVE_TRY_COUNT:
logger.info('Neighbour %s does not respond. removing it...'%superior)
remove_nodes.append((NT_SUPERIOR, superior, True))
#check upper nodes...
uppers = self.get_neighbours(NT_UPPER)
self._lock()
try:
cur_dt = datetime.now()
for upper in uppers:
ka_dt = self.__upper_keep_alives.get(upper, None)
if ka_dt == None:
self.__upper_keep_alives[upper] = datetime.now()
continue
if total_seconds(cur_dt - ka_dt) >= KEEP_ALIVE_MAX_WAIT_TIME:
logger.info('No keep alive packets from upper neighbour %s. removing it...'%upper)
remove_nodes.append((NT_UPPER, upper, False))
finally:
self._unlock()
for n_type, nodeaddr, is_not_respond in remove_nodes:
self.remove_neighbour(n_type, nodeaddr)
if is_not_respond:
self.on_neigbour_not_respond(n_type, nodeaddr)
if remove_nodes or \
(len(uppers) < ONE_DIRECT_NEIGHBOURS_COUNT) or \
(len(superiors) < ONE_DIRECT_NEIGHBOURS_COUNT):
self._rebalance_nodes()
return s_packet
except PermissionDeniedException, err:
return FabnetPacketResponse(from_node=self.__self_address,
message_id=packet.message_id, ret_code=RC_PERMISSION_DENIED,
ret_message='Permission denied!')
except Exception, err:
err_packet = FabnetPacketResponse(from_node=self.__self_address,
message_id=packet.message_id,
ret_code=1, ret_message= '[OpPROC] %s'%err)
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[OperationsManager.process] %s'%err)
return err_packet
finally:
if self.__op_stat is not None and t0:
dt = total_seconds(datetime.now()-t0)
self.__op_stat.update(packet.method, dt)
def after_process(self, packet, ret_packet):
"""process some logic after response is send"""
if packet.method == KEEP_ALIVE_METHOD:
return
operation_obj = self.__operations.get(packet.method, None)
if operation_obj is None:
return
try:
operation_obj.after_process(packet, ret_packet)
except Exception, err:
