def run(self):
logger.debug('OSProcessesStatisticCollector is started!')
while True:
try:
for i in xrange(self.timeout):
time.sleep(1)
if self.stop_flag.is_set():
break
if self.stop_flag.is_set():
break
for pid_desr, pid in self.pid_list:
p_stat = self.get_process_stat(pid)
if p_stat:
self.operator_cl.update_statistic('%sProcStat'%pid_desr, pid, p_stat)
for workers_manager in self.workers_manager_list:
w_count, w_busy = workers_manager.get_workers_stat()
self.operator_cl.update_statistic('%sWMStat'%workers_manager.get_workers_name(), \
'WM', {'workers': (None, w_count), 'busy': (None, w_busy)})
for child in workers_manager.iter_children():
pid = child.pid
p_stat = self.get_process_stat(pid)
if p_stat:
self.operator_cl.update_statistic('%sProcStat'%workers_manager.get_workers_name(), pid, p_stat)
except Exception, err:
import traceback
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error(str(err))
def run(self):
self.before_start()
logger.debug('worker is started!')
while True:
data = self.__queue.get()
if data == STOP_WORKER_EVENT:
break
self.__busy_flag.set()
try:
self.worker_routine(data)
except Exception, err:
logger.error(err)
logger.write = logger.debug
traceback.print_exc(file=logger)
finally:
def process(self, packet, role):
"""process request fabnet packet
@param packet - object of FabnetPacketRequest class
@param role - requestor role (None for disable auth)
"""
t0 = None
try:
rcode = self.operator_cl.register_request(packet.message_id, packet.method, packet.sender)
if rcode == RC_ALREADY_PROCESSED:
return
if packet.method == KEEP_ALIVE_METHOD:
return FabnetPacketResponse(ret_code=rcode)
if self.__op_stat is not None:
t0 = datetime.now()
operation_obj = self.__operations.get(packet.method, None)
if operation_obj is None:
if packet.is_multicast:
#transit packet
self.operator_cl.call_to_neighbours(packet.message_id, packet.method, packet.parameters, packet.is_multicast)
return
else:
raise Exception('Method "%s" does not implemented! Available methods: %s'%(packet.method, self.__operations.keys()))
operation_obj.check_role(role)
packet.role = role
logger.debug('processing %s'%packet)
message_id = packet.message_id
n_packet = operation_obj.before_resend(packet)
if n_packet:
self.operator_cl.call_to_neighbours(message_id, packet.method, packet.parameters, packet.is_multicast)
s_packet = operation_obj.process(packet)
if s_packet:
s_packet.message_id = packet.message_id
s_packet.from_node = self.__self_address
return s_packet
except PermissionDeniedException, err:
return FabnetPacketResponse(from_node=self.__self_address,
message_id=packet.message_id, ret_code=RC_PERMISSION_DENIED,
ret_message='Permission denied!')
def __spawn_worker(self):
logger.debug('starting new worker')
if self.stopped.is_set():
return
self.__lock.acquire()
try:
if len(self.__workers) == self.max_count:
return
worker_name = '%s-%s#%02i' % (self.server_name, self.worker_class.__name__, self.__workers_idx)
worker = self.worker_class(worker_name, self.queue, *self.init_params)
self.__workers_idx += 1
worker.start()
self.__workers.append(worker)
finally:
self.__lock.release()
def __init__(self, operations_classes, server_name, key_storage=None, op_auth_key=None):
self.__operations = {}
self.operator_cl = OperatorClient(server_name, op_auth_key)
self.__self_address = self.operator_cl.get_self_address()
home_dir = self.operator_cl.get_home_dir()
self.__key_storage = key_storage
self.__fri_client = FriClient(key_storage)
for op_class in operations_classes:
if not issubclass(op_class, OperationBase):
raise Exception('Class %s does not inherit OperationBase class'%op_class)
logger.debug('registering %s operation class...'% op_class.__name__)
lock = RLock()
operation = op_class(self.operator_cl, self.__fri_client, self.__self_address, home_dir, lock)
operation.get_operation_object = self.__get_operation_object
self.__operations[op_class.get_name()] = operation
def __init__(self, operations_classes, server_name, key_storage=None):
self.__operations = {}
self.__op_stat = None
self.operator_cl = OperatorClient(server_name)
self.__self_address = self.operator_cl.get_self_address()
home_dir = self.operator_cl.get_home_dir()
if key_storage:
cert = key_storage.get_node_cert()
ckey = key_storage.get_node_cert_key()
else:
cert = ckey = None
self.__fri_client = FriClient(bool(cert), cert, ckey)
for op_class in operations_classes:
if not issubclass(op_class, OperationBase):
raise Exception('Class %s does not inherit OperationBase class'%op_class)
logger.debug('registering %s operation class...'% op_class.__name__)
lock = RLock()
operation = op_class(self.operator_cl, self.__fri_client, self.__self_address, home_dir, lock)
self.__operations[op_class.get_name()] = operation
def __stop_worker(self):
if self.stopped.is_set():
return
self.__lock.acquire()
try:
for_delete = []
for i, worker in enumerate(self.__workers):
if not worker.is_alive():
for_delete.append(worker)
for thr in for_delete:
del self.__workers[self.__workers.index(thr)]
if len(self.__workers) <= self.min_count:
logger.debug('trying stopping worker but min workers count occured')
return
self.queue.put(STOP_WORKER_EVENT)
finally:
self.__lock.release()
logger.debug('stopped one worker')
def run(self):
logger.debug('StatisticCollector is started!')
last_stat = None
while True:
try:
for i in xrange(self.timeout):
time.sleep(1)
if self.stop_flag.is_set():
break
if self.stop_flag.is_set():
break
#send statistic to operator...
stat = self.stat_map.dump()
if stat == last_stat:
continue
last_stat = stat
self.operator_cl.update_statistic(self.stat_obj, self.stat_owner, stat)
except Exception, err:
import traceback
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error(str(err))
if data == STOP_WORKER_EVENT:
break
self.__busy_flag.set()
try:
self.worker_routine(data)
except Exception, err:
logger.error(err)
logger.write = logger.debug
traceback.print_exc(file=logger)
finally:
#self.__queue.task_done()
self.__busy_flag.clear()
self.after_stop()
logger.debug('worker is stopped!')
def worker_routine(self, data):
"""This method must be implemented in inherited class"""
raise RuntimeError('Not implemented')
def before_start(self):
"""This method can be implemented in inherited class"""
pass
def after_stop(self):
"""This method can be implemented in inherited class"""
pass
class ProcessBasedAbstractWorker(mp.Process):