def run(self): logger.debug('OSProcessesStatisticCollector is started!') while True: try: for i in xrange(self.timeout): time.sleep(1) if self.stop_flag.is_set(): break if self.stop_flag.is_set(): break for pid_desr, pid in self.pid_list: p_stat = self.get_process_stat(pid) if p_stat: self.operator_cl.update_statistic('%sProcStat'%pid_desr, pid, p_stat) for workers_manager in self.workers_manager_list: w_count, w_busy = workers_manager.get_workers_stat() self.operator_cl.update_statistic('%sWMStat'%workers_manager.get_workers_name(), \ 'WM', {'workers': (None, w_count), 'busy': (None, w_busy)}) for child in workers_manager.iter_children(): pid = child.pid p_stat = self.get_process_stat(pid) if p_stat: self.operator_cl.update_statistic('%sProcStat'%workers_manager.get_workers_name(), pid, p_stat) except Exception, err: import traceback logger.write = logger.debug traceback.print_exc(file=logger) logger.error(str(err))
def run(self): self.before_start() logger.debug('worker is started!') while True: data = self.__queue.get() if data == STOP_WORKER_EVENT: break self.__busy_flag.set() try: self.worker_routine(data) except Exception, err: logger.error(err) logger.write = logger.debug traceback.print_exc(file=logger) finally:
def process(self, packet, role): """process request fabnet packet @param packet - object of FabnetPacketRequest class @param role - requestor role (None for disable auth) """ t0 = None try: rcode = self.operator_cl.register_request(packet.message_id, packet.method, packet.sender) if rcode == RC_ALREADY_PROCESSED: return if packet.method == KEEP_ALIVE_METHOD: return FabnetPacketResponse(ret_code=rcode) if self.__op_stat is not None: t0 = datetime.now() operation_obj = self.__operations.get(packet.method, None) if operation_obj is None: if packet.is_multicast: #transit packet self.operator_cl.call_to_neighbours(packet.message_id, packet.method, packet.parameters, packet.is_multicast) return else: raise Exception('Method "%s" does not implemented! Available methods: %s'%(packet.method, self.__operations.keys())) operation_obj.check_role(role) packet.role = role logger.debug('processing %s'%packet) message_id = packet.message_id n_packet = operation_obj.before_resend(packet) if n_packet: self.operator_cl.call_to_neighbours(message_id, packet.method, packet.parameters, packet.is_multicast) s_packet = operation_obj.process(packet) if s_packet: s_packet.message_id = packet.message_id s_packet.from_node = self.__self_address return s_packet except PermissionDeniedException, err: return FabnetPacketResponse(from_node=self.__self_address, message_id=packet.message_id, ret_code=RC_PERMISSION_DENIED, ret_message='Permission denied!')
def __spawn_worker(self): logger.debug('starting new worker') if self.stopped.is_set(): return self.__lock.acquire() try: if len(self.__workers) == self.max_count: return worker_name = '%s-%s#%02i' % (self.server_name, self.worker_class.__name__, self.__workers_idx) worker = self.worker_class(worker_name, self.queue, *self.init_params) self.__workers_idx += 1 worker.start() self.__workers.append(worker) finally: self.__lock.release()
def __init__(self, operations_classes, server_name, key_storage=None, op_auth_key=None): self.__operations = {} self.operator_cl = OperatorClient(server_name, op_auth_key) self.__self_address = self.operator_cl.get_self_address() home_dir = self.operator_cl.get_home_dir() self.__key_storage = key_storage self.__fri_client = FriClient(key_storage) for op_class in operations_classes: if not issubclass(op_class, OperationBase): raise Exception('Class %s does not inherit OperationBase class'%op_class) logger.debug('registering %s operation class...'% op_class.__name__) lock = RLock() operation = op_class(self.operator_cl, self.__fri_client, self.__self_address, home_dir, lock) operation.get_operation_object = self.__get_operation_object self.__operations[op_class.get_name()] = operation
def __init__(self, operations_classes, server_name, key_storage=None): self.__operations = {} self.__op_stat = None self.operator_cl = OperatorClient(server_name) self.__self_address = self.operator_cl.get_self_address() home_dir = self.operator_cl.get_home_dir() if key_storage: cert = key_storage.get_node_cert() ckey = key_storage.get_node_cert_key() else: cert = ckey = None self.__fri_client = FriClient(bool(cert), cert, ckey) for op_class in operations_classes: if not issubclass(op_class, OperationBase): raise Exception('Class %s does not inherit OperationBase class'%op_class) logger.debug('registering %s operation class...'% op_class.__name__) lock = RLock() operation = op_class(self.operator_cl, self.__fri_client, self.__self_address, home_dir, lock) self.__operations[op_class.get_name()] = operation
def __stop_worker(self): if self.stopped.is_set(): return self.__lock.acquire() try: for_delete = [] for i, worker in enumerate(self.__workers): if not worker.is_alive(): for_delete.append(worker) for thr in for_delete: del self.__workers[self.__workers.index(thr)] if len(self.__workers) <= self.min_count: logger.debug('trying stopping worker but min workers count occured') return self.queue.put(STOP_WORKER_EVENT) finally: self.__lock.release() logger.debug('stopped one worker')
def run(self): logger.debug('StatisticCollector is started!') last_stat = None while True: try: for i in xrange(self.timeout): time.sleep(1) if self.stop_flag.is_set(): break if self.stop_flag.is_set(): break #send statistic to operator... stat = self.stat_map.dump() if stat == last_stat: continue last_stat = stat self.operator_cl.update_statistic(self.stat_obj, self.stat_owner, stat) except Exception, err: import traceback logger.write = logger.debug traceback.print_exc(file=logger) logger.error(str(err))
if data == STOP_WORKER_EVENT: break self.__busy_flag.set() try: self.worker_routine(data) except Exception, err: logger.error(err) logger.write = logger.debug traceback.print_exc(file=logger) finally: #self.__queue.task_done() self.__busy_flag.clear() self.after_stop() logger.debug('worker is stopped!') def worker_routine(self, data): """This method must be implemented in inherited class""" raise RuntimeError('Not implemented') def before_start(self): """This method can be implemented in inherited class""" pass def after_stop(self): """This method can be implemented in inherited class""" pass class ProcessBasedAbstractWorker(mp.Process):