def start(self, neighbour):
address = '%s:%s' % (self.hostname, self.port)
if not neighbour:
is_init_node = True
else:
is_init_node = False
operator_class = OPERATORS_MAP.get(self.node_type, None)
if operator_class is None:
logger.error('Node type "%s" does not found!'%self.node_type)
return False
op_proc = OperatorProcess(operator_class, address, self.home_dir, self.keystore, \
is_init_node, self.node_name, config=self.config)
op_proc.start_carefully()
try:
oper_manager = OperationsManager(operator_class.OPERATIONS_LIST, self.node_name, self.keystore)
workers_mgr = WorkersManager(OperationsProcessor, server_name=self.node_name, \
init_params=(oper_manager, self.keystore))
fri_server = FriServer(self.bind_host, self.port, workers_mgr, self.node_name)
started = fri_server.start()
if not started:
raise Exception('FriServer does not started!')
except Exception, err:
time.sleep(1)
op_proc.stop()
op_proc.join()
raise err
def callback(self, packet):
"""process callback fabnet packet
@param packet - object of FabnetPacketResponse class
"""
ret = self.operator_cl.register_callback(packet.message_id)
if ret == RC_MESSAGE_ID_NOT_FOUND:
raise Exception('MessageID does not found! Failed packet: %s'%packet)
operation_name, sender = ret
operation_obj = self.__operations.get(operation_name, None)
if operation_obj is None:
if sender:
#transit packet
self.send_to_sender(sender, packet)
return
s_packet = None
try:
s_packet = operation_obj.callback(packet, sender)
except Exception, err:
logger.error('%s callback failed. Details: %s'%(operation_name, err))
logger.write = logger.debug
traceback.print_exc(file=logger)
def run(self):
logger.info('Starting connection handler thread...')
self.__bind_socket()
logger.info('Connection handler thread started!')
while True:
try:
(sock, addr) = self.sock.accept()
except Exception, err:
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[FriConnectionHandler.accept crash] %s'%err)
break
try:
if self.stopped.is_set():
sock.close()
break
if self.need_reduce:
sock = reduce_handle(sock.fileno())
self.queue.put(sock)
except Exception, err:
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error('[FriConnectionHandler.run] %s'%err)
def run(self):
logger.debug('OSProcessesStatisticCollector is started!')
while True:
try:
for i in xrange(self.timeout):
time.sleep(1)
if self.stop_flag.is_set():
break
if self.stop_flag.is_set():
break
for pid_desr, pid in self.pid_list:
p_stat = self.get_process_stat(pid)
if p_stat:
self.operator_cl.update_statistic('%sProcStat'%pid_desr, pid, p_stat)
for workers_manager in self.workers_manager_list:
w_count, w_busy = workers_manager.get_workers_stat()
self.operator_cl.update_statistic('%sWMStat'%workers_manager.get_workers_name(), \
'WM', {'workers': (None, w_count), 'busy': (None, w_busy)})
for child in workers_manager.iter_children():
pid = child.pid
p_stat = self.get_process_stat(pid)
if p_stat:
self.operator_cl.update_statistic('%sProcStat'%workers_manager.get_workers_name(), pid, p_stat)
except Exception, err:
import traceback
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error(str(err))
def __bind_socket(self):
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self.sock.bind((self.hostname, self.port))
self.sock.listen(5)
except Exception, err:
self.status = S_ERROR
logger.error('[__bind_socket] %s'%err)
def after_process(self, packet, ret_packet):
"""process some logic after response is send"""
if packet.method == KEEP_ALIVE_METHOD:
return
operation_obj = self.__operations.get(packet.method, None)
if operation_obj is None:
return
try:
operation_obj.after_process(packet, ret_packet)
except Exception, err:
logger.error('%s after_process routine failed. Details: %s'%(packet.method, err))
logger.write = logger.debug
traceback.print_exc(file=logger)
def start(self):
self.stopped = False
self.workers_manager.start_carefully()
self.__conn_handler_thread.start()
while self.__conn_handler_thread.status == S_PENDING:
time.sleep(.1)
if self.__conn_handler_thread.status == S_ERROR:
self.stop()
logger.error('FriServer does not started!')
return False
else:
logger.info('FriServer is started!')
return True
def run(self):
self.before_start()
logger.info('worker is started!')
while True:
data = self.__queue.get()
if data == STOP_WORKER_EVENT:
break
self.__busy_flag.set()
try:
self.worker_routine(data)
except Exception, err:
logger.error(err)
logger.write = logger.debug
traceback.print_exc(file=logger)
finally:
def stop(self):
logger.info('stopping workers manager for %s...'%self.worker_class.__name__)
self.stopped.set()
self.__lock.acquire()
try:
act_count = 0
for worker in self.__workers:
if worker.is_alive():
act_count += 1
for i in xrange(act_count):
self.queue.put(STOP_WORKER_EVENT)
for worker in self.__workers:
if worker.is_alive():
worker.join()
except Exception, err:
logger.error('stopping error: %s'%err)
def run(self):
try:
cur_thread = threading.current_thread()
cur_thread.setName('%s-%s-Manager'%(self.server_name, self.worker_class.__name__))
for i in range(self.min_count):
self.__spawn_worker()
self.__lock.acquire()
self.__status = S_INWORK
self.__lock.release()
except Exception, err:
self.__lock.acquire()
self.__status = S_ERROR
self.__lock.release()
logger.error(err)
import traceback
logger.write = logger.debug
traceback.print_exc(file=logger)
raise err
def run(self):
cur_thread = threading.current_thread()
cur_thread.setName(self.__name)
self.before_start()
logger.info('worker is started!')
while True:
data = self.__queue.get()
if data == STOP_WORKER_EVENT:
break
self.__is_busy.value = True
try:
self.worker_routine(data)
except Exception, err:
logger.error(err)
logger.write = logger.debug
traceback.print_exc(file=logger)
finally:
def get_process_stat(cls, pid):
rss = threads = ''
if not os.path.exists('/proc/%i/status'%pid):
return None
try:
lines = open('/proc/%i/status'%pid,'r').readlines()
for line in lines:
(param, value) = line.split()[:2]
if param.startswith('VmRSS'):
rss = value.strip()
elif param.startswith('Threads'):
threads = value.strip()
procinfo = {}
procinfo['memory'] = int(rss)
procinfo['threads'] = (None, int(threads))
return procinfo
except Exception, err:
logger.error('get_process_stat failed: %s'%err)
return None
def run(self):
logger.debug('StatisticCollector is started!')
last_stat = None
while True:
try:
for i in xrange(self.timeout):
time.sleep(1)
if self.stop_flag.is_set():
break
if self.stop_flag.is_set():
break
#send statistic to operator...
stat = self.stat_map.dump()
if stat == last_stat:
continue
last_stat = stat
self.operator_cl.update_statistic(self.stat_obj, self.stat_owner, stat)
except Exception, err:
import traceback
logger.write = logger.debug
traceback.print_exc(file=logger)
logger.error(str(err))