def __init__(self): self.context = zmq.Context() self.socket = self.context.socket(zmq.PUSH) self.logger = logger.get_timed_rotating_logger( logger_name='cs_db_collector', file_name=os.path.join(os.getenv('MAUM_ROOT'), 'logs/cs_db_collector_process.log'), backup_count=5, log_level='debug') self.conf = Config() self.conf.init('biz.conf') self.pipeline_info = util.create_pipeline_info() self.tree_map = self.pipeline_info['tree'] self.select_cnt = 10000
def main(): """ Router process """ global LOGGER LOGGER.info('Router Process started...') conf = Config() conf.init('biz.conf') backend_map = dict() pipeline_info = util.create_pipeline_info() tree_map = pipeline_info['tree'] category_list = [ ('STT', conf.get('stt.pull.port'), conf.get('stt.router.port')), ('HMD', conf.get('hmd.pull.port'), conf.get('hmd.router.port')), ('NLP', conf.get('nlp.pull.port'), conf.get('nlp.router.port')), ('DNN', conf.get('dnn.pull.port'), conf.get('dnn.router.port')) ] process_list = list() for category, frontend_port, backend_port in category_list: p = BizRouter(category, frontend_port, backend_port) p.start() backend_map[category] = zmq.Context().socket(zmq.PUSH) backend_map[category].connect( 'tcp://127.0.0.1:{0}'.format(frontend_port)) process_list.append(p) # PULLER frontend = zmq.Context().socket(zmq.PULL) # For clients frontend.bind("tcp://*:{0}".format(conf.get('router.pull.port'))) cli_socket = zmq.Context().socket(zmq.REP) cli_socket.bind('ipc:///tmp/biz_cli.zmq') poller = zmq.Poller() poller.register(frontend, zmq.POLLIN) poller.register(cli_socket, zmq.POLLIN) while True: oracle = db_connection.Oracle() socks = dict(poller.poll(1 * 1000)) if socks.get(frontend) == zmq.POLLIN: header, body = frontend.recv_multipart() hdr = common_pb2.Header() hdr.ParseFromString(header) LOGGER.debug('FRONT(PULL) received info : {0}'.format(hdr)) # Select tree tree = tree_map[hdr.pipeline_id] for child in tree.children(hdr.router_id): # data: model_params, proc_id, proc_name, router_id, model_id, config_id, proc_meta, model_name, proc_type_code model_params = child.data[0] proc_id = child.data[1] proc_name = child.data[2] # router_id = child.data[3] # model_id = child.data[4] # config_id = child.data[5] # config_meta = child.data[6] # proc_meta = child.data[7] # model_name = child.data[8] # proc_type_code = child.data[9] # Make Proto buf Message hdr.router_id = child.identifier hdr.proc_id = proc_id hdr.status_id = util.ProcStatus.PS_WAITING hdr.model_params = model_params header = hdr.SerializeToString() LOGGER.debug('NEXT FLOW info : {0}'.format(hdr)) if 'HMD' in proc_name: proc_name = 'HMD' backend_map[proc_name].send_multipart([header, body]) util.insert_proc_result(oracle, LOGGER, hdr) LOGGER.debug('Sent message to worker') elif socks.get(cli_socket) == zmq.POLLIN: cmd = cli_socket.recv() LOGGER.info('Get cmd : {0}'.format(cmd)) if cmd == 'reload': pipeline_info = util.create_pipeline_info() tree_map = pipeline_info['tree'] cli_socket.send('success') else: LOGGER.error('Invalid Command') cli_socket.send('failed') oracle.disconnect()