def __init__(self):
self.context = zmq.Context()
self.socket = self.context.socket(zmq.PUSH)
self.logger = logger.get_timed_rotating_logger(
logger_name='cs_db_collector',
file_name=os.path.join(os.getenv('MAUM_ROOT'),
'logs/cs_db_collector_process.log'),
backup_count=5,
log_level='debug')
self.conf = Config()
self.conf.init('biz.conf')
self.pipeline_info = util.create_pipeline_info()
self.tree_map = self.pipeline_info['tree']
self.select_cnt = 10000
def main():
"""
Router process
"""
global LOGGER
LOGGER.info('Router Process started...')
conf = Config()
conf.init('biz.conf')
backend_map = dict()
pipeline_info = util.create_pipeline_info()
tree_map = pipeline_info['tree']
category_list = [
('STT', conf.get('stt.pull.port'), conf.get('stt.router.port')),
('HMD', conf.get('hmd.pull.port'), conf.get('hmd.router.port')),
('NLP', conf.get('nlp.pull.port'), conf.get('nlp.router.port')),
('DNN', conf.get('dnn.pull.port'), conf.get('dnn.router.port'))
]
process_list = list()
for category, frontend_port, backend_port in category_list:
p = BizRouter(category, frontend_port, backend_port)
p.start()
backend_map[category] = zmq.Context().socket(zmq.PUSH)
backend_map[category].connect(
'tcp://127.0.0.1:{0}'.format(frontend_port))
process_list.append(p)
# PULLER
frontend = zmq.Context().socket(zmq.PULL)
# For clients
frontend.bind("tcp://*:{0}".format(conf.get('router.pull.port')))
cli_socket = zmq.Context().socket(zmq.REP)
cli_socket.bind('ipc:///tmp/biz_cli.zmq')
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(cli_socket, zmq.POLLIN)
while True:
oracle = db_connection.Oracle()
socks = dict(poller.poll(1 * 1000))
if socks.get(frontend) == zmq.POLLIN:
header, body = frontend.recv_multipart()
hdr = common_pb2.Header()
hdr.ParseFromString(header)
LOGGER.debug('FRONT(PULL) received info : {0}'.format(hdr))
# Select tree
tree = tree_map[hdr.pipeline_id]
for child in tree.children(hdr.router_id):
# data: model_params, proc_id, proc_name, router_id, model_id, config_id, proc_meta, model_name, proc_type_code
model_params = child.data[0]
proc_id = child.data[1]
proc_name = child.data[2]
# router_id = child.data[3]
# model_id = child.data[4]
# config_id = child.data[5]
# config_meta = child.data[6]
# proc_meta = child.data[7]
# model_name = child.data[8]
# proc_type_code = child.data[9]
# Make Proto buf Message
hdr.router_id = child.identifier
hdr.proc_id = proc_id
hdr.status_id = util.ProcStatus.PS_WAITING
hdr.model_params = model_params
header = hdr.SerializeToString()
LOGGER.debug('NEXT FLOW info : {0}'.format(hdr))
if 'HMD' in proc_name:
proc_name = 'HMD'
backend_map[proc_name].send_multipart([header, body])
util.insert_proc_result(oracle, LOGGER, hdr)
LOGGER.debug('Sent message to worker')
elif socks.get(cli_socket) == zmq.POLLIN:
cmd = cli_socket.recv()
LOGGER.info('Get cmd : {0}'.format(cmd))
if cmd == 'reload':
pipeline_info = util.create_pipeline_info()
tree_map = pipeline_info['tree']
cli_socket.send('success')
else:
LOGGER.error('Invalid Command')
cli_socket.send('failed')
oracle.disconnect()