def do_work(self, frames):
header, body = frames
hdr = common_pb2.Header()
oracle = db_connection.Oracle()
try:
hdr.ParseFromString(header)
# 통 문장 분석 시
# stt_result = stt_pb2.SttResult()
# 다중 문장 분석 시
stt_result = stt_pb2.SttResultDetail()
stt_result.ParseFromString(body)
client = nlp_client.NlpClient()
hdr.status_id = util.ProcStatus.PS_INPROGRESS
oracle = db_connection.Oracle()
util.insert_proc_result(oracle, self.logger, hdr)
level1, level2 = hdr.model_params.split('_')
rc = client.get_provider()
if rc is not None:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, rc)
return
nlp_result_list = nlp_pb2.nlpResultDetail()
nlp_document = brain_nlp_pb2.Document()
sentence_cnt = 0
for sentence in stt_result.sentence:
sentence_cnt += 1
result = client.analyze(sentence.sentence, int(level1),
int(level2))
nlp_result_id = record_result(oracle, hdr, sentence, result)
nlp_result_list.documentList.add(nlp_result_id=nlp_result_id,
sentence=sentence.sentence,
document=result[2])
record_result_detail(oracle, hdr, result, nlp_result_id)
print "{0} NLP Process Success".format(sentence_cnt)
self.log(logging.INFO,
"{0} NLP Process Success".format(sentence_cnt))
# Insert CS_PERFECT_SALES_CALL_LIST_TB
if hdr.call_metadata.speaker_code == 'ST0002':
if hdr.call_metadata.business_dcd in ['CJ0016', 'CJ0017']:
insert_perfect_sales_tb(oracle, hdr)
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
new_body = nlp_result_list.SerializeToString()
self.sendto_router_ex(hdr, new_body)
except grpc.RpcError as e:
print 'grpc.RpcError: {0}'.format(e)
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, e)
except Exception:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())
finally:
oracle.disconnect()
def do_work(self, frames):
print 'stt receive frames ({0})'.format(frames)
header, body = frames
hdr = common_pb2.Header()
hdr.ParseFromString(header)
audio = common_pb2.Audio()
audio.ParseFromString(body)
print 'Received request {0}, {1}'.format(hdr, audio)
client = stt_client.SttClient(self.stt_addr)
img_name, img_lang, img_sample_rate = hdr.model_params.split('-')
hdr.status_id = util.ProcStatus.PS_INPROGRESS
oracle = db_connection.Oracle()
util.insert_proc_result(oracle, self.logger, hdr)
if client.get_servers(_name=img_name, _lang=img_lang, _sample_rate=int(img_sample_rate)):
pcm_filename = audio.filename
if pcm_filename.endswith('wav'):
# 화자 분리
# https://www.nesono.com/node/275
# sox infile.wav outfile.l.wav remix 1
# wav 파일 pcm 파일로 전환, sox 에서는 raw 파일이 pcm 으로 취급
pcm_filename = audio.filename[:-3] + 'raw'
tfm = sox.Transformer()
# 같은 파일이 다른 모델로 동시에 처리될 때
if os.path.exists(pcm_filename):
pass
else:
tfm.build(audio.filename, pcm_filename)
result = client.detail_recognize(pcm_filename)
if pcm_filename != audio.filename:
print pcm_filename, audio.filename
# TODO : CS 후처리
# os.remove(pcm_filename)
self.complete_cnt += 1
# STT_RESULT 테이블에 기록
stt_result_id = insert_stt_result(oracle, hdr, result)
stt_result_detail = self.insert_stt_result_detail(oracle, hdr, stt_result_id, result)
# FLOW_RESULT 테이블에 기록
hdr.status_id = util.ProcStatus.PS_COMPLETED
hdr.stt_result_id = stt_result_id
util.insert_proc_result(oracle, self.logger, hdr)
# 다음 처리할 프로세스로 데이타 전달
self.log(logging.DEBUG, "stt result is {0}".format(result.txt))
# self.sendto_router(info.flow_set, info.flow_id, info.call_id, result)
# 여러문장 동시 전송
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for sentence in stt_result_detail:
print sentence
stt_result.sentence.add(stt_result_detail_id=sentence[0], sequence=sentence[1], sentence=sentence[2])
# 전체 문장 한꺼번에 전송
# stt_result = stt_pb2.SttResult()
# stt_result.result = result.txt
print stt_result.SerializeToString()
new_body = stt_result.SerializeToString()
self.sendto_router_ex(hdr, new_body)
else:
self.log(logging.DEBUG, 'CS Service unavailable')
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
oracle.disconnect()
def do_work(self, frames):
header, body = frames
hdr = common_pb2.Header()
oracle = db_connection.Oracle()
try:
hdr.ParseFromString(header)
# 통 문장 분석 시
# stt_result = stt_pb2.SttResult()
# 다중 문장 분석 시
stt_result = stt_pb2.SttResultDetail()
stt_result.ParseFromString(body)
client = nlp_client.NlpClient()
hdr.status_id = util.ProcStatus.PS_INPROGRESS
oracle = db_connection.Oracle()
util.insert_proc_result(oracle, self.logger, hdr)
level1, level2 = hdr.model_params.split('_')
rc = client.get_provider()
if rc is not None:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, rc)
return
nlp_result_list = nlp_pb2.nlpResultDetail()
nlp_document = brain_nlp_pb2.Document()
sentence_cnt = 0
for sentence in stt_result.sentence:
sentence_cnt += 1
try:
result = client.analyze(sentence.sentence, int(level1),
int(level2))
except Exception:
try:
result = client.analyze(sentence.sentence, int(level1),
int(level2))
except Exception:
try:
result = client.analyze(sentence.sentence,
int(level1), int(level2))
except Exception:
self.log(
logging.ERROR,
"REC_ID = {0}, RFILE_NAME = {1}".format(
hdr.call_metadata.record_key,
hdr.call_metadata.file_name))
self.log(logging.ERROR, traceback.format_exc())
continue
nlp_result_id = record_result(oracle, hdr, sentence, result)
nlp_result_list.documentList.add(
nlp_result_id=nlp_result_id,
sequence=sentence.sequence,
sentence=sentence.sentence,
document=result[2],
start_time=sentence.start_time,
end_time=sentence.end_time,
stt_result_detail_id=sentence.stt_result_detail_id)
try:
record_result_detail(oracle, hdr, result, nlp_result_id)
except Exception:
continue
self.log(logging.INFO,
"{0} NLP Process Success".format(sentence_cnt))
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
new_body = nlp_result_list.SerializeToString()
self.sendto_router_ex(hdr, new_body)
except grpc.RpcError:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(
logging.ERROR, "REC_ID = {0}, RFILE_NAME = {1}".format(
hdr.call_metadata.record_key, hdr.call_metadata.file_name))
self.log(logging.ERROR, traceback.format_exc())
except Exception:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())
finally:
oracle.disconnect()
def do_work(self, frames):
""" BizWorker 에서 호출하는 callback 함수"""
header, body = frames
hdr = common_pb2.Header()
oracle = db_connection.Oracle()
st = datetime.now()
try:
hdr.ParseFromString(header)
hmd_client_obj = hmd_client.HmdClient(self.hmd_addr)
project_code = hdr.call_metadata.project_code
call_type_code = hdr.call_metadata.call_type_code
speaker_code = hdr.call_metadata.speaker_code
if speaker_code == 'ST0001':
# RX(LEFT) 파일 고객
file_dcd = 'FS0001'
elif speaker_code == 'ST0002':
# TX(RIGHT) 파일 상담사
file_dcd = 'FS0002'
else:
file_dcd = 'FS0003'
# PROJECT_CODE(CS OR TM), CALL_TYPE_CODE(IN OR OUTBOUND CALL), FILE_DCD(화자)
# 기준으로 TA_HMD_MODEL_TB 에서 수행해야 할 HMD 목록을 가져온다.
hmd_list = select_hmd_list(oracle, project_code, call_type_code,
file_dcd)
sentence_cnt = 0
if hdr.model_params in hmd_list:
nlp_result_detail = nlp_pb2.nlpResultDetail()
nlp_result_detail.ParseFromString(body)
# 관심 키워드
if hdr.model_params in [
'interest_keyword_cs_hmd', 'interest_keyword_tm_hmd'
]:
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, hdr.model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
insert_interest_hmd_result(oracle, hmd_result,
document, hdr)
# 호전환 콜
elif hdr.model_params == 'calltransfer_hmd':
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, hdr.model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
insert_transfer_call_analysis_hmd_result(
oracle, hmd_result, hdr, document)
break
# 관심 구간
elif hdr.model_params in [
'interest_section_cs_hmd', 'interest_section_tm_hmd'
]:
hmd_result_list = list()
start_point_dict = dict()
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, hdr.model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
hmd_result_list.append((hmd_result, document))
for cls in hmd_result.keys():
cat_list = cls.split('_')
if len(cat_list) != 3:
continue
section_code, section_number, cust_yn = cat_list
if section_number == '001':
if not section_code in start_point_dict:
start_point_dict[section_code] = [
document.sequence,
document.start_time
]
insert_interest_section_hmd_result(oracle, hmd_result_list,
hdr, start_point_dict,
self.log)
# 상담 유형 분류, 상담 유형별 스크립트 미준수율
elif hdr.model_params in [
'call_driver_classify_cs_hmd',
'call_driver_classify_tm_hmd'
]:
total_category_dict = dict()
if hdr.model_params == 'call_driver_classify_cs_hmd':
name = 'call_driver_classify_cs'
else:
name = 'call_driver_classify_tm'
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, hdr.model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
# 상담 유형 분류
category_dict = insert_call_driver_classify_hmd_result(
oracle, hmd_result, document, hdr, name)
for category in category_dict.keys():
if category not in total_category_dict:
total_category_dict[category] = 1
if total_category_dict:
# 상담 유형별 스크립트 미준수율
category_list = total_category_dict.keys()
execute_call_driver_classify_script_quality(
oracle, category_list, nlp_result_detail, hdr,
hmd_client_obj)
# 에러 콜
elif hdr.model_params == 'error_call_hmd':
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, hdr.model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
insert_error_call_hmd_result(
oracle, hmd_result, hdr, document)
break
# 상담 품질 필수 문장
elif hdr.model_params == 'require_sentence_hmd':
hmd_result_list = list()
if hdr.call_metadata.call_type_code == 'CT0001':
model_params = 'require_sentence_in_hmd'
else:
model_params = 'require_sentence_out_hmd'
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
hmd_result_list.append((hmd_result, document))
insert_require_sentence_hmd_result(oracle, hmd_result_list,
hdr)
# 부정/민원/불만
elif hdr.model_params in [
'negative_keyword_cs_hmd', 'negative_keyword_tm_hmd'
]:
for document in nlp_result_detail.documentList:
hmd_result = execute_hmd(document, hdr.model_params)
if len(hmd_result) > 0:
sentence_cnt += 1
insert_negative_keyword_hmd_result(
oracle, hmd_result, document, hdr)
self.log(
logging.INFO,
"{0} {1} HMD Process REQUIRED TIME = {2}".format(
sentence_cnt, hdr.model_params,
str(datetime.now() - st)))
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
oracle.conn.commit()
except grpc.RpcError as e:
oracle.conn.rollback()
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, e)
except Exception:
oracle.conn.rollback()
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())
finally:
oracle.disconnect()
def do_cs_job(self):
oracle = db_connection.Oracle()
# Select CS target list
cs_list = get_cs_list(oracle, self.select_cnt)
# Extract holiday list
holiday_list = select_holiday(oracle)
call_id = ''
rec_id = ''
rfile_name = ''
for item in cs_list:
try:
st = datetime.now()
(
rec_id,
rfile_name,
) = item
# Get call meta information
meta_info, stt_spch_sped_rx, stt_spch_sped_tx = get_cs_meta_info(
oracle, rfile_name, rec_id)
if not meta_info:
self.logger.error(
"Can't select CS meta information. [REC_ID = {0}, RFILE_NAME = {0}]"
.format(rec_id, rfile_name))
continue
call_date = meta_info[0].strftime(
'%Y-%m-%d') if meta_info[0] else ''
start_time = meta_info[6]
ruser_id = meta_info[10]
# Extract TEAMCD
team_cd = select_code_from_cm_call_meta_tb(oracle, ruser_id)
meta_info += team_cd if team_cd else ('', )
# Extract OFFICE_HOUR
if call_date.replace("-", "") in holiday_list:
office_hour = 'OH0003' if 9 <= start_time.hour < 18 else 'OH0004'
elif datetime.strptime(call_date,
'%Y-%m-%d').weekday() in [5, 6]:
office_hour = 'OH0003' if 9 <= start_time.hour < 18 else 'OH0004'
else:
office_hour = 'OH0001' if 9 <= start_time.hour < 18 else 'OH0002'
meta_info += (office_hour, )
# Create CALL_ID
call_id = create_call_id(oracle, meta_info)
# Get stt result
stt_result = get_cs_stt_result(oracle, rfile_name, rec_id)
for stt_data in stt_result:
# Create ticket id
ticket_id = create_ticket_id(oracle, call_date, call_id)
for pipeline_id, tree in self.tree_map.iteritems():
root = tree.get_node(tree.root)
if root.data[9] == 'PT0006':
# Create pipeline_event_id
pipeline_event_id = create_pipeline_event_id(
oracle, call_date, pipeline_id, ticket_id)
if stt_data[4] == 'ST0001':
sc = 'C'
elif stt_data[4] == 'ST0002':
sc = 'A'
else:
sc = 'M'
speed = stt_spch_sped_rx if sc == 'C' else stt_spch_sped_tx
# Insert stt result
stt_result_id = insert_stt_result(
oracle, call_date, stt_data, call_id,
ticket_id, pipeline_event_id, speed)
# Get stt detail
stt_detail_list = get_cs_stt_detail(
oracle, rfile_name, sc, rec_id)
# Insert stt_detail_list
stt_result_detail = insert_stt_detail(
oracle, call_date, stt_result_id,
stt_detail_list)
# Create message
hdr = self.create_common_header(
pipeline_id=pipeline_id,
router_id=root.identifier,
proc_id=root.data[1],
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
meta_info=meta_info,
speaker_code=stt_data[4])
hdr.stt_result_id = stt_result_id
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for sentence in stt_result_detail:
stt_result.sentence.add(
stt_result_detail_id=sentence[0],
sequence=sentence[1],
sentence=sentence[2],
start_time=sentence[3],
end_time=sentence[4])
self.send_to_router(hdr.SerializeToString(),
stt_result.SerializeToString())
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'Y', rec_id)
oracle.conn.commit()
self.logger.info(
"[SUCCESS] CALL_ID : {0}, REC_ID : {1}, RFILE_NAME : {2}, REQUIRED TIME = {3}"
.format(call_id, rec_id, rfile_name,
str(datetime.now() - st)))
except Exception:
oracle.conn.rollback()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'F', rec_id)
oracle.conn.commit()
self.logger.error(traceback.format_exc())
self.logger.error(
"[ERROR] CALL_ID : {0}, REC_ID : {1}, RFILE_NAME : {2}".
format(call_id, rec_id, rfile_name))
continue
oracle.disconnect()
def main():
"""
Router process
"""
global LOGGER
LOGGER.info('Router Process started...')
conf = Config()
conf.init('biz.conf')
backend_map = dict()
pipeline_info = util.create_pipeline_info()
tree_map = pipeline_info['tree']
category_list = [
('STT', conf.get('stt.pull.port'), conf.get('stt.router.port')),
('HMD', conf.get('hmd.pull.port'), conf.get('hmd.router.port')),
('NLP', conf.get('nlp.pull.port'), conf.get('nlp.router.port')),
('DNN', conf.get('dnn.pull.port'), conf.get('dnn.router.port'))
]
process_list = list()
for category, frontend_port, backend_port in category_list:
p = BizRouter(category, frontend_port, backend_port)
p.start()
backend_map[category] = zmq.Context().socket(zmq.PUSH)
backend_map[category].connect(
'tcp://127.0.0.1:{0}'.format(frontend_port))
process_list.append(p)
# PULLER
frontend = zmq.Context().socket(zmq.PULL)
# For clients
frontend.bind("tcp://*:{0}".format(conf.get('router.pull.port')))
cli_socket = zmq.Context().socket(zmq.REP)
cli_socket.bind('ipc:///tmp/biz_cli.zmq')
poller = zmq.Poller()
poller.register(frontend, zmq.POLLIN)
poller.register(cli_socket, zmq.POLLIN)
while True:
oracle = db_connection.Oracle()
socks = dict(poller.poll(1 * 1000))
if socks.get(frontend) == zmq.POLLIN:
header, body = frontend.recv_multipart()
hdr = common_pb2.Header()
hdr.ParseFromString(header)
LOGGER.debug('FRONT(PULL) received info : {0}'.format(hdr))
# Select tree
tree = tree_map[hdr.pipeline_id]
for child in tree.children(hdr.router_id):
# data: model_params, proc_id, proc_name, router_id, model_id, config_id, proc_meta, model_name, proc_type_code
model_params = child.data[0]
proc_id = child.data[1]
proc_name = child.data[2]
# router_id = child.data[3]
# model_id = child.data[4]
# config_id = child.data[5]
# config_meta = child.data[6]
# proc_meta = child.data[7]
# model_name = child.data[8]
# proc_type_code = child.data[9]
# Make Proto buf Message
hdr.router_id = child.identifier
hdr.proc_id = proc_id
hdr.status_id = util.ProcStatus.PS_WAITING
hdr.model_params = model_params
header = hdr.SerializeToString()
LOGGER.debug('NEXT FLOW info : {0}'.format(hdr))
if 'HMD' in proc_name:
proc_name = 'HMD'
backend_map[proc_name].send_multipart([header, body])
util.insert_proc_result(oracle, LOGGER, hdr)
LOGGER.debug('Sent message to worker')
elif socks.get(cli_socket) == zmq.POLLIN:
cmd = cli_socket.recv()
LOGGER.info('Get cmd : {0}'.format(cmd))
if cmd == 'reload':
pipeline_info = util.create_pipeline_info()
tree_map = pipeline_info['tree']
cli_socket.send('success')
else:
LOGGER.error('Invalid Command')
cli_socket.send('failed')
oracle.disconnect()
def do_cs_job(self):
oracle = db_connection.Oracle()
# Select CS target list
cs_list = get_cs_list(oracle, self.select_cnt)
for item in cs_list:
call_id = ""
(rfile_name, ) = item
try:
# Get call meta information
meta_info_temp = get_cs_meta_info(oracle, rfile_name)
if not meta_info_temp:
self.logger.error(
"Can't select CS meta information. [RFILE_NAME = {0}]".
format(rfile_name))
continue
stt_spch_sped_rx = meta_info_temp[28]
stt_spch_sped_tx = meta_info_temp[29]
meta_info = meta_info_temp[:28]
# Insert meta information
call_id = create_call_id(oracle, meta_info)
print '[START] CALL_ID : {0}, RFILE_NAME : {1}'.format(
call_id, rfile_name)
self.logger.info(
'[START] CALL_ID : {0}, RFILE_NAME : {1}'.format(
call_id, rfile_name))
# Get stt result
stt_result = get_cs_stt_result(oracle, rfile_name)
for stt_data in stt_result:
# Create ticket id
ticket_id = create_ticket_id(oracle, call_id, 'CS_REC',
'CS_REC')
for pipeline_id, tree in self.tree_map.iteritems():
root = tree.get_node(tree.root)
if root.data[9] == 'PT0006':
# Create pipeline_event_id
pipeline_event_id = create_pipeline_event_id(
oracle, pipeline_id, ticket_id, meta_info[24],
meta_info[25])
if stt_data[4] == 'ST0001':
sc = 'C'
elif stt_data[4] == 'ST0002':
sc = 'A'
else:
sc = 'M'
speed = stt_spch_sped_rx if sc == 'C' else stt_spch_sped_tx
# Insert stt result
stt_result_id = insert_stt_result(
oracle, stt_data, call_id, ticket_id,
pipeline_event_id, speed)
# Get stt detail
stt_detail_list = get_cs_stt_detail(
oracle, rfile_name, sc)
# Insert stt_detail_list
stt_result_detail = insert_stt_detail(
oracle, stt_result_id, stt_detail_list)
# Create message
hdr = self.create_common_header(
pipeline_id=pipeline_id,
router_id=root.identifier,
proc_id=root.data[1],
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
meta_info=meta_info,
speaker_code=stt_data[4])
hdr.stt_result_id = stt_result_id
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for sentence in stt_result_detail:
stt_result.sentence.add(
stt_result_detail_id=sentence[0],
sequence=sentence[1],
sentence=sentence[2])
self.send_to_router(hdr.SerializeToString(),
stt_result.SerializeToString())
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'Y')
oracle.conn.commit()
print "[E N D] CALL_ID : {0}, RFILE_NAME : {1}".format(
call_id, rfile_name)
self.logger.info(
"[E N D] CALL_ID : {0}, RFILE_NAME : {1}".format(
call_id, rfile_name))
except Exception:
oracle.conn.rollback()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'F')
oracle.conn.commit()
self.logger.error(traceback.format_exc())
self.logger.error(
"[ERROR] CALL_ID : {0}, RFILE_NAME : {1}".format(
call_id, rfile_name))
continue
oracle.disconnect()
def do_work(self, frames):
header, body = frames
hdr = common_pb2.Header()
hdr.ParseFromString(header)
nlp_result = nlp_pb2.nlpResultDetail()
nlp_result.ParseFromString(body)
self.log(logging.INFO, "Message (pipeline_event_id:{0}) received".format(hdr.pipeline_event_id))
client = dnn_client.ClassifierClient(self.dnn_addr)
oracle = db_connection.Oracle()
try:
hdr.status_id = util.ProcStatus.PS_INPROGRESS
util.insert_proc_result(oracle, self.logger, hdr)
img_name, img_lang = hdr.model_params.split('-')
if client.get_server(img_name, img_lang):
detect_cnt = 0
call_date = '' if hdr.call_metadata.call_date == 'None' else hdr.call_metadata.call_date
for document in nlp_result.documentList:
nlp_id = document.nlp_result_id
nlp_result = document.sentence
summary = client.get_class(nlp_result)
summary.c1_top = summary.c1_top.replace("'", "\'")
if summary.c1_top == 'voc':
detect_cnt += 1
self.log(logging.DEBUG, "NLP_ID[{0}] DNN result is {1}".format(nlp_id, summary.c1_top))
dnn_classify_code = 'voc' if detect_cnt > 0 else 'normal'
query = """
INSERT INTO TA_DNN_RESULT_TB
(
CALL_DATE,
CALL_ID,
SPEAKER_CODE,
DNN_CLASSIFY_CODE,
DETECT_CNT,
CREATED_DTM,
UPDATED_DTM,
CREATOR_ID,
UPDATOR_ID
)
VALUES
(
TO_DATE(:1, 'YYYY/MM/DD'),
:2, :3, :4, :5, SYSDATE,
SYSDATE, :6, :7
)
"""
bind = (
call_date,
hdr.call_id,
hdr.call_metadata.speaker_code,
dnn_classify_code,
detect_cnt,
hdr.creator_id,
hdr.creator_id,
)
oracle.cursor.execute(query, bind)
oracle.conn.commit()
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
else:
self.log(logging.ERROR, "Classifier Service unavailable")
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
oracle.disconnect()
except grpc.RpcError as e:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, e)
except Exception:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())
def do_cs_job(self):
oracle = db_connection.Oracle()
# Select CS target list
self.logger.info("Get CS list")
cs_list = get_cs_list(oracle, self.select_cnt)
self.logger.info("Done get CS list")
for rec_id, rfile_name in cs_list:
st = datetime.now()
call_id = ''
try:
# Get call meta information
self.logger.info("Get call meta information")
meta_info, stt_spch_sped_rx, stt_spch_sped_tx, cntc_cls_double_dict, cntc_user_info = get_cs_meta_info(
oracle, rfile_name, rec_id)
self.logger.info("Done get call meta information")
if not meta_info:
self.logger.error(
"Can't select CS meta information. [REC_ID = {0}, RFILE_NAME = {1}]"
.format(rec_id, rfile_name))
continue
# Create CALL_ID
call_id = create_call_id(oracle, meta_info, cntc_user_info)
# Get STT result
stt_result = get_cs_stt_result(oracle, rfile_name, rec_id)
# Create CALL_DATE
call_date = meta_info[0].strftime(
'%Y-%m-%d') if meta_info[0] else ''
for stt_data in stt_result:
# Create ticket id
ticket_id = create_ticket_id(oracle, call_date, call_id)
for pipeline_id, tree in self.tree_map.iteritems():
root = tree.get_node(tree.root)
if root.data[9] == 'PT0006':
if stt_data[4] == 'ST0001':
sc = 'C'
elif stt_data[4] == 'ST0002':
sc = 'S'
else:
sc = 'M'
speed = stt_spch_sped_rx if sc == 'C' else stt_spch_sped_tx
# Create pipeline_event_id
pipeline_event_id = create_pipeline_event_id(
oracle, call_date, pipeline_id, ticket_id)
# Insert stt result
stt_result_id = insert_stt_result(
oracle=oracle,
call_date=call_date,
stt_data=stt_data,
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
speed=speed)
# Get stt detail
stt_detail_list = get_cs_stt_detail(
oracle, rfile_name, sc, rec_id)
# Insert stt_detail_list
stt_result_detail = insert_stt_detail(
oracle, call_id, call_date, stt_result_id,
stt_detail_list)
# Create message
hdr = self.create_common_header(
pipeline_id=pipeline_id,
router_id=root.identifier,
proc_id=root.data[1],
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
meta_info=meta_info,
speaker_code=stt_data[4],
cntc_user_info=cntc_user_info,
cntc_cls_double_dict=cntc_cls_double_dict)
hdr.stt_result_id = stt_result_id
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for item in stt_result_detail:
stt_result.sentence.add(
stt_result_detail_id=item[0],
sequence=item[1],
sentence=item[2],
start_time=item[3],
end_time=item[4])
self.send_to_router(hdr.SerializeToString(),
stt_result.SerializeToString())
oracle.conn.commit()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'Y', rec_id)
self.logger.info(
"[SUCCESS] CALL_DATE = {0}, CALL_ID : {1}, REC_ID : {2}, RFILE_NAME : {3},"
" REQUIRED TIME = {4}".format(call_date, call_id, rec_id,
rfile_name,
str(datetime.now() - st)))
except Exception:
oracle.conn.rollback()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'F', rec_id)
oracle.conn.commit()
self.logger.error(traceback.format_exc())
self.logger.error(
"[ERROR] CALL_ID : {0}, REC_ID : {1}, RFILE_NAME : {2}".
format(call_id, rec_id, rfile_name))
continue
time.sleep(1)
oracle.disconnect()
def do_work(self, frames):
header, body = frames
hdr = common_pb2.Header()
hdr.ParseFromString(header)
nlp_result = nlp_pb2.nlpResultDetail()
nlp_result.ParseFromString(body)
self.log(logging.INFO, "Message (pipeline_event_id:{0}) received".format(hdr.pipeline_event_id))
client = dnn_client.ClassifierClient(self.dnn_addr)
oracle = db_connection.Oracle()
try:
hdr.status_id = util.ProcStatus.PS_INPROGRESS
util.insert_proc_result(oracle, self.logger, hdr)
img_name, img_lang = hdr.model_params.split('-')
if client.get_server(img_name, img_lang):
for document in nlp_result.documentList:
nlp_id = document.nlp_result_id
nlp_result = document.sentence
summary = client.get_class(nlp_result)
summary.c1_top = summary.c1_top.replace("'", "\'")
query = """
INSERT INTO TA_DNN_RESULT_TB
(
NLP_RESULT_ID,
DNN_CLASSIFY_CODE,
PROBABILITY,
CREATED_DTM,
UPDATED_DTM,
CREATOR_ID,
UPDATOR_ID
)
VALUES
(
:1, :2, :3,
SYSDATE, SYSDATE,
:4, :5
)
"""
bind = (
nlp_id,
summary.c1_top,
summary.probability_top,
hdr.creator_id,
hdr.creator_id,
)
oracle.cursor.execute(query, bind)
oracle.conn.commit()
self.log(logging.DEBUG, "DNN result is {0}".format(summary))
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
else:
self.log(logging.ERROR, "Classifier Service unavailable")
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
oracle.disconnect()
except grpc.RpcError as e:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, e)
except Exception:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())