def do_work(self, frames):
print 'stt receive frames ({0})'.format(frames)
header, body = frames
hdr = common_pb2.Header()
hdr.ParseFromString(header)
audio = common_pb2.Audio()
audio.ParseFromString(body)
print 'Received request {0}, {1}'.format(hdr, audio)
client = stt_client.SttClient(self.stt_addr)
img_name, img_lang, img_sample_rate = hdr.model_params.split('-')
hdr.status_id = util.ProcStatus.PS_INPROGRESS
oracle = db_connection.Oracle()
util.insert_proc_result(oracle, self.logger, hdr)
if client.get_servers(_name=img_name, _lang=img_lang, _sample_rate=int(img_sample_rate)):
pcm_filename = audio.filename
if pcm_filename.endswith('wav'):
# 화자 분리
# https://www.nesono.com/node/275
# sox infile.wav outfile.l.wav remix 1
# wav 파일 pcm 파일로 전환, sox 에서는 raw 파일이 pcm 으로 취급
pcm_filename = audio.filename[:-3] + 'raw'
tfm = sox.Transformer()
# 같은 파일이 다른 모델로 동시에 처리될 때
if os.path.exists(pcm_filename):
pass
else:
tfm.build(audio.filename, pcm_filename)
result = client.detail_recognize(pcm_filename)
if pcm_filename != audio.filename:
print pcm_filename, audio.filename
# TODO : CS 후처리
# os.remove(pcm_filename)
self.complete_cnt += 1
# STT_RESULT 테이블에 기록
stt_result_id = insert_stt_result(oracle, hdr, result)
stt_result_detail = self.insert_stt_result_detail(oracle, hdr, stt_result_id, result)
# FLOW_RESULT 테이블에 기록
hdr.status_id = util.ProcStatus.PS_COMPLETED
hdr.stt_result_id = stt_result_id
util.insert_proc_result(oracle, self.logger, hdr)
# 다음 처리할 프로세스로 데이타 전달
self.log(logging.DEBUG, "stt result is {0}".format(result.txt))
# self.sendto_router(info.flow_set, info.flow_id, info.call_id, result)
# 여러문장 동시 전송
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for sentence in stt_result_detail:
print sentence
stt_result.sentence.add(stt_result_detail_id=sentence[0], sequence=sentence[1], sentence=sentence[2])
# 전체 문장 한꺼번에 전송
# stt_result = stt_pb2.SttResult()
# stt_result.result = result.txt
print stt_result.SerializeToString()
new_body = stt_result.SerializeToString()
self.sendto_router_ex(hdr, new_body)
else:
self.log(logging.DEBUG, 'CS Service unavailable')
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
oracle.disconnect()
def do_work(self, frames):
header, body = frames
hdr = common_pb2.Header()
oracle = db_connection.Oracle()
try:
hdr.ParseFromString(header)
# 통 문장 분석 시
# stt_result = stt_pb2.SttResult()
# 다중 문장 분석 시
stt_result = stt_pb2.SttResultDetail()
stt_result.ParseFromString(body)
client = nlp_client.NlpClient()
hdr.status_id = util.ProcStatus.PS_INPROGRESS
oracle = db_connection.Oracle()
util.insert_proc_result(oracle, self.logger, hdr)
level1, level2 = hdr.model_params.split('_')
rc = client.get_provider()
if rc is not None:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, rc)
return
nlp_result_list = nlp_pb2.nlpResultDetail()
nlp_document = brain_nlp_pb2.Document()
sentence_cnt = 0
for sentence in stt_result.sentence:
sentence_cnt += 1
result = client.analyze(sentence.sentence, int(level1),
int(level2))
nlp_result_id = record_result(oracle, hdr, sentence, result)
nlp_result_list.documentList.add(nlp_result_id=nlp_result_id,
sentence=sentence.sentence,
document=result[2])
record_result_detail(oracle, hdr, result, nlp_result_id)
print "{0} NLP Process Success".format(sentence_cnt)
self.log(logging.INFO,
"{0} NLP Process Success".format(sentence_cnt))
# Insert CS_PERFECT_SALES_CALL_LIST_TB
if hdr.call_metadata.speaker_code == 'ST0002':
if hdr.call_metadata.business_dcd in ['CJ0016', 'CJ0017']:
insert_perfect_sales_tb(oracle, hdr)
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
new_body = nlp_result_list.SerializeToString()
self.sendto_router_ex(hdr, new_body)
except grpc.RpcError as e:
print 'grpc.RpcError: {0}'.format(e)
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, e)
except Exception:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())
finally:
oracle.disconnect()
def do_work(self, frames):
header, body = frames
hdr = common_pb2.Header()
oracle = db_connection.Oracle()
try:
hdr.ParseFromString(header)
# 통 문장 분석 시
# stt_result = stt_pb2.SttResult()
# 다중 문장 분석 시
stt_result = stt_pb2.SttResultDetail()
stt_result.ParseFromString(body)
client = nlp_client.NlpClient()
hdr.status_id = util.ProcStatus.PS_INPROGRESS
oracle = db_connection.Oracle()
util.insert_proc_result(oracle, self.logger, hdr)
level1, level2 = hdr.model_params.split('_')
rc = client.get_provider()
if rc is not None:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, rc)
return
nlp_result_list = nlp_pb2.nlpResultDetail()
nlp_document = brain_nlp_pb2.Document()
sentence_cnt = 0
for sentence in stt_result.sentence:
sentence_cnt += 1
try:
result = client.analyze(sentence.sentence, int(level1),
int(level2))
except Exception:
try:
result = client.analyze(sentence.sentence, int(level1),
int(level2))
except Exception:
try:
result = client.analyze(sentence.sentence,
int(level1), int(level2))
except Exception:
self.log(
logging.ERROR,
"REC_ID = {0}, RFILE_NAME = {1}".format(
hdr.call_metadata.record_key,
hdr.call_metadata.file_name))
self.log(logging.ERROR, traceback.format_exc())
continue
nlp_result_id = record_result(oracle, hdr, sentence, result)
nlp_result_list.documentList.add(
nlp_result_id=nlp_result_id,
sequence=sentence.sequence,
sentence=sentence.sentence,
document=result[2],
start_time=sentence.start_time,
end_time=sentence.end_time,
stt_result_detail_id=sentence.stt_result_detail_id)
try:
record_result_detail(oracle, hdr, result, nlp_result_id)
except Exception:
continue
self.log(logging.INFO,
"{0} NLP Process Success".format(sentence_cnt))
hdr.status_id = util.ProcStatus.PS_COMPLETED
util.insert_proc_result(oracle, self.logger, hdr)
new_body = nlp_result_list.SerializeToString()
self.sendto_router_ex(hdr, new_body)
except grpc.RpcError:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(
logging.ERROR, "REC_ID = {0}, RFILE_NAME = {1}".format(
hdr.call_metadata.record_key, hdr.call_metadata.file_name))
self.log(logging.ERROR, traceback.format_exc())
except Exception:
hdr.status_id = util.ProcStatus.PS_FAILED
util.insert_proc_result(oracle, self.logger, hdr)
self.log(logging.ERROR, traceback.format_exc())
finally:
oracle.disconnect()
def do_cs_job(self):
oracle = db_connection.Oracle()
# Select CS target list
cs_list = get_cs_list(oracle, self.select_cnt)
# Extract holiday list
holiday_list = select_holiday(oracle)
call_id = ''
rec_id = ''
rfile_name = ''
for item in cs_list:
try:
st = datetime.now()
(
rec_id,
rfile_name,
) = item
# Get call meta information
meta_info, stt_spch_sped_rx, stt_spch_sped_tx = get_cs_meta_info(
oracle, rfile_name, rec_id)
if not meta_info:
self.logger.error(
"Can't select CS meta information. [REC_ID = {0}, RFILE_NAME = {0}]"
.format(rec_id, rfile_name))
continue
call_date = meta_info[0].strftime(
'%Y-%m-%d') if meta_info[0] else ''
start_time = meta_info[6]
ruser_id = meta_info[10]
# Extract TEAMCD
team_cd = select_code_from_cm_call_meta_tb(oracle, ruser_id)
meta_info += team_cd if team_cd else ('', )
# Extract OFFICE_HOUR
if call_date.replace("-", "") in holiday_list:
office_hour = 'OH0003' if 9 <= start_time.hour < 18 else 'OH0004'
elif datetime.strptime(call_date,
'%Y-%m-%d').weekday() in [5, 6]:
office_hour = 'OH0003' if 9 <= start_time.hour < 18 else 'OH0004'
else:
office_hour = 'OH0001' if 9 <= start_time.hour < 18 else 'OH0002'
meta_info += (office_hour, )
# Create CALL_ID
call_id = create_call_id(oracle, meta_info)
# Get stt result
stt_result = get_cs_stt_result(oracle, rfile_name, rec_id)
for stt_data in stt_result:
# Create ticket id
ticket_id = create_ticket_id(oracle, call_date, call_id)
for pipeline_id, tree in self.tree_map.iteritems():
root = tree.get_node(tree.root)
if root.data[9] == 'PT0006':
# Create pipeline_event_id
pipeline_event_id = create_pipeline_event_id(
oracle, call_date, pipeline_id, ticket_id)
if stt_data[4] == 'ST0001':
sc = 'C'
elif stt_data[4] == 'ST0002':
sc = 'A'
else:
sc = 'M'
speed = stt_spch_sped_rx if sc == 'C' else stt_spch_sped_tx
# Insert stt result
stt_result_id = insert_stt_result(
oracle, call_date, stt_data, call_id,
ticket_id, pipeline_event_id, speed)
# Get stt detail
stt_detail_list = get_cs_stt_detail(
oracle, rfile_name, sc, rec_id)
# Insert stt_detail_list
stt_result_detail = insert_stt_detail(
oracle, call_date, stt_result_id,
stt_detail_list)
# Create message
hdr = self.create_common_header(
pipeline_id=pipeline_id,
router_id=root.identifier,
proc_id=root.data[1],
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
meta_info=meta_info,
speaker_code=stt_data[4])
hdr.stt_result_id = stt_result_id
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for sentence in stt_result_detail:
stt_result.sentence.add(
stt_result_detail_id=sentence[0],
sequence=sentence[1],
sentence=sentence[2],
start_time=sentence[3],
end_time=sentence[4])
self.send_to_router(hdr.SerializeToString(),
stt_result.SerializeToString())
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'Y', rec_id)
oracle.conn.commit()
self.logger.info(
"[SUCCESS] CALL_ID : {0}, REC_ID : {1}, RFILE_NAME : {2}, REQUIRED TIME = {3}"
.format(call_id, rec_id, rfile_name,
str(datetime.now() - st)))
except Exception:
oracle.conn.rollback()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'F', rec_id)
oracle.conn.commit()
self.logger.error(traceback.format_exc())
self.logger.error(
"[ERROR] CALL_ID : {0}, REC_ID : {1}, RFILE_NAME : {2}".
format(call_id, rec_id, rfile_name))
continue
oracle.disconnect()
def do_cs_job(self):
oracle = db_connection.Oracle()
# Select CS target list
cs_list = get_cs_list(oracle, self.select_cnt)
for item in cs_list:
call_id = ""
(rfile_name, ) = item
try:
# Get call meta information
meta_info_temp = get_cs_meta_info(oracle, rfile_name)
if not meta_info_temp:
self.logger.error(
"Can't select CS meta information. [RFILE_NAME = {0}]".
format(rfile_name))
continue
stt_spch_sped_rx = meta_info_temp[28]
stt_spch_sped_tx = meta_info_temp[29]
meta_info = meta_info_temp[:28]
# Insert meta information
call_id = create_call_id(oracle, meta_info)
print '[START] CALL_ID : {0}, RFILE_NAME : {1}'.format(
call_id, rfile_name)
self.logger.info(
'[START] CALL_ID : {0}, RFILE_NAME : {1}'.format(
call_id, rfile_name))
# Get stt result
stt_result = get_cs_stt_result(oracle, rfile_name)
for stt_data in stt_result:
# Create ticket id
ticket_id = create_ticket_id(oracle, call_id, 'CS_REC',
'CS_REC')
for pipeline_id, tree in self.tree_map.iteritems():
root = tree.get_node(tree.root)
if root.data[9] == 'PT0006':
# Create pipeline_event_id
pipeline_event_id = create_pipeline_event_id(
oracle, pipeline_id, ticket_id, meta_info[24],
meta_info[25])
if stt_data[4] == 'ST0001':
sc = 'C'
elif stt_data[4] == 'ST0002':
sc = 'A'
else:
sc = 'M'
speed = stt_spch_sped_rx if sc == 'C' else stt_spch_sped_tx
# Insert stt result
stt_result_id = insert_stt_result(
oracle, stt_data, call_id, ticket_id,
pipeline_event_id, speed)
# Get stt detail
stt_detail_list = get_cs_stt_detail(
oracle, rfile_name, sc)
# Insert stt_detail_list
stt_result_detail = insert_stt_detail(
oracle, stt_result_id, stt_detail_list)
# Create message
hdr = self.create_common_header(
pipeline_id=pipeline_id,
router_id=root.identifier,
proc_id=root.data[1],
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
meta_info=meta_info,
speaker_code=stt_data[4])
hdr.stt_result_id = stt_result_id
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for sentence in stt_result_detail:
stt_result.sentence.add(
stt_result_detail_id=sentence[0],
sequence=sentence[1],
sentence=sentence[2])
self.send_to_router(hdr.SerializeToString(),
stt_result.SerializeToString())
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'Y')
oracle.conn.commit()
print "[E N D] CALL_ID : {0}, RFILE_NAME : {1}".format(
call_id, rfile_name)
self.logger.info(
"[E N D] CALL_ID : {0}, RFILE_NAME : {1}".format(
call_id, rfile_name))
except Exception:
oracle.conn.rollback()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'F')
oracle.conn.commit()
self.logger.error(traceback.format_exc())
self.logger.error(
"[ERROR] CALL_ID : {0}, RFILE_NAME : {1}".format(
call_id, rfile_name))
continue
oracle.disconnect()
def do_cs_job(self):
oracle = db_connection.Oracle()
# Select CS target list
self.logger.info("Get CS list")
cs_list = get_cs_list(oracle, self.select_cnt)
self.logger.info("Done get CS list")
for rec_id, rfile_name in cs_list:
st = datetime.now()
call_id = ''
try:
# Get call meta information
self.logger.info("Get call meta information")
meta_info, stt_spch_sped_rx, stt_spch_sped_tx, cntc_cls_double_dict, cntc_user_info = get_cs_meta_info(
oracle, rfile_name, rec_id)
self.logger.info("Done get call meta information")
if not meta_info:
self.logger.error(
"Can't select CS meta information. [REC_ID = {0}, RFILE_NAME = {1}]"
.format(rec_id, rfile_name))
continue
# Create CALL_ID
call_id = create_call_id(oracle, meta_info, cntc_user_info)
# Get STT result
stt_result = get_cs_stt_result(oracle, rfile_name, rec_id)
# Create CALL_DATE
call_date = meta_info[0].strftime(
'%Y-%m-%d') if meta_info[0] else ''
for stt_data in stt_result:
# Create ticket id
ticket_id = create_ticket_id(oracle, call_date, call_id)
for pipeline_id, tree in self.tree_map.iteritems():
root = tree.get_node(tree.root)
if root.data[9] == 'PT0006':
if stt_data[4] == 'ST0001':
sc = 'C'
elif stt_data[4] == 'ST0002':
sc = 'S'
else:
sc = 'M'
speed = stt_spch_sped_rx if sc == 'C' else stt_spch_sped_tx
# Create pipeline_event_id
pipeline_event_id = create_pipeline_event_id(
oracle, call_date, pipeline_id, ticket_id)
# Insert stt result
stt_result_id = insert_stt_result(
oracle=oracle,
call_date=call_date,
stt_data=stt_data,
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
speed=speed)
# Get stt detail
stt_detail_list = get_cs_stt_detail(
oracle, rfile_name, sc, rec_id)
# Insert stt_detail_list
stt_result_detail = insert_stt_detail(
oracle, call_id, call_date, stt_result_id,
stt_detail_list)
# Create message
hdr = self.create_common_header(
pipeline_id=pipeline_id,
router_id=root.identifier,
proc_id=root.data[1],
call_id=call_id,
ticket_id=ticket_id,
pipeline_event_id=pipeline_event_id,
meta_info=meta_info,
speaker_code=stt_data[4],
cntc_user_info=cntc_user_info,
cntc_cls_double_dict=cntc_cls_double_dict)
hdr.stt_result_id = stt_result_id
stt_result = stt_pb2.SttResultDetail()
stt_result.stt_result_id = stt_result_id
for item in stt_result_detail:
stt_result.sentence.add(
stt_result_detail_id=item[0],
sequence=item[1],
sentence=item[2],
start_time=item[3],
end_time=item[4])
self.send_to_router(hdr.SerializeToString(),
stt_result.SerializeToString())
oracle.conn.commit()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'Y', rec_id)
self.logger.info(
"[SUCCESS] CALL_DATE = {0}, CALL_ID : {1}, REC_ID : {2}, RFILE_NAME : {3},"
" REQUIRED TIME = {4}".format(call_date, call_id, rec_id,
rfile_name,
str(datetime.now() - st)))
except Exception:
oracle.conn.rollback()
# Update TRANSFER_YN status
update_cs_meta_trans_status(oracle, rfile_name, 'F', rec_id)
oracle.conn.commit()
self.logger.error(traceback.format_exc())
self.logger.error(
"[ERROR] CALL_ID : {0}, REC_ID : {1}, RFILE_NAME : {2}".
format(call_id, rec_id, rfile_name))
continue
time.sleep(1)
oracle.disconnect()