def __init__(self, cb_id, ner_model_id):
"""
init global variables
"""
self.cb_id = cb_id
self.ner_model_id = ner_model_id
self.bilstmcrf_model = PredictNetBiLstmCrf()
def post(self, request, type, nnid, ver):
"""
- desc : insert cnn configuration data
"""
try:
if (ver == 'active'):
if (type == 'w2v'):
return_data = PredictNetW2V().run(nnid, request.data)
elif (type == "d2v"):
return_data = PredictNetD2V().run(nnid, request.data)
elif (type == "cnn"):
# TO-DO : need predict function for active taged version
raise Exception("on developing now !")
elif (type == "wdnn"):
return_data = PredictNetWdnn().run(nnid, ver,
request.FILES)
elif (type == "seq2seq"):
return_data = PredictNetSeq2Seq().run(nnid, request.data)
elif (type == "autoencoder"):
return_data = PredictNetAutoEncoder().run(
nnid, request.data)
elif (type == "renet"):
#return_data = PredictNetRenet().run(nnid, ver, request.FILES)
# TO-DO : need to create PredictNetRenet class first
raise Exception("on developing now !")
elif (type == "anomaly"):
return_data = PredictNetAnomaly().run(nnid, request.data)
elif (type == "wcnn"):
return_data = PredictNetWcnn().run(nnid, request.data)
elif (type == "bilstmcrf"):
return_data = PredictNetBiLstmCrf().run(nnid, request.data)
else:
raise Exception("Not defined type error")
else:
if (type == 'w2v'):
# TO-DO : need predict function for specific version
raise Exception("on developing now !")
elif (type == "d2v"):
# TO-DO : need predict function for specific version
raise Exception("on developing now !")
elif (type == "cnn"):
return_data = PredictNetCnn().run(nnid, ver, request.FILES)
elif (type == "wdnn"):
return_data = PredictNetWdnn().run(nnid, ver,
request.FILES)
elif (type == "seq2seq"):
# TO-DO : need predict function for specific version
raise Exception("on developing now !")
elif (type == "renet"):
return_data = PredictNetRenet().run(
nnid, ver, request.FILES)
else:
raise Exception("Not defined type error")
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
def post(self, request, type, nnid, ver):
"""
Request Deep Neural Network to predict result with given data \n
input formats can be varies on type of networks \n
but usually you can use it with parm input_data \n
---
# Class Name : ServiceManagerPredict
# Description:
request predict service via rest service
It caches the model and vectors on first request
It may can take some time at first for caching, after than we can response the request
within 1.0 sec
"""
try:
if ver == 'active':
condition = {'nn_id': nnid}
ver = NNCommonManager().get_nn_info(
condition)[0]['nn_wf_ver_id']
if (type == "resnet" or type == "inceptionv4"):
return_data = PredictNetImage().run(nnid, ver, request)
elif (type == 'w2v'):
return_data = PredictNetW2V().run(nnid, request.data)
elif (type == "d2v"):
return_data = PredictNetD2V().run(nnid, request.data)
elif (type == "cnn"):
return_data = PredictNetCnn().run(nnid, ver, request.FILES)
elif (type == "wdnn"):
return_data = PredictNetWdnn().run(nnid, ver, request)
elif (type == "seq2seq"):
return_data = PredictNetSeq2Seq().run(nnid, request.data)
elif (type == "autoencoder"):
return_data = PredictNetAutoEncoder().run(nnid, request.data)
elif (type == "anomaly"):
return_data = PredictNetAnomaly().run(nnid, request.data)
elif (type == "wcnn"):
return_data = PredictNetWcnn().run(nnid, request.data)
elif (type == "bilstmcrf"):
return_data = PredictNetBiLstmCrf().run(nnid, request.data)
elif (type == "ngram_mro"):
return_data = PredictNetNgram().run(type, nnid, ver,
request.data)
elif (type == "xgboost_reg"):
return_data = PredictNetXgboost().run(nnid, ver, request.FILES)
return Response(return_data)
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))
class EntityRecognizer(ShareData):
def __init__(self, cb_id, ner_model_id):
"""
init global variables
"""
self.cb_id = cb_id
self.ner_model_id = ner_model_id
self.bilstmcrf_model = PredictNetBiLstmCrf()
def parse(self, share_data):
if self.ner_model_id != '':
logging.info("■■■■■■■■■■ NER 분석 전 : " + str(share_data.get_morphed_data()))
ner_data = self._get_ner_data(' '.join(share_data.get_morphed_data()))
share_data, ner_data = self._match_ngram_dict(share_data, share_data.get_morphed_data(), ner_data)
self._get_convert_data(ner_data, share_data)
logging.info("■■■■■■■■■■ NER 분석 결과 : " + str(ner_data))
return share_data
def _get_convert_data(self, ner_data, share_data):
"""
make sentence for charn cnn intent prediction
:param ner_data:
:param share_data:
:return:
"""
buff_list = []
for val_a, val_b in zip(ner_data, share_data.get_morphed_data()):
if val_a == 'O':
buff_list.append(val_b)
else:
buff_list.append(val_a)
share_data.set_convert_data(sorted(set(buff_list), key=lambda x: buff_list.index(x)))
def _get_ner_data(self, input_sentence):
result = self.bilstmcrf_model.run(self.ner_model_id, {"input_data": input_sentence, "num": 0, "clean_ans": False})
return result
def _match_ngram_dict(self, share_data, input_sentence, ner_data_input):
"""
match ngram dict with ner analized
:return:
"""
try:
result = {}
ner_data = ner_data_input.copy()
if len(input_sentence) != len(ner_data):
logging.info("■■■■■■■■■■ 길이 차이로 NER 처리 불가 ■■■■■■■■■■")
pass
cb_data = ChatKnowledgeMemDict.ngram.get(self.cb_id)
cb_data_order = ChatKnowledgeMemDict.ngram_order.get(self.cb_id)
cb_data_th = ChatKnowledgeMemDict.ngram_conf.get(self.cb_id)
dist_keys = dict(Counter(ner_data))
index = 0
for key, val in zip(ner_data, input_sentence):
if key == 'O':
continue
if result.get(key):
continue
if cb_data.get(key) is None:
continue
model = ngram.NGram(key=self.lower)
model.update(cb_data.get(key))
if dist_keys.get(key) > 1:
ner_conv = ' '.join(list(map(lambda x: x[0], list(filter(lambda x: x[1] == key, zip(input_sentence, ner_data))))))
result[key] = list(map(lambda x: x[0], model.search(ner_conv.replace(' ', '').lower(), threshold=1.0)))
if len(result[key]) == 0:
result[key] = list(map(lambda x: x[0], model.search(ner_conv.replace(' ', '').lower(), threshold=cb_data_th[key])))[0:4]
if len(result[key]) == 0:
logging.info("■■■■■■■■■■ NER 오류로 전수 조사 시작 (시간소요발생) ■■■■■■■■■■")
data, id = self.check_all_dict(ner_conv.replace(' ', '').lower(), cb_data, cb_data_order, cb_data_th)
if id is not None:
result[id] = data
key = id
ner_data_input[index] = id
else:
ner_conv = val
result[key] = list(map(lambda x: x[0], model.search(ner_conv.lower(), threshold=1.0)))
if len(result[key]) == 0:
result[key] = list(map(lambda x: x[0], model.search(ner_conv.lower(), threshold=cb_data_th[key])))[0:4]
if len(result[key]) == 0:
logging.info("■■■■■■■■■■ NER 오류로 전수 조사 시작 (시간소요발생) ■■■■■■■■■■")
data, id = self.check_all_dict(ner_conv.lower(), cb_data, cb_data_order, cb_data_th)
if id is not None:
result[id] = data
key = id
ner_data_input[index] = id
if len(result[key]) == 0:
del result[key]
else:
if key is not None and key in ['tagorg', 'tagname']:
share_data.set_story_ner_entity(key, [ner_conv] + result[key])
else:
share_data.set_story_ner_entity(key, result[key])
index = index + 1
return share_data, ner_data_input
except Exception as e:
raise Exception("Error on matching ngram afger bilstm crf : {0}".format(e))
def check_all_dict(self, ner_conv, cb_data, cb_data_order, cb_data_th):
"""
check other dict when failed to find matching value
:param ner_conv:
:return:
"""
result = []
for key in cb_data_order:
model = ngram.NGram(key=self.lower)
model.update(cb_data.get(key))
result = list(map(lambda x: x[0], model.search(ner_conv, threshold=cb_data_th[key])))[0:4]
if len(result) > 0:
return result, key
return result, None
def lower(self, s):
return s.lower()
def post(self, request, type, nnid, ver):
"""
Request Deep Neural Network to predict result with given data \n
input formats can be varies on type of networks \n
but usually you can use it with parm input_data \n
---
# Class Name : ServiceManagerPredict
# Description:
request predict service via rest service
It caches the model and vectors on first request
It may can take some time at first for caching, after than we can response the request
within 1.0 sec
"""
try:
if (ver == 'active'):
if (type == 'w2v'):
return_data = PredictNetW2V().run(nnid, request.data)
elif (type == "d2v"):
return_data = PredictNetD2V().run(nnid, request.data)
elif (type == "cnn"):
# TO-DO : need predict function for active taged version
raise Exception("on developing now !")
elif (type == "wdnn"):
return_data = PredictNetWdnn().run(nnid, ver,
request.FILES)
elif (type == "seq2seq"):
return_data = PredictNetSeq2Seq().run(nnid, request.data)
elif (type == "autoencoder"):
return_data = PredictNetAutoEncoder().run(
nnid, request.data)
elif (type == "renet"):
#return_data = PredictNetRenet().run(nnid, ver, request.FILES)
# TO-DO : need to create PredictNetRenet class first
raise Exception("on developing now !")
elif (type == "anomaly"):
return_data = PredictNetAnomaly().run(nnid, request.data)
elif (type == "wcnn"):
return_data = PredictNetWcnn().run(nnid, request.data)
elif (type == "bilstmcrf"):
return_data = PredictNetBiLstmCrf().run(nnid, request.data)
else:
raise Exception("Not defined type error")
else:
if (type == 'w2v'):
# TO-DO : need predict function for specific version
raise Exception("on developing now !")
elif (type == "d2v"):
# TO-DO : need predict function for specific version
raise Exception("on developing now !")
elif (type == "cnn"):
return_data = PredictNetCnn().run(nnid, ver, request.FILES)
elif (type == "wdnn"):
return_data = PredictNetWdnn().run(nnid, ver,
request.FILES)
elif (type == "seq2seq"):
# TO-DO : need predict function for specific version
raise Exception("on developing now !")
elif (type == "renet"):
return_data = PredictNetRenet().run(
nnid, ver, request.FILES)
else:
raise Exception("Not defined type error")
return Response(json.dumps(return_data))
except Exception as e:
return_data = {"status": "404", "result": str(e)}
return Response(json.dumps(return_data))