def distinguish(self, ask, ask_keyword):
sent_vec = self.word_sequence.transfroms(
[ask], max_len=self.distinguish_seq_len)
word_vec = get_keyword_vec([ask_keyword],
max_len=self.distinguish_seq_len,
word_sequence=self.word_sequence)
sent_vec = torch.LongTensor(sent_vec).view(1, -1)
word_vec = torch.LongTensor(word_vec).view(1, -1)
output = self.distinguish_model.forward(sent_vec, word_vec)
label = self.distinguish_id_label[output.argmax(1).item()]
if self.debug:
print("profile distinguish sent vec: ", sent_vec)
print("profile distinguish word vec: ", word_vec)
print("profile distinguish output: ", output)
print("profile distinguish label: ", label)
return label
def category(self, ask, ask_keyword):
sent_vec = self.word_sequence.transfroms([ask],
max_len=self.category_seq_len)
word_vec = get_keyword_vec([ask_keyword],
max_len=self.category_seq_len,
word_sequence=self.word_sequence)
sent_vec = torch.LongTensor(sent_vec).view(1, -1)
word_vec = torch.LongTensor(word_vec).view(1, -1)
output = self.category_model.forward(sent_vec, word_vec)
label = self.category_id_label[output.argmax(1).item()]
if self.debug:
print("knowledge category ask: ", ask)
print("knowledge category ask_keyword", ask_keyword)
print("knowledge category sent vec: ", sent_vec)
print("knowledge category word vec: ", word_vec)
print("knowledge category predict output: ", output)
print("knowledge category label: ", label)
print()
return label
def test_distinguish(model_file,
mysql,
word_sequences_file,
class_json_file,
seq_len=30):
model = load_model(model_file)
word_sequence = load_word_sequences(word_sequences_file)
with open(class_json_file, 'r') as rf:
class_id = json.load(rf)
id_class = {i: c for c, i in class_id.items()}
dataset = mysql.query_dialog()
result = []
for data in dataset:
sent_vec = word_sequence.transfrom(data['ask'], seq_len)
word_vec = get_keyword_vec([data['ask_keyword']],
max_len=seq_len,
word_sequence=word_sequence)
sent_vec = torch.LongTensor(sent_vec).view(1, -1)
word_vec = torch.LongTensor(word_vec).view(1, -1)
output = model.forward(sent_vec, word_vec)
category = id_class[output.argmax(1).item()]
value = {'category': category, 'sentence': data['ask']}
result.append(value)
for v in result:
if v['category'] == 'profile':
print("sentence: ", v['sentence'])
print("category: ", v['category'])
print()
def retrieval(self, ask, ask_keyword, mysql_retrieval):
correct_answer, error_answer = list(), list()
for m_data in mysql_retrieval:
answer = m_data['answer']
answer_keyword = m_data['answer_keyword']
ask_vec = self.word_sequence.transfroms(
[ask], max_len=self.retrieval_seq_len)
answer_vec = self.word_sequence.transfroms(
[answer], max_len=self.retrieval_seq_len)
ask_keyword_vec = get_keyword_vec([ask_keyword],
word_sequence=self.word_sequence,
max_len=self.retrieval_seq_len)
answer_keyword_vec = get_keyword_vec(
[answer_keyword],
word_sequence=self.word_sequence,
max_len=self.retrieval_seq_len)
ask_vec = torch.LongTensor(ask_vec).view(1, -1)
answer_vec = torch.LongTensor(answer_vec).view(1, -1)
ask_keyword_vec = torch.LongTensor(ask_keyword_vec).view(1, -1)
answer_keyword_vec = torch.LongTensor(answer_keyword_vec).view(
1, -1)
predict_output = self.retrieval_model(ask_vec, answer_vec,
ask_keyword_vec,
answer_keyword_vec)
label_id = predict_output.argmax(1).item()
value = {
'answer': answer,
'predict_correct': predict_output[0][self.correct].item(),
'predict_error': predict_output[0][self.error].item(),
'label_id': label_id
}
if label_id == self.correct:
correct_answer.append(value)
elif label_id == self.error:
error_answer.append(value)
if self.debug:
print("profile retrieval ask: ", ask)
print("profile retrieval answer: ", answer)
print("profile retrieval predict output: ", predict_output)
print("profile retrieval label id: ", label_id)
print("profile retrieval predict label: ",
self.retrieval_id_label[label_id])
print()
retrieval_answer = None
if len(correct_answer) == 0:
value = 0
for e_answer in error_answer:
if e_answer['predict_error'] > value:
value = e_answer['predict_error']
retrieval_answer = e_answer['answer']
if self.debug:
print("choose error answer predict: {}".format(error_answer))
elif len(correct_answer) > 1:
value = 0
for c_answer in correct_answer:
if c_answer['predict_correct'] > value:
value = c_answer['predict_correct']
retrieval_answer = c_answer['answer']
if self.debug:
print(
"choose correct answer predict: {}".format(correct_answer))
print("retrieval final answer: ", retrieval_answer)
print("retrieval predict correct output: ", value)
else:
retrieval_answer = correct_answer[0]['answer']
return retrieval_answer