def distinguish(self, ask, ask_keyword): sent_vec = self.word_sequence.transfroms( [ask], max_len=self.distinguish_seq_len) word_vec = get_keyword_vec([ask_keyword], max_len=self.distinguish_seq_len, word_sequence=self.word_sequence) sent_vec = torch.LongTensor(sent_vec).view(1, -1) word_vec = torch.LongTensor(word_vec).view(1, -1) output = self.distinguish_model.forward(sent_vec, word_vec) label = self.distinguish_id_label[output.argmax(1).item()] if self.debug: print("profile distinguish sent vec: ", sent_vec) print("profile distinguish word vec: ", word_vec) print("profile distinguish output: ", output) print("profile distinguish label: ", label) return label
def category(self, ask, ask_keyword): sent_vec = self.word_sequence.transfroms([ask], max_len=self.category_seq_len) word_vec = get_keyword_vec([ask_keyword], max_len=self.category_seq_len, word_sequence=self.word_sequence) sent_vec = torch.LongTensor(sent_vec).view(1, -1) word_vec = torch.LongTensor(word_vec).view(1, -1) output = self.category_model.forward(sent_vec, word_vec) label = self.category_id_label[output.argmax(1).item()] if self.debug: print("knowledge category ask: ", ask) print("knowledge category ask_keyword", ask_keyword) print("knowledge category sent vec: ", sent_vec) print("knowledge category word vec: ", word_vec) print("knowledge category predict output: ", output) print("knowledge category label: ", label) print() return label
def test_distinguish(model_file, mysql, word_sequences_file, class_json_file, seq_len=30): model = load_model(model_file) word_sequence = load_word_sequences(word_sequences_file) with open(class_json_file, 'r') as rf: class_id = json.load(rf) id_class = {i: c for c, i in class_id.items()} dataset = mysql.query_dialog() result = [] for data in dataset: sent_vec = word_sequence.transfrom(data['ask'], seq_len) word_vec = get_keyword_vec([data['ask_keyword']], max_len=seq_len, word_sequence=word_sequence) sent_vec = torch.LongTensor(sent_vec).view(1, -1) word_vec = torch.LongTensor(word_vec).view(1, -1) output = model.forward(sent_vec, word_vec) category = id_class[output.argmax(1).item()] value = {'category': category, 'sentence': data['ask']} result.append(value) for v in result: if v['category'] == 'profile': print("sentence: ", v['sentence']) print("category: ", v['category']) print()
def retrieval(self, ask, ask_keyword, mysql_retrieval): correct_answer, error_answer = list(), list() for m_data in mysql_retrieval: answer = m_data['answer'] answer_keyword = m_data['answer_keyword'] ask_vec = self.word_sequence.transfroms( [ask], max_len=self.retrieval_seq_len) answer_vec = self.word_sequence.transfroms( [answer], max_len=self.retrieval_seq_len) ask_keyword_vec = get_keyword_vec([ask_keyword], word_sequence=self.word_sequence, max_len=self.retrieval_seq_len) answer_keyword_vec = get_keyword_vec( [answer_keyword], word_sequence=self.word_sequence, max_len=self.retrieval_seq_len) ask_vec = torch.LongTensor(ask_vec).view(1, -1) answer_vec = torch.LongTensor(answer_vec).view(1, -1) ask_keyword_vec = torch.LongTensor(ask_keyword_vec).view(1, -1) answer_keyword_vec = torch.LongTensor(answer_keyword_vec).view( 1, -1) predict_output = self.retrieval_model(ask_vec, answer_vec, ask_keyword_vec, answer_keyword_vec) label_id = predict_output.argmax(1).item() value = { 'answer': answer, 'predict_correct': predict_output[0][self.correct].item(), 'predict_error': predict_output[0][self.error].item(), 'label_id': label_id } if label_id == self.correct: correct_answer.append(value) elif label_id == self.error: error_answer.append(value) if self.debug: print("profile retrieval ask: ", ask) print("profile retrieval answer: ", answer) print("profile retrieval predict output: ", predict_output) print("profile retrieval label id: ", label_id) print("profile retrieval predict label: ", self.retrieval_id_label[label_id]) print() retrieval_answer = None if len(correct_answer) == 0: value = 0 for e_answer in error_answer: if e_answer['predict_error'] > value: value = e_answer['predict_error'] retrieval_answer = e_answer['answer'] if self.debug: print("choose error answer predict: {}".format(error_answer)) elif len(correct_answer) > 1: value = 0 for c_answer in correct_answer: if c_answer['predict_correct'] > value: value = c_answer['predict_correct'] retrieval_answer = c_answer['answer'] if self.debug: print( "choose correct answer predict: {}".format(correct_answer)) print("retrieval final answer: ", retrieval_answer) print("retrieval predict correct output: ", value) else: retrieval_answer = correct_answer[0]['answer'] return retrieval_answer