def printInfo(event): seg = Seg() seg.load_userdict('../userdict/userdict.txt') # 读取数据 List_kw, questionList, answerList = read_corpus1() # 初始化模型 ss = SentenceSimilarity(seg) ss.set_sentences(questionList) ss.TfidfModel() # tfidf模型 # ss.LsiModel() # lsi模型 # ss.LdaModel() # lda模型 text2.delete(1.0, END) question = (text1.get('1.0', END)) #if question == 'q': #break time1 = time.time() question_k = ss.similarity_k(question, 5) text2.insert("insert", ": {}".format(answerList[question_k[0][0]])) #print(": {}".format(answerList[question_k[0][0]])) #for idx, score in zip(*question_k): # print("same questions: {}, score: {}".format(questionList[idx], score)) #time2 = time.time() #cost = time2 - time1 #print('Time cost: {} s'.format(cost)) #entry2.insert(10,question) #清空entry2控件 text1.delete(1.0, END) syn(": {}".format(answerList[question_k[0][0]]))
def dictTest(): dict = {} seg = Seg() original_ss = SentenceSimilarity(seg) readDictData(original_ss, dict) original_ss.TfidfModel() # original_ss.LdaModel() # original_ss.LsiModel() total_data_len = len(X_test) success_len = 0 f1 = open('ah_data_lsi.txt', 'w', encoding='utf-8') for i in range(len(X_test)): print("-------------------------------------") text = checkData(X_test[i]) text = "".join(seg.cut_for_search(text)) print("测试内容: " + text) try: sentences = original_ss.similarityArray(text) sentences = sorted(sentences, key=lambda e: e.get_score(), reverse=True) count = 0 for sentence in sentences: if sentence.get_score() > 0.9: print(sentence.get_score()) if sentence.get_score() == 1.0: count = count + 1 sentence = original_ss.similarity(text) if count < 2 and dict.get( sentence.get_origin_sentence()) == Y_test[i]: success_len = success_len + 1 else: y = Y_test[i] f1.writelines("-------------------------------------\n") f1.writelines("测试内容: " + text + "\n") for sentence in sentences: f1.writelines("匹配标签: 【" + dict.get(sentence.get_origin_sentence()) + "】 真实标签:【" + y + "】 评分: " + str(sentence.get_score()) + "\n") except Exception as e: print(e) print(success_len / total_data_len)
def tf(): dt = {} # if __name__ == '__main__': # 读入训练集 file_obj = FileObj(r"train_data.txt") train_sentences = file_obj.read_lines() # 读入测试集 file_obj = FileObj(r"test_data.txt") test1_sentences = file_obj.read_lines() # 分词工具,基于jieba分词,我自己加了一次封装,主要是去除停用词 seg = Seg() # 训练模型 ss = SentenceSimilarity(seg) ss.set_sentences(train_sentences) ss.TfidfModel() # tfidf模型 # ss.LsiModel() # lsi模型 # ss.LdaModel() # lda模型 # 测试集1 right_count = 0 # w=open("result510tf.txt",'w') # w.write(str("source_id") + '\t' + str("target_id") + '\n') for i in range(len(test1_sentences)): print "*********************" print i print test1_sentences[i] test = str(test1_sentences[i].encode("utf-8")) t = test.split(',')[0] dict = ss.similarity(test1_sentences[i]) # dict的key为句子的(序号-1),value为计算出的距离 for k, v in dict: print t, k + 1, v # 如2784 2784 1.0 ind2 = k + 1 if (str(k + 1) == str(t)): print "same" else: # w.write(str(t) + '\t' + str(k+1) + '\n') addtodict2(dt, int(t), int(ind2), v) # w.close() return dt
def run_prediction(input_file_path, output_file_path): # 读入训练集 file_obj = FileObj(r"./TFIDF_baseline/dataSet/trainQuestions.txt") train_sentences = file_obj.read_lines() # 读入测试集 file_obj = FileObj(input_file_path) test_sentences = file_obj.read_lines() # 分词工具,基于jieba分词,并去除停用词 seg = Seg() # 训练模型 ss = SentenceSimilarity(seg) ss.set_sentences(train_sentences) ss.TfidfModel() # tfidf模型 # 测试集 right_count = 0 file_result=open(output_file_path,'w') with open("./TFIDF_baseline/dataSet/trainAnswers.txt",'r',encoding = 'utf-8') as file_answer: line = file_answer.readlines() for i in range(0,len(test_sentences)): top_15 = ss.similarity(test_sentences[i]) ''' for j in range(0,len(top_15)): answer_index=top_15[j][0] answer=line[answer_index] file_result.write(str(top_15[j][1])+'\t'+str(answer)) file_result.write("\n") ''' file_result.write(line[top_15[0][0]]+'\n') file_result.close() file_answer.close()
def main(question, top_k, task='faq'): # 读取数据 if task == 'chat': qList_kw, questionList, answerList = read_corpus2() else: qList_kw, questionList, answerList = read_corpus1() """简单的倒排索引""" # 计算倒排表 invertTable = invert_idxTable(qList_kw) inputQuestionKW = seg.cut(question) # 利用关键词匹配得到与原来相似的问题集合 questionList_s, answerList_s = filter_questionByInvertTab( inputQuestionKW, questionList, answerList, invertTable) # 初始化模型 ss = SentenceSimilarity(seg) ss.set_sentences(questionList_s) ss.TfidfModel() # tfidf模型 # ss.LsiModel() # lsi模型 # ss.LdaModel() # lda模型 question_k = ss.similarity_k(question, top_k) return question_k, questionList_s, answerList_s
class kuakuaChat(): def __init__(self): """ 初始化夸夸话题回复表 """ self.qa_dict = {} self.q_list = [] with open('./douban_kuakua_topic.txt', 'r', encoding='utf8') as in_file: for line in in_file.readlines(): que = line.split('<######>')[0].strip() ans_list = [] for ans in line.split('<######>')[-1].split('<$$$$$$>'): if len(ans) > 2: ans_list.append(ans) if len(que) > 5: self.q_list.append(que) self.qa_dict[que] = ans_list zhcn_seg = zhcnSeg() self.sent_sim = SentenceSimilarity(zhcn_seg) self.sent_sim.set_sentences(self.q_list) # 默认用tfidf self.sent_sim.TfidfModel() def answer_question(self, question_str): """ 返回与输入问句最相似的问句的固定回答 :param question_str: :return: """ most_sim_questions = self.sent_sim.similarity_top_k(question_str, 4) answer_list = [] for item in most_sim_questions: answer = self.qa_dict[item[0]] answer_list += answer return answer_list
# 读入测试集1 file_obj = FileObj(r"D:/Github Project/sentence Similarity/testSet/testSet1.txt") test1_sentences = file_obj.read_lines() # 读入测试集2 file_obj = FileObj(r"D:/Github Project/sentence Similarity/testSet/testSet2.txt") test2_sentences = file_obj.read_lines() # 分词工具,基于jieba分词,我自己加了一次封装,主要是去除停用词 seg = Seg() # 训练模型 ss = SentenceSimilarity(seg) ss.set_sentences(train_sentences) ss.TfidfModel() # tfidf模型 # ss.LsiModel() # lsi模型 # ss.LdaModel() # lda模型 # 测试集1 right_count = 0 for i in range(0,len(train_sentences)): sentence = ss.similarity(test1_sentences[i]) if i != sentence.id: print (str(i) + " wrong! score: " + str(sentence.score)) else: right_count += 1 print (str(i) + " right! score: " + str(sentence.score)) print ("正确率为: " + str(float(right_count)/len(train_sentences)))
from sentenceSimilarity import SentenceSimilarity from sentence import Sentence import time from time import ctime import threading file_obj = FileObj(r"dataSet/train_q.txt") train_sentences = file_obj.read_lines() with open("dataSet/train_a.txt", 'r', encoding='utf-8') as file_answer: line = file_answer.readlines() seg = Seg() # 训练模型 ss1 = SentenceSimilarity(seg) ss1.set_sentences(train_sentences) ss1.TfidfModel() # tfidf模型 ss2 = SentenceSimilarity(seg) ss2.set_sentences(train_sentences) ss2.LsiModel() # LSI模型 def tfidf_model(sentence): top = ss1.similarity(sentence) answer_index = top[0][0] answer = line[answer_index] return top[0][1], answer def lsi_model(sentence): top = ss2.similarity(sentence)