def printInfo(event):
seg = Seg()
seg.load_userdict('../userdict/userdict.txt')
# 读取数据
List_kw, questionList, answerList = read_corpus1()
# 初始化模型
ss = SentenceSimilarity(seg)
ss.set_sentences(questionList)
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
text2.delete(1.0, END)
question = (text1.get('1.0', END))
#if question == 'q':
#break
time1 = time.time()
question_k = ss.similarity_k(question, 5)
text2.insert("insert", "： {}".format(answerList[question_k[0][0]]))
#print("： {}".format(answerList[question_k[0][0]]))
#for idx, score in zip(*question_k):
# print("same questions： {}, score： {}".format(questionList[idx], score))
#time2 = time.time()
#cost = time2 - time1
#print('Time cost: {} s'.format(cost))
#entry2.insert(10,question)
#清空entry2控件
text1.delete(1.0, END)
syn("： {}".format(answerList[question_k[0][0]]))
def tf():
dt = {}
# if __name__ == '__main__':
# 读入训练集
file_obj = FileObj(r"train_data.txt")
train_sentences = file_obj.read_lines()
# 读入测试集
file_obj = FileObj(r"test_data.txt")
test1_sentences = file_obj.read_lines()
# 分词工具，基于jieba分词，我自己加了一次封装，主要是去除停用词
seg = Seg()
# 训练模型
ss = SentenceSimilarity(seg)
ss.set_sentences(train_sentences)
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
# 测试集1
right_count = 0
# w=open("result510tf.txt",'w')
# w.write(str("source_id") + '\t' + str("target_id") + '\n')
for i in range(len(test1_sentences)):
print "*********************"
print i
print test1_sentences[i]
test = str(test1_sentences[i].encode("utf-8"))
t = test.split('，')[0]
dict = ss.similarity(test1_sentences[i])
# dict的key为句子的（序号-1），value为计算出的距离
for k, v in dict:
print t, k + 1, v # 如2784 2784 1.0
ind2 = k + 1
if (str(k + 1) == str(t)):
print "same"
else:
# w.write(str(t) + '\t' + str(k+1) + '\n')
addtodict2(dt, int(t), int(ind2), v)
# w.close()
return dt
def run_prediction(input_file_path, output_file_path):
# 读入训练集
file_obj = FileObj(r"./TFIDF_baseline/dataSet/trainQuestions.txt")
train_sentences = file_obj.read_lines()
# 读入测试集
file_obj = FileObj(input_file_path)
test_sentences = file_obj.read_lines()
# 分词工具，基于jieba分词，并去除停用词
seg = Seg()
# 训练模型
ss = SentenceSimilarity(seg)
ss.set_sentences(train_sentences)
ss.TfidfModel() # tfidf模型
# 测试集
right_count = 0
file_result=open(output_file_path,'w')
with open("./TFIDF_baseline/dataSet/trainAnswers.txt",'r',encoding = 'utf-8') as file_answer:
line = file_answer.readlines()
for i in range(0,len(test_sentences)):
top_15 = ss.similarity(test_sentences[i])
'''
for j in range(0,len(top_15)):
answer_index=top_15[j][0]
answer=line[answer_index]
file_result.write(str(top_15[j][1])+'\t'+str(answer))
file_result.write("\n")
'''
file_result.write(line[top_15[0][0]]+'\n')
file_result.close()
file_answer.close()
def main(question, top_k, task='faq'):
# 读取数据
if task == 'chat':
qList_kw, questionList, answerList = read_corpus2()
else:
qList_kw, questionList, answerList = read_corpus1()
"""简单的倒排索引"""
# 计算倒排表
invertTable = invert_idxTable(qList_kw)
inputQuestionKW = seg.cut(question)
# 利用关键词匹配得到与原来相似的问题集合
questionList_s, answerList_s = filter_questionByInvertTab(
inputQuestionKW, questionList, answerList, invertTable)
# 初始化模型
ss = SentenceSimilarity(seg)
ss.set_sentences(questionList_s)
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
question_k = ss.similarity_k(question, top_k)
return question_k, questionList_s, answerList_s
class kuakuaChat():
def __init__(self):
"""
初始化夸夸话题回复表
"""
self.qa_dict = {}
self.q_list = []
with open('./douban_kuakua_topic.txt', 'r',
encoding='utf8') as in_file:
for line in in_file.readlines():
que = line.split('<######>')[0].strip()
ans_list = []
for ans in line.split('<######>')[-1].split('<$$$$$$>'):
if len(ans) > 2:
ans_list.append(ans)
if len(que) > 5:
self.q_list.append(que)
self.qa_dict[que] = ans_list
zhcn_seg = zhcnSeg()
self.sent_sim = SentenceSimilarity(zhcn_seg)
self.sent_sim.set_sentences(self.q_list)
# 默认用tfidf
self.sent_sim.TfidfModel()
def answer_question(self, question_str):
"""
返回与输入问句最相似的问句的固定回答
:param question_str:
:return:
"""
most_sim_questions = self.sent_sim.similarity_top_k(question_str, 4)
answer_list = []
for item in most_sim_questions:
answer = self.qa_dict[item[0]]
answer_list += answer
return answer_list
if __name__ == '__main__':
start = clock()
# 读入后半部分语料
file_obj = FileObj(r"sentence2.txt")
train_sentences = file_obj.read_lines()
# 读入前半部分语料
file_obj = FileObj(r"sentence1.txt")
test1_sentences = file_obj.read_lines()
# 分词工具，基于jieba分词，加了一次封装，主要是去除停用词
seg = Seg()
# 生成模型
ss = SentenceSimilarity(seg)
ss.set_sentences(train_sentences)
#ss.TfidfModel() # tfidf模型
ss.LsiModel() # lsi模型
#ss.LdaModel() # lda模型
# 计算句子相似度
# for i in range(0,len(train_sentences)/100):
# mysims = ss.mysimilarity(test1_sentences[i*100])
# # 每一百行为一个整体
# sims_divided = mysims[i*100:(i+1)*100]
# # 对一百行内的相似度进行排序
# sort_sims = sorted(enumerate(sims_divided),key = lambda item : -item[1])
# # 选择前五个最高的相似度进行输出
# chosen_sims = sort_sims[:5]
# for j in range(0,5):
# print str(chosen_sims[j][0]) + " score:" + str(chosen_sims[j][1])
from fileObject import FileObj
from sentenceSimilarity import SentenceSimilarity
from sentence import Sentence
import time
from time import ctime
import threading
file_obj = FileObj(r"dataSet/train_q.txt")
train_sentences = file_obj.read_lines()
with open("dataSet/train_a.txt", 'r', encoding='utf-8') as file_answer:
line = file_answer.readlines()
seg = Seg()
# 训练模型
ss1 = SentenceSimilarity(seg)
ss1.set_sentences(train_sentences)
ss1.TfidfModel() # tfidf模型
ss2 = SentenceSimilarity(seg)
ss2.set_sentences(train_sentences)
ss2.LsiModel() # LSI模型
def tfidf_model(sentence):
top = ss1.similarity(sentence)
answer_index = top[0][0]
answer = line[answer_index]
return top[0][1], answer
def lsi_model(sentence):
if __name__ == '__main__':
# 读入训练集
# file_obj = FileObj(r"dataSet/trainQuestions.txt")
# train_sentences = file_obj.read_lines()
# 读入测试集
file_obj = FileObj(r"dataSet/devQuestions.txt")
test_sentences = file_obj.read_lines()
# 分词工具，基于jieba分词，并去除停用词
seg = Seg()
# 训练模型
ss = SentenceSimilarity(seg)
ss.set_sentences(test_sentences)
ss.TfidfModel() # tfidf模型
# 测试集
right_count = 0
file_result = open('dataSet/result.txt', 'w')
with open("dataSet/trainAnswers.txt", 'r',
encoding='utf-8') as file_answer:
line = file_answer.readlines()
for i in tqdm(range(0, len(test_sentences))):
top_15 = ss.similarity(test_sentences[i])
for j in range(0, len(top_15)):
answer_index = top_15[j][0]
fDist = FreqDist(wordList)
#print(fDist.most_common())
print("单词总数: ",fDist.N())
print("不同单词数: ",fDist.B())
fDist.plot(10)
if __name__ == '__main__':
# 设置外部词
seg = Seg()
seg.load_userdict('./userdict/userdict.txt')
# 读取数据
List_kw, questionList, answerList = read_corpus()
# 初始化模型
ss = SentenceSimilarity(seg)
ss.set_sentences(questionList)
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
while True:
question = input("请输入问题(q退出): ")
if question == 'q':
break
time1 = time.time()
question_k = ss.similarity_k(question, 5)
print("亲，我们给您找到的答案是： {}".format(answerList[question_k[0][0]]))
for idx, score in zip(*question_k):
print("same questions： {}, score： {}".format(questionList[idx], score))
time2 = time.time()
cost = time2 - time1
numQuestions_train = len(question_list) - 100
numQuestions_test = 100
question_list_train = question_list[:numQuestions_train]
question_list_test = question_list[numQuestions_train:]
answer_list_train = answer_list[:numQuestions_train]
answer_list_test = answer_list[numQuestions_train:]
# 分词工具，基于jieba分词，我自己加了一次封装，主要是去除停用词
seg = Seg()
# 训练模型
ss = SentenceSimilarity(seg)
ss.set_sentences(question_list_train)
# ss.TfidfModel() # tfidf模型
ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
# 测试集1
right_count = 0
for i in range(0,len(question_list_test)):
sentenceK = ss.similarityK(question_list_test[i])
print ' '
print 'question: %s' % question_list_test[i]
for k in range(len(sentenceK)):
sentence_k = sentenceK[k]
org_sentence = sentence_k.origin_sentence
sentence_id = sentence_k.id
answer_k = answer_list_train[sentence_id]
train_sentences = file_obj.read_lines()
# 读入测试集1
file_obj = FileObj(r"testSet/testSet1.txt")
test1_sentences = file_obj.read_lines()
# 读入测试集2
file_obj = FileObj(r"testSet/testSet2.txt")
test2_sentences = file_obj.read_lines()
# 分词工具，基于jieba分词，我自己加了一次封装，主要是去除停用词
seg = Seg()
# 训练模型
ss = SentenceSimilarity(seg)
ss.set_sentences(train_sentences)
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
# 测试集1
right_count = 0
for i in range(0,len(train_sentences)):
sentence = ss.similarity(test1_sentences[i])
if i != sentence.id:
print str(i) + " wrong! score: " + str(sentence.score)
else:
right_count += 1
print str(i) + " right! score: " + str(sentence.score)
fDist = FreqDist(wordList)
# print(fDist.most_common())
print("单词总数: ", fDist.N())
print("不同单词数: ", fDist.B())
fDist.plot(10)
if __name__ == '__main__':
# 设置外部词
seg = Seg()
seg.load_userdict('userdict/userdict.txt') # 添加自己的词库到默认词库中
# 读取数据
_, questionList, answerList = read_corpus()
# 初始化模型
ss = SentenceSimilarity(seg) # 设置self.reg属性
ss.set_sentences(questionList) # 设置self.sentences属性，列表类型，列表中每个值为Sentence对象
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
while True:
question = input("请输入问题(q退出): ")
if question == 'q':
break
time1 = time.time()
question_k = ss.similarity_k(question, 5)
print("亲，我们给您找到的答案是： {}".format(answerList[question_k[0][0]]))
for idx, score in zip(*question_k):
print("same questions： {}, score： {}".format(
questionList[idx], score))
time2 = time.time()
break
inputQuestionKW = seg.cut(question)
# 利用关键词匹配得到与原来相似的问题集合
#questionList_s, answerList_s = filter_questionByInvertTab(inputQuestionKW, questionList, answerList,
# invertTable)
questionList_s, answerList_s = filter_questionByInvertTab(inputQuestionKW, questionList_corpus, answerList_corpus,
invertTable)
print(questionList_s)
if len(questionList_s) > 1:
questionList = questionList_s
answerList = answerList_s
# 初始化模型
ss = SentenceSimilarity(seg)
#ss.set_sentences(questionList)
ss.set_sentences(questionList_corpus)
ss.TfidfModel() # tfidf模型
# ss.LsiModel() # lsi模型
# ss.LdaModel() # lda模型
question_k = ss.similarity_k(question, 5)
#print("亲，我们给您找到的答案是： {}".format(answerList[question_k[0][0]]))
print("亲，我们给您找到的答案是： {}".format(answerList_corpus[question_k[0][0]]))
for idx, score in zip(*question_k):
#print("same questions： {}, score： {}".format(questionList[idx], score))
print("same questions： {}, score： {}".format(questionList_corpus[idx], score))
time2 = time.time()
cost = time2 - time1
print('Time cost: {} s'.format(cost))
