def get_fen_result(zz):
all_sen=[]
data=[]
sentences = cut_sent(zz)
for sent in sentences:
sent = sent.replace("\n", "")
sent = sent.replace("\t", "")
sent = sent.replace(" ", "")
if sent:
all_sen.append(sent)
for line in all_sen:
word_list = [x for x in jieba.cut(line.strip())]
data.append(word_list)
if os.path.exists(root_name):
root = load_model(root_name)
else:
dict_name = FLAGS.data_path + 'dict.txt'
word_freq = load_dictionary(dict_name)
root = TrieNode('*', word_freq)
save_model(root, root_name)
for word_list in data:
ngrams = generate_ngram(word_list, FLAGS.ngram)
for d in ngrams:
root.add(d)
te_re, add_word = root.find_word(FLAGS.topN, stop_word, jieba_dict, l_zre)
del root
return te_re
data = []
with open('../data/demo.txt', 'r') as f:
lines = f.readlines()
for line in lines:
line = line.strip()
line = [x for x in jieba.cut(line, cut_all=False) if x not in stopword]
data.append(line)
print('------> 初始化字典树')
root = TrieNode('*', word_freq)
print('------> 插入节点')
for i in data:
tmp = generate_ngram(i, 3)
for d in tmp:
root.add(d)
result, add_word = root.wordFind(5)
print('增加了%d个新词, 词语和得分分别为' % len(add_word))
print('#############################')
for word, score in add_word.items():
print(word + ' ----> ', score)
print('#############################')
# 如果想要调试和选择其他的阈值，可以print result来调整
# print(result)
test = '蔡英文在昨天应民进党当局的邀请，准备和陈时中一道前往世界卫生大会，和谈有关九二共识问题'
print('添加前：')
print("".join([(x + '/ ') for x in jieba.cut(test, cut_all=False)