def run(self): starttime = time.time() rootName = (self.rootDir) if os.path.exists(rootName): root = loadModel(rootName) else: dictName = self.dictDir word_freq = loadWords(dictName) root = TrieNode('*', word_freq) saveModel(root, rootName) # 加载新的文章 fileName = self.demoDir data = self.loadData(fileName, self.stopwords) # 将新的文章插入到Root中 self.loadData2Root(root, data) # 定义取TOP5个 N = 5 result, add_word = root.wordFind(N) # 如果想要调试和选择其他的阈值,可以print result来调整 print("\n----\n", '增加了 %d 个新词, 词语和得分分别为: \n' % len(add_word)) print('#############################') for word, score in add_word.items(): print(word + ' ----> ', score) print('#############################\n') for word, score in add_word.items(): jieba.add_word(word) print("互信息、信息熵:") print("".join([(x + '/ ') for x in jieba.cut(self.test_text, cut_all=False) if x not in self.stopwords])) endtime = time.time() print('time cost:' + str(round((endtime - starttime), 4)) + ' seconds.\n')
lines = f.readlines() for line in lines: line = line.strip() line = [x for x in jieba.cut(line, cut_all=False) if x not in stopword] data.append(line) print('------> 初始化字典树') root = TrieNode('*', word_freq) print('------> 插入节点') for i in data: tmp = generate_ngram(i, 3) for d in tmp: root.add(d) result, add_word = root.wordFind(5) print('增加了%d个新词, 词语和得分分别为' % len(add_word)) print('#############################') for word, score in add_word.items(): print(word + ' ----> ', score) print('#############################') # 如果想要调试和选择其他的阈值,可以print result来调整 # print(result) test = '蔡英文在昨天应民进党当局的邀请,准备和陈时中一道前往世界卫生大会,和谈有关九二共识问题' print('添加前:') print("".join([(x + '/ ') for x in jieba.cut(test, cut_all=False) if x not in stopword]))