forked from gcbanana/find_confused_words
/
main.py
30 lines (22 loc) · 935 Bytes
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#!/usr/bin/env python
# -*- coding:utf-8 -*-
# Author: Jeffrey Gao
# Time: 2020/5/11 22:09
# Description: 发现混淆词主文件
from match.match import Match
from find_words.find_words import FindWords
corpus = []
with open('data/sentences.txt', 'r', encoding='utf-8') as f:
for line in f:
corpus.append(line.strip())
find_words_obj = FindWords(corpus=corpus,
top_k=500,
chunk_size=10000,
min_n=3,
max_n=5,
min_freq=10)
find_words_obj.find_new_words()
find_words_obj.to_file(my_list=find_words_obj.new_words, file_path='data/new_words.txt')
match_obj = Match(build_words_path='data/new_words.txt', input_words_path='data/new_words.txt')
print(match_obj.match_all())
# [('婚贝请柬', '婚贝请间'), ('黑马矿友', '黑码矿友'), ('填锐绿盾', '天锐绿盾'), ... ]