def text_corrector(text, confusion: bool = True, non_level="char"):
"""Correct Text
Check text whether contains wrong word. If `confusion` is True, add
customize confusion words(can add more words, file path is
`config/ConfusionWords.txt`) that is a file path. Word level can use `char`,
`word` or None, `char` is character level which can check single wrong word;
None is can't support single word level
Parameters:
@type bool, confusion, whether add confusion word
@type string, non_level, choose a level, `char` close character level, `word`
close word level. None can support the two level
Results:
@type dict, results, wrong words, and those start index
Examples:
>>> text_corrector('少先队员因该为老人让坐') # add confusion and use char level
defaultdict(dict, {0: {'word': '因该', 'start_index': 4}})
>>> # add confusion and use all level
>>> text_corrector('少先队员因该为老人让坐', non_level=None)
{'因该': 4, '坐': 10}
"""
# add confusion dict
if confusion:
pycorrector.set_custom_confusion_dict(CONFUSION_PATH)
# import ipdb; ipdb.set_trace()
# get wrong word information
report = pycorrector.detect(text)
if len(report) > 0:
results = []
else:
return []
for index, item in enumerate(report):
result = {}
if non_level is None:
result["word"] = item[0]
result["start_index"] = item[1]
elif non_level == "char" and len(item[0]) > 1:
result["word"] = item[0]
result["start_index"] = item[1]
elif non_level == "word" and len(item[0]) == 1:
result["word"] = item[0]
result["start_index"] = item[1]
if result:
results.append(result)
return results
def test_confusion_dict(self):
sents = [
'买iphonex，要多少钱',
'共同实际控制人萧华、霍荣铨、张旗康',
]
res = []
for name in sents:
s, r = pycorrector.correct(name)
print(r)
res.append(r)
self.assertEqual(res[0], [])
self.assertEqual(res[1], [('张旗康', '张启康', 14, 17)])
pycorrector.set_custom_confusion_dict('../examples/my_custom_confusion.txt')
res = []
for name in sents:
s, r = pycorrector.correct(name)
print(r)
res.append(r)
self.assertEqual(res[0], [('iphonex', 'iphoneX', 1, 8)])
self.assertEqual(res[1], [])
def initialize(self):
jieba.set_dictionary(f'{self.server_assets_path}/dict.big.txt')
jieba.load_userdict(f'{self.server_assets_path}/ase_jieba_dict.txt')
set_custom_confusion_dict(
path=f'{self.server_assets_path}/my_custom_confusion.txt')
set_custom_word_freq(
path=f'{self.server_assets_path}/my_custom_word_freq.txt')
with open(f'{self.server_assets_path}/{self.stopword_file}',
'r',
encoding='utf-8') as f:
self.stop_words = f.read().split('\n')
f.close()
with open(f'{self.server_assets_path}/{self.keyword_file}',
'r',
encoding='utf-8') as f:
self.keywords = [
keyword for keyword in map(lambda x: x.replace('\n', ''),
f.readlines())
]
f.close()
if __name__ == '__main__':
error_sentences = [
'买iphonex，要多少钱', # 漏召回
'我想喝小明同学。', # 漏召回
'哪里卖苹果吧？请大叔给我让坐', # 漏召回
'交通先行了怎么过去啊？', # 漏召回
'共同实际控制人萧华、霍荣铨、张旗康', # 误杀
'上述承诺内容系本人真实意思表示', # 正常
'大家一哄而伞怎么回事', # 成语
]
for line in error_sentences:
print(pycorrector.correct(line))
print('*' * 42)
pycorrector.set_custom_confusion_dict(path='./my_custom_confusion.txt')
for line in error_sentences:
print(pycorrector.correct(line))
# ('买iphonex，要多少钱', [])
# ('我想喝小明同学。', [])
# ('哪里卖苹果吧？请大叔给我让坐', [])
# ('交通先行了怎么过去啊？', [])
# ('共同实际控制人萧华、霍荣铨、张启康', [['张旗康', '张启康', 14, 17]])
# ('上述承诺内容系本人真实意思表示', [])
# *****************************************************
# ('买iphoneX，要多少钱', [['iphonex', 'iphoneX', 1, 8]])
# ('我想喝小茗同学。', [['小明同学', '小茗同学', 3, 7]])
# ('哪里卖苹果八？请大叔给我让坐', [['苹果吧', '苹果八', 3, 6]])
# ('交通限行了怎么过去啊？', [['交通先行', '交通限行', 0, 4]])
# ('共同实际控制人萧华、霍荣铨、张旗康', [])