result[c] |= set(parts)
return result
char_dict_path = os.path.join(pwd_path, config.char_dict_path)
cn_char_set = load_char_dict(char_dict_path)
two_char_dict = load_2char_dict(pwd_path + '/data/char_two_set.txt')
# # word dictionary
word_dict_text_path = os.path.join(pwd_path, config.word_dict_path)
word_dict_model_path = os.path.join(pwd_path, config.word_dict_model_path)
if os.path.exists(word_dict_model_path):
cn_word_set = load_pkl(word_dict_model_path)
else:
default_logger.debug('load word dict from text file:', word_dict_model_path)
cn_word_set = load_word_dict(word_dict_text_path)
dump_pkl(cn_word_set, word_dict_model_path)
# similar pronuciation
same_pinyin_text_path = os.path.join(pwd_path, config.same_pinyin_text_path)
same_pinyin_model_path = os.path.join(pwd_path, config.same_pinyin_model_path)
# same_pinyin = load_same_pinyin(same_pinyin_text_path)
if os.path.exists(same_pinyin_model_path):
same_pinyin = load_pkl(same_pinyin_model_path)
else:
default_logger.debug('load same pinyin from text file:', same_pinyin_text_path)
same_pinyin = load_same_pinyin(same_pinyin_text_path)
dump_pkl(same_pinyin, same_pinyin_model_path)
# similar shape
same_stroke_text_path = os.path.join(pwd_path, config.same_stroke_text_path)
same_stroke_model_path = os.path.join(pwd_path, config.same_stroke_model_path)
info = line.split()
word = info[0]
freq = int(info[1])
word_freq[word] = freq
return word_freq
# 字频统计
word_dict_path = os.path.join(pwd_path, config.word_dict_path)
word_dict_model_path = os.path.join(pwd_path, config.word_dict_model_path)
if os.path.exists(word_dict_model_path):
word_freq = load_pkl(word_dict_model_path)
else:
default_logger.debug('load word freq from text file:', word_dict_path)
word_freq = load_word_freq_dict(word_dict_path)
dump_pkl(word_freq, word_dict_model_path)
def get_ngram_score(chars, mode=trigram_char):
"""
取n元文法得分
:param chars: list, 以词或字切分
:param mode:
:return:
"""
return mode.score(' '.join(chars), bos=False, eos=False)
def get_ppl_score(words, mode=trigram_char):
"""
取语言模型困惑度得分，越小句子越通顺
info = line.split()
word = info[0]
freq = int(info[1])
word_freq[word] = freq
return word_freq
# 字频统计
word_freq_path = os.path.join(pwd_path, config.word_freq_path)
word_freq_model_path = os.path.join(pwd_path, config.word_freq_model_path)
if os.path.exists(word_freq_model_path):
word_freq = load_pkl(word_freq_model_path)
else:
default_logger.debug('load word freq from text file:', word_freq_path)
word_freq = load_word_freq_dict(word_freq_path)
dump_pkl(word_freq, word_freq_model_path)
def get_ngram_score(chars, mode=trigram_char):
"""
取n元文法得分
:param chars: list, 以词或字切分
:param mode:
:return:
"""
return mode.score(' '.join(chars), bos=False, eos=False)
def get_ppl_score(words, mode=trigram_char):
"""
取语言模型困惑度得分，越小句子越通顺
for i, c in enumerate(parts):
result[c] = set(list(parts[:i] + parts[i + 1:]))
return result
cn_char_set = load_word_dict(char_file_path)
same_pinyin_text_path = os.path.join(pwd_path, config.same_pinyin_text_path)
same_pinyin_model_path = os.path.join(pwd_path, config.same_pinyin_model_path)
# 同音字
if os.path.exists(same_pinyin_model_path):
same_pinyin = load_pkl(same_pinyin_model_path)
else:
default_logger.debug('load same pinyin from text file:',
same_pinyin_text_path)
same_pinyin = load_same_pinyin(same_pinyin_text_path)
dump_pkl(same_pinyin, same_pinyin_model_path)
# 形似字
same_stroke_text_path = os.path.join(pwd_path, config.same_stroke_text_path)
same_stroke_model_path = os.path.join(pwd_path, config.same_stroke_model_path)
if os.path.exists(same_stroke_model_path):
same_stroke = load_pkl(same_stroke_model_path)
else:
default_logger.debug('load same stroke from text file:',
same_stroke_text_path)
same_stroke = load_same_stroke(same_stroke_text_path)
dump_pkl(same_stroke, same_stroke_model_path)
def get_same_pinyin(char):
"""
if parts and len(parts) > 1:
for i, c in enumerate(parts):
result[c] = set(list(parts[:i] + parts[i + 1:]))
return result
cn_char_set = load_word_dict(char_file_path)
same_pinyin_text_path = os.path.join(pwd_path, config.same_pinyin_text_path)
same_pinyin_model_path = os.path.join(pwd_path, config.same_pinyin_model_path)
# 同音字
if os.path.exists(same_pinyin_model_path):
same_pinyin = load_pkl(same_pinyin_model_path)
else:
default_logger.debug('load same pinyin from text file:', same_pinyin_text_path)
same_pinyin = load_same_pinyin(same_pinyin_text_path)
dump_pkl(same_pinyin, same_pinyin_model_path)
# 形似字
same_stroke_text_path = os.path.join(pwd_path, config.same_stroke_text_path)
same_stroke_model_path = os.path.join(pwd_path, config.same_stroke_model_path)
if os.path.exists(same_stroke_model_path):
same_stroke = load_pkl(same_stroke_model_path)
else:
default_logger.debug('load same stroke from text file:', same_stroke_text_path)
same_stroke = load_same_stroke(same_stroke_text_path)
dump_pkl(same_stroke, same_stroke_model_path)
def get_same_pinyin(char):
"""
取同音字
info = line.split()
word = info[0]
freq = int(info[1])
word_freq[word] = freq
return word_freq
# 字频统计
word_freq_path = os.path.join(pwd_path, config.word_freq_path)
word_freq_model_path = os.path.join(pwd_path, config.word_freq_model_path)
if os.path.exists(word_freq_model_path):
word_freq = load_pkl(word_freq_model_path)
else:
default_logger.debug('load word freq from text file:', word_freq_path)
word_freq = load_word_freq_dict(word_freq_path)
dump_pkl(word_freq, word_freq_model_path)
def get_ngram_score(chars, mode=trigram_char):
"""
取n元文法得分
:param chars: list, 以词或字切分
:param mode:
:return:
"""
return mode.score(' '.join(chars), bos=False, eos=False)
def get_ppl_score(words, mode=trigram_char):
"""
取语言模型困惑度得分，越小句子越通顺