def title_preprocess(df_title):
"""标题分词处理"""
df_title['title_'] = df_title['title'].map(
lambda x: preprocessing.clean_title(x))
df_title['title_'] = df_title['title_'].map(
lambda x: preprocessing.get_num_en_ch(x))
df_title['title_cut'] = df_title['title_'].map(
lambda x: preprocessing.pseg_cut(
x,
userdict_path=os.path.join(extra_dict_path, 'self_userdict.txt')))
df_title['title_cut'] = df_title['title_cut'].map(
lambda x: preprocessing.get_words_by_flags(
x, flags=['n.*', '.*n', 'v.*', 's', 'j', 'l', 'i', 'eng']))
df_title['title_cut'] = df_title['title_cut'].map(
lambda x: preprocessing.stop_words_cut(
x, os.path.join(extra_dict_path, 'HIT_stop_words.txt')))
df_title['title_cut'] = df_title['title_cut'].map(
lambda x: preprocessing.stop_words_cut(
x, os.path.join(extra_dict_path, 'self_stop_words.txt')))
df_title['title_cut'] = df_title['title_cut'].map(
lambda x: preprocessing.disambiguation_cut(
x, os.path.join(extra_dict_path, 'self_disambiguation_dict.json')))
df_title['title_cut'] = df_title['title_cut'].map(
lambda x: preprocessing.individual_character_cut(
x,
os.path.join(extra_dict_path, 'self_individual_character_dict.txt')
))
df_title['title_'] = df_title['title_cut'].map(lambda x: ' '.join(x))
return df_title
def preprocess():
if filter_df0.shape[0] == 0:
messagebox.showinfo('Message', '未选择新闻数据！')
return
df = filter_df0.copy()
df['title_'] = df['title'].map(
lambda x: preprocessing.clean_title_blank(x))
df['content_'] = df['content'].map(
lambda x: preprocessing.clean_content(x))
df['content_'] = df['content_'].map(
lambda x: preprocessing.get_num_en_ch(x))
df['content_cut'] = df['content_'].map(lambda x: preprocessing.pseg_cut(
x, userdict_path=os.path.join(extra_dict_path, 'self_userdict.txt')))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.get_words_by_flags(
x, flags=['n.*', 'v.*', 'eng', 't', 's', 'j', 'l', 'i']))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.stop_words_cut(
x, os.path.join(extra_dict_path, 'self_stop_words.txt')))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.disambiguation_cut(
x, os.path.join(extra_dict_path, 'self_disambiguation_dict.json')))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.individual_character_cut(
x,
os.path.join(extra_dict_path, 'self_individual_character_dict.txt')
))
df['content_'] = df['content_cut'].map(lambda x: ' '.join(x))
news_pandas.save_news(df, os.path.join(temp_news_path, 'news_cut.csv'))
messagebox.showinfo('Message', '数据预处理完成！')