def crawler():
sina_top_n = Entry_Sina.get()
sohu_top_n = Entry_Sohu.get()
xinhuanet_top_n = Entry_XinhuaNet.get()
sina_top_n = 0 if sina_top_n == '' else int(sina_top_n)
sohu_top_n = 0 if sohu_top_n == '' else int(sohu_top_n)
xinhuanet_top_n = 0 if xinhuanet_top_n == '' else int(xinhuanet_top_n)
sina_top_n = 0 if sina_top_n <= 0 else sina_top_n
sohu_top_n = 0 if sohu_top_n <= 0 else sohu_top_n
xinhuanet_top_n = 0 if xinhuanet_top_n <= 0 else xinhuanet_top_n
if sina_top_n + sohu_top_n + xinhuanet_top_n == 0:
messagebox.showinfo('Message', '新闻数量不能全部为非正数！')
return
news_df_file_path = os.path.join(news_path, 'news_df.csv')
threaded_crawler(sina_top_n,
sohu_top_n,
xinhuanet_top_n,
save_file_path=news_df_file_path)
news_df = load_news(news_df_file_path)
global filter_df
filter_df = preprocessing.data_filter(news_df)
news_pandas.save_news(filter_df,
os.path.join(temp_news_path, 'filter_news.csv'))
news_num = filter_df.shape[0]
sum_top_n.set(news_num)
messagebox.showinfo('Message', '爬取即时新闻完成！共{}条有效新闻！'.format(news_num))
def cluster_word():
n_clusters = Entry_N_Clusters.get()
if n_clusters == '':
messagebox.showinfo('Message', '请输入词汇聚类的类别数！')
return
n_clusters = int(n_clusters)
top_words_list = counter.get_most_common_words(
df_rank_i['content_cut'], top_n=5000, min_frequency=1)
model = news_pandas.load_element(
os.path.join(models_path, 'word2vec_model.pkl'))
word_list, wordvec_list = modeling.get_word_and_wordvec(
model, top_words_list)
kmeans = modeling.get_cluster(wordvec_list,
cluster='KMeans',
cluster_args={
'n_clusters': n_clusters,
'random_state': 9
})
word_label = kmeans.labels_
word_df = pd.DataFrame()
word_df['word'] = word_list
word_df['wordvec'] = wordvec_list
word_df['word_label'] = word_label
news_pandas.save_news(word_df, os.path.join(results_path,
'word_df.csv'))
messagebox.showinfo('Message', '词汇聚类完成！')
def preprocess():
if filter_df0.shape[0] == 0:
messagebox.showinfo('Message', '未选择新闻数据！')
return
df = filter_df0.copy()
df['title_'] = df['title'].map(
lambda x: preprocessing.clean_title_blank(x))
df['content_'] = df['content'].map(
lambda x: preprocessing.clean_content(x))
df['content_'] = df['content_'].map(
lambda x: preprocessing.get_num_en_ch(x))
df['content_cut'] = df['content_'].map(lambda x: preprocessing.pseg_cut(
x, userdict_path=os.path.join(extra_dict_path, 'self_userdict.txt')))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.get_words_by_flags(
x, flags=['n.*', 'v.*', 'eng', 't', 's', 'j', 'l', 'i']))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.stop_words_cut(
x, os.path.join(extra_dict_path, 'self_stop_words.txt')))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.disambiguation_cut(
x, os.path.join(extra_dict_path, 'self_disambiguation_dict.json')))
df['content_cut'] = df['content_cut'].map(
lambda x: preprocessing.individual_character_cut(
x,
os.path.join(extra_dict_path, 'self_individual_character_dict.txt')
))
df['content_'] = df['content_cut'].map(lambda x: ' '.join(x))
news_pandas.save_news(df, os.path.join(temp_news_path, 'news_cut.csv'))
messagebox.showinfo('Message', '数据预处理完成！')
def cluster_content():
eps_var = Entry_Eps.get()
min_samples_var = Entry_MinSamples.get()
if eps_var == '' or min_samples_var == '':
messagebox.showinfo('Message', '请输全聚类参数！')
return
eps_var = float(eps_var)
min_samples_var = int(min_samples_var)
try:
df = news_pandas.load_news(os.path.join(temp_news_path,
'news_cut.csv'))
df['content_cut'] = df['content_cut'].map(eval)
df['content_'] = df['content_'].map(str)
except FileNotFoundError:
messagebox.showinfo('Message', '请先对新闻内容文本进行预处理！')
return
word_library_list = counter.get_word_library(df['content_cut'])
single_frequency_words_list = counter.get_single_frequency_words(
df['content_cut'])
max_features = len(
word_library_list) - len(single_frequency_words_list) // 2
matrix = modeling.feature_extraction(df['content_'],
vectorizer='TfidfVectorizer',
vec_args={
'max_df': 0.95,
'min_df': 1,
'max_features': max_features
})
dbscan = modeling.get_cluster(matrix,
cluster='DBSCAN',
cluster_args={
'eps': eps_var,
'min_samples': min_samples_var,
'metric': 'cosine'
})
labels = modeling.get_labels(dbscan)
df['label'] = labels
ranks = modeling.label2rank(labels)
df['rank'] = ranks
news_pandas.save_news(df, os.path.join(results_path, 'news_label.csv'))
df['matrix'] = matrix.toarray().tolist()
df_non_outliers = df[df['label'] != -1].copy()
if df_non_outliers.shape[0] == 0:
messagebox.showinfo('Message', '不能聚类出任何热点，请重新设置聚类参数！')
return
data_pca_tsne = modeling.feature_reduction(
df_non_outliers['matrix'].tolist(),
pca_n_components=3,
tsne_n_components=2)
df_non_outliers['pca_tsne'] = data_pca_tsne.tolist()
del df_non_outliers['matrix']
news_pandas.save_news(df_non_outliers,
os.path.join(results_path, 'news_non_outliers.csv'))
rank_num = counter.get_num_of_value_no_repeat(df_non_outliers['rank'])
hot_num.set(rank_num)
messagebox.showinfo('Message', '按照新闻内容聚类完成！')
def select_news():
filename = filedialog.askopenfilename(filetypes=[("csv file", "*.csv")])
if len(filename) == 0:
return
news_df = news_pandas.load_news(filename)
news_pandas.save_news(news_df, os.path.join(news_path, 'news_df.csv'))
global filter_df
filter_df = preprocessing.data_filter(news_df)
news_pandas.save_news(filter_df, os.path.join(temp_news_path, 'filter_news.csv'))
news_num = filter_df.shape[0]
sum_top_n.set(news_num)
def data_filter():
if filter_df.shape[0] == 0:
messagebox.showinfo('Message', '未选择任何新闻数据！')
return
date_f = Entry_Date.get()
day_f = Entry_Day.get()
if date_f == '' or day_f == '':
messagebox.showinfo('Message', '请先填写筛选的日期和天数！')
return
global filter_df0
filter_df0 = preprocessing.get_data(filter_df, last_time=date_f + ' 23:59', delta=int(day_f))
news_pandas.save_news(filter_df0, os.path.join(temp_news_path, 'filter_news_by_time.csv'))
news_num = filter_df0.shape[0]
filter_n.set(news_num)
def crawler():
sina_top_n = Entry_Sina.get()
sohu_top_n = Entry_Sohu.get()
xinhuanet_top_n = Entry_XinhuaNet.get()
sina_top_n = 0 if sina_top_n == '' else int(sina_top_n)
sohu_top_n = 0 if sohu_top_n == '' else int(sohu_top_n)
xinhuanet_top_n = 0 if xinhuanet_top_n == '' else int(xinhuanet_top_n)
sina_top_n = 0 if sina_top_n <= 0 else sina_top_n
sohu_top_n = 0 if sohu_top_n <= 0 else sohu_top_n
xinhuanet_top_n = 0 if xinhuanet_top_n <= 0 else xinhuanet_top_n
if sina_top_n + sohu_top_n + xinhuanet_top_n == 0:
messagebox.showinfo('Message', '新闻数量不能全部为非正数！')
return
news_crawler.threaded_crawler(sina_top_n, sohu_top_n, xinhuanet_top_n)
sina_news_df = pd.DataFrame()
sohu_news_df = pd.DataFrame()
xinhuanet_news_df = pd.DataFrame()
if sina_top_n > 0:
sina_news_df = news_pandas.load_news(
os.path.join(news_path, 'sina_latest_news.csv'))
if sohu_top_n > 0:
sohu_news_df = news_pandas.load_news(
os.path.join(news_path, 'sohu_latest_news.csv'))
if xinhuanet_top_n > 0:
xinhuanet_news_df = news_pandas.load_news(
os.path.join(news_path, 'xinhuanet_latest_news.csv'))
news_df = pd.concat([sina_news_df, sohu_news_df, xinhuanet_news_df],
ignore_index=True)
news_pandas.save_news(news_df, os.path.join(news_path, 'news_df.csv'))
global filter_df
filter_df = preprocessing.data_filter(news_df)
news_pandas.save_news(filter_df,
os.path.join(temp_news_path, 'filter_news.csv'))
news_num = filter_df.shape[0]
sum_top_n.set(news_num)
messagebox.showinfo('Message', '爬取即时新闻完成！共{}条有效新闻！'.format(news_num))
