def cal_f1_each_author(model_res, real_res): for author in model_res.keys(): model_author = {author: model_res[author]} real_author = {author: real_res[author]} f1 = pairwise_f1(real_author, model_author) print(author, '真实聚类数', len(real_author[author]), '预测聚类数', len(model_author[author]), '%.2f' % f1)
def get_score(self,graph): """ 计算给定图的得分 :param graph: :return: """ real_cluster={self.author_name: self.tag} model_cluster={self.author_name: self.get_graph_components(graph)} f1 = pairwise_f1(real_cluster, model_cluster) print(self.author_name, 'RealNum', len(real_cluster[self.author_name]), 'PreNum', len(model_cluster[self.author_name]),'F1','%.2f' % f1) return len(real_cluster[self.author_name]),len(model_cluster[self.author_name]),f1
def cal_train_f1(eps_cluster, n_component, valid_data, tag): author_name_list = list(valid_data.keys()) res_model_all = {} res_real_all = {} for i, author_name in enumerate(author_name_list): author_paper = valid_data[author_name] # 同名作者文章列表 if len(author_paper) == 0: print(author_name, '没有文章') continue res_model_author = { author_name: cluster_one_author(author_paper, 10, 0.5) } res_real_author = {author_name: tag[author_name]} # f1 = pairwise_f1(res_real_author, res_model_author) # print('i=', i, '同名作者:', author_name, '文章数量:', len(author_paper), '预测消歧作者数', len(res_model_author[author_name]), # '实际消歧作者数', len(tag[author_name]), 'f1-value', f1) res_model_all.update(res_model_author) res_real_all.update(res_real_author) f1 = pairwise_f1(res_real_all, res_model_all) return n_component, eps_cluster, f1
# -*- coding: utf-8 -*- """ Created on Thu Nov 14 10:38:05 2019 @author: chizj """ from graph import get_train_data from graph import GraphAuthors from common import pairwise_f1 if __name__ == '__main__': a_list = ['li_guo', 'qiang_xu'] train_data, train_tag = get_train_data(a_list) g_author = GraphAuthors(train_data['li_guo'], 'li_guo') g_author.add_edge() model_res = {'li_guo': g_author.get_connected_components()} real_res = {'li_guo': train_tag['li_guo']} pairwise_f1(real_res, model_res)
f1_dict = {} # 分析某个同名作者的文章信息 author_list_all = list(tag.keys()) author_list = author_list_all[1:20] for author_select in author_list: p_list = valid_data[author_select] pids, titles, keywords, abstract, authors, venue, year = get_paper_detail( p_list) real_result = {author_select: tag[author_select]} # model_result={author_select:rand_cluster(p_list)} model_result = { author_select: co_author_cluster(pids, authors, author_select) } f1 = pairwise_f1(real_result, model_result) print(author_select, f1) f1_dict[author_select] = f1 # org_test=[] # for co_au in authors: # for au in co_au: # if precessname(au['name']) == 'bo_shen': # oo=au['org'].split(';')[0] # if oo!='':org_test.append(oo) author_list_all = list(valid_data.keys()) author_list = author_list_all[0:20] # 根据共同作者计算保留的图结构 graph_dict = {} for author_select in author_list:
for plist in train_author_data[author].values(): p_merge.append(plist) res_real[author] = p_merge papers = {} for author in author_selects: p_merge = [] for key, value in train_author_data[author].items(): for p in value: # print(p) p_merge.append(train_pub_data[p]) papers[author] = p_merge # 根据合作者的图模型进行的聚类(名字预处理的结果会好一些,0.25》0.21) res_model2 = disambiguate_by_graph(papers) pairwise_f1(res_real, res_model2) # 模型测试 li_guo = papers['li_guo'] paper_words = get_papar_words(li_guo) dictionary = corpora.Dictionary(paper_words) bow_corpus = [dictionary.doc2bow(wl) for wl in paper_words] # 语料向量化 tfidf = models.TfidfModel(bow_corpus) # 基于向量化的语料构建tfidf模型 index = similarities.Similarity('E:\\gensim_test', tfidf[bow_corpus], len(dictionary)) sims = index[tfidf[bow_corpus]] # 计算相似性矩阵 i_cluster = graph_sim_matrix(sims, 0.15) res_modelx = {} res_modelx['li_guo'] = [[li_guo[index]['id'] for index in l_inside] for l_inside in i_cluster]
""" from PaperGraph import get_sample_data,get_valid_data from common import pairwise_f1,pairwise_f1_new import json if __name__=='__main__': data,tag=get_sample_data() from PaperGraph import PaperGraph authors=data.keys() for author in authors: p_graph=PaperGraph(name=author) p_graph.set_paper_info(data[author]) con=p_graph.get_res1() f1=pairwise_f1({author:tag[author]},{author:con}) print(author,'%.2f'%f1,len(tag[author]),len(con)) for author in authors: p_graph=PaperGraph(name=author) p_graph.set_paper_info(data[author]) p_graph.cal_node_pair_info() con=p_graph.get_res2() f1=pairwise_f1({author:tag[author]},{author:con}) print(author,'%.2f'%f1,len(tag[author]),len(con)) for author in authors: p_graph=PaperGraph(name=author) p_graph.set_paper_info(data[author]) p_graph.cal_node_pair_info() con=p_graph.get_res3()
for author_select in author_list: real_res = {author_select: t_tag[author_select]} model_author = { author_select: graph_author_dict[author_select].get_connected_components() } # model_author2={author_select: graph_author_dict2[author_select].get_connected_components()} # model_title={author_select:graph_title_dict[author_select].get_connected_components()} # model_at={author_select:graph_at_dict[author_select].get_connected_components()} # model_a2t={author_select:graph_a2t_dict[author_select].get_connected_components()} # model_at2={author_select:graph_at2_dict[author_select]} # model_at21={author_select:graph_at21_dict[author_select].get_cluster2(0.005,100)} # model_a2t21={author_select:graph_a2t21_dict[author_select].get_cluster2(0.1,500)} print(author_select, len(real_res[author_select])) print('----> model_author', '%.2f' % pairwise_f1(real_res, model_author), len(model_author[author_select])) # print('----> model_author2', # '%.2f'%pairwise_f1(real_res,model_author2),len(model_author2[author_select])) # print('----> model_title', # '%.2f'%pairwise_f1(real_res,model_title),len(model_title[author_select])) # print('----> model_at', # '%.2f'%pairwise_f1(real_res,model_at),len(model_at[author_select])) # print('----> model_a2t', # '%.2f'%pairwise_f1(real_res,model_a2t),len(model_a2t[author_select])) # print('----> model_at2', # '%.2f'%pairwise_f1(real_res,model_at2),len(model_at2[author_select])) # print('----> model_at21', # '%.2f'%pairwise_f1(real_res,model_at21),len(model_at21[author_select])) # print('----> model_a2t21', # '%.2f'%pairwise_f1(real_res,model_a2t21),len(model_a2t21[author_select]))
train_data = {} for author in author_selects: pid_list = [ train_pub_data[p] for pp in train_author_data[author].values() for p in pp ] train_data[author] = pid_list real_result = { key: [pid for pid in train_author_data[key].values()] for key in author_selects } # 训练数据真实标签 modle_result = disambiguate_by_graph_model(train_data, 0.5, 2) pairwise_f1(real_result, modle_result) # 采用多进程加快速度 train_data_list = [] for i in range(6): train_data = {} for author in author_list[i:20:6]: pid_list = [ train_pub_data[p] for pp in train_author_data[author].values() for p in pp ] train_data[author] = pid_list train_data_list.append(train_data) pool = Pool(processes=6) map_res = pool.map(disambiguate_by_graph_model, train_data_list) result_model = {}