def __init__(self, text_dict, data_set_name, relscore_type, tf_ver, idf_ver, k1, b): self.data_set_name = data_set_name # generate relscore by Relscore Relscore(text_dict, data_set_name, tf_ver, idf_ver, k1, k1, b, b) # generate relscore by make_relscore make_relscore(k1, b, self.change_to_num(relscore_type)) # open relScore and rel_score/.. relscore_dir_name = "%s_%s_%s" % (relscore_type, str(tf_ver), str(idf_ver)) if relscore_type == "bm25": relscore_dir_name += "_%s_%s" % (str(k1), str(b)) for file_name in text_dict.keys(): old_relscore = read_relscore("../relScore", file_name) refactor_relscore = read_relscore("../rel_score/%s/%s" % (data_set_name, relscore_dir_name), file_name) # compare relscore # check relscore.keys()s are equal and relscore.values()'s equality self.check_equality(file_name, old_relscore, refactor_relscore)
def run_evaluation(self, ref, crank_flag, single_rank_flag, expand_rank_flag): file_set = set(self.text_dict.keys()) ref_file_list = ref.keys() relscore = dict() eval_score = self.init_eval_score() for file_name in deepcopy(file_set): if file_name in ref_file_list: relscore[file_name] = read_relscore(self.relscore_path, file_name) else: print 'ref file<%s> does not exist in the given gold_standard data' % (file_name) file_set.remove(file_name) self.excluded_files.add(file_name) if expand_rank_flag: graph_dict = dict() for winsize in self.winsize_list: graph_dict[winsize] = dict() for file_name in file_set: graph_dict[winsize][file_name] = Graph(file_name, self.text_dict[file_name], relscore[file_name], winsize, self.lamb_list, self.cutoff_list, return_graph_flag=True, crank_flag=crank_flag, single_rank_flag=single_rank_flag, input_graph=None).graph self.update_neighbor_weight(graph_dict[winsize]) for winsize in self.winsize_list: new_gs = dict() for file_name in file_set: input_graph = None if expand_rank_flag: input_graph = graph_dict[winsize][file_name] graph = Graph(file_name, self.text_dict[file_name], relscore[file_name], winsize, self.lamb_list, self.cutoff_list, return_graph_flag=False, crank_flag=crank_flag, single_rank_flag=single_rank_flag, input_graph=input_graph, lda_dict=self.lda_dict) checked_gs_list = self.check_gold_standard(graph, ref[file_name]) new_gs[file_name] = checked_gs_list for cutoff in self.cutoff_list: for lamb in self.lamb_list: candidates = graph.score_candidates(cutoff, lamb) file_path = '../checked_inspec.ref' save_candidates(file_path, winsize, cutoff, lamb, candidates); eval_score[winsize][cutoff][lamb][file_name] = ( evaluate(self.stem_list(candidates), self.stem_list(checked_gs_list))) write_gs('./checked_inspec.ref', new_gs) return eval_score
if len(relscore) > 4: k1 = relscore[3] b = relscore[4] ft1 = RelscoreFunctionalTest( text_dict, data_set_name, relscore[0], int(relscore[1]), int(relscore[2]), float(k1), float(b) ) # check equality of two methods of getting keyphrase_candidate for relscore_type in settings["relscore"]: for winsize in settings["winsize"]: print "relscore_type : %s, winsize : %s" % (relscore_type, winsize) old_eval_score = dict() refactor_eval_score = dict() old_gs = old_read_gs("/Users/KimKR/Desktop/NEXT_LAB/keyword/gold_standard/%s.ref" % (data_set_name)) refactor_gs = read_gs("/Users/KimKR/Desktop/NEXT_LAB/keyword/gold_standard/%s.ref" % (data_set_name)) for file_name in text_dict.keys(): sentences = text_dict[file_name] old_relscore = read_relscore( ("/Users/KimKR/Desktop/NEXT_LAB/keyword/rel_score/%s/%s" % (settings["dataset"], relscore_type)), file_name, ) refact_relscore = read_relscore("/Users/KimKR/Desktop/NEXT_LAB/keyword/relScore", file_name) # check equality of keyphrase_candidates ft = GraphFunctionalTest(file_name, sentences, old_relscore, refact_relscore, int(winsize)) # old_gs[file_name] = old_check_gold_standard(sentences, old_gs[file_name]) # refactor_gs[file_name] = check_gold_standard(sentences, refactor_gs[file_name]) old_eval_score[file_name] = evaluate(ft.kokako_graph.score_candidates(), old_gs[file_name]) refactor_eval_score[file_name] = evaluate( ft.refactored_graph.score_candidates(1, ft.lamb), refactor_gs[file_name] ) dict_check_equality(file_name, old_eval_score[file_name], refactor_eval_score[file_name])