def setUp(self): self.evaluator = rouge.TimelineRougeEvaluator() self.ground_truth = timelines.GroundTruth( [ timelines.Timeline( { datetime.date(2010, 1, 1): ["timeline summarization ."], datetime.date(2010, 1, 2): ["timeline summarization is awesome .", "coreference resolution is , too ."], datetime.date(2010, 1, 4): ["alignments are really nice"] } ), timelines.Timeline( { datetime.date(2010, 1, 2): ["metrics are complicated ."] } ), ] ) self.output_same_number_of_dates = timelines.Timeline( { datetime.date(2010, 1, 2): ["timeline summarization ."], datetime.date(2010, 1, 3): ["doing some metric checks ."], datetime.date(2010, 1, 5): ["checks for alignments ."], } ) self.output_same_number_of_dates_scores_higher_with_date_content_costs = timelines.Timeline( { datetime.date(2010, 1, 2): ["timeline summarization ."], datetime.date(2010, 1, 3): ["alignments are really nice"], datetime.date(2010, 1, 5): ["timeline summarization ."], } ) self.output_less_dates = timelines.Timeline( { datetime.date(2010, 1, 2): ["timeline summarization ."], datetime.date(2010, 1, 3): ["doing some metric checks ."], } ) self.output_more_dates = timelines.Timeline( { datetime.date(2010, 1, 2): ["timeline summarization ."], datetime.date(2010, 1, 3): ["doing some metric checks ."], datetime.date(2010, 1, 5): ["checks for alignments ."], datetime.date(2010, 1, 6): ["asdf"], } )
# news_corpora[topic] = news_corpora[topic].filter_by_keywords_contained(keyword_mapping[topic]) # read groundtruth timelines for filename in sorted( list(os.listdir(raw_directory + "/" + topic + "/timelines/"))): full_path = raw_directory + "/" + topic + "/timelines/" + filename temp_reference_timelines[topic].append( timelines.Timeline.from_file( codecs.open(full_path, "r", "utf-8", "replace"))) for topic in temp_reference_timelines: reference_timelines[topic] = timelines.GroundTruth( temp_reference_timelines[topic]) evaluator = rouge.TimelineRougeEvaluator(measures=["rouge_1", "rouge_2"], beta=1) model_avg_scores = {} for _config in configs: config_file = modelbase_dir + _config print(_config) print(config_file) config = json.load(open(config_file)) config["rouge_computation"] = "reimpl" logging.info(config) algorithm = None if config["algorithm"] == "chieu":
def evaluate_tl_main(): parser = argparse.ArgumentParser() parser.add_argument("-f", dest="filter_corpus", default=False, action="store_true") parser.add_argument("-c", dest="constraint", default="sent") parser.add_argument("-t", dest="timelines", nargs="+") parser.add_argument("-m", dest="num_multi_selection_runs", type=int, default=None) parser.add_argument("--queryfile") parser.add_argument("corpus_pickle") parser.add_argument("config") args = parser.parse_args() if args.constraint == "sent": use_token_count = False elif args.constraint == "tok": use_token_count = True else: raise ValueError("Unknown constraint {}".format(args.constraint)) corpus = load_corpus(args.corpus_pickle, filter_blacklist=args.filter_corpus) timelines = [] for tl_fname in args.timelines: with open(tl_fname, errors="ignore") as f: timeline = Timeline.from_file(f) timelines.append((os.path.basename(tl_fname), timeline)) #tl_gen = APClusteringTimelineGenerator(True) with open(args.config) as f: config = json.load(f) tl_gen = GloballyClusteredSentenceCompressionTimelineGenerator(config) corpus_basename = os.path.basename(corpus.name).split(".")[0] print(corpus_basename) config_basename = os.path.basename(args.config) results_basename = config_basename if args.queryfile: results_basename += "+queryfilter" out_timelines_dir = os.path.join("system_timelines", results_basename + "+" + args.constraint, corpus_basename) results_dir = os.path.join("evaluation_results", results_basename + "+" + args.constraint) if not os.path.isdir(out_timelines_dir): os.makedirs(out_timelines_dir) if not os.path.isdir(results_dir): os.makedirs(results_dir) query_words = None if args.queryfile is not None: with open(args.queryfile) as f: query_words = [l.strip() for l in f] debug_identifier = results_basename + "+" + corpus_basename if use_token_count: config["scoring"]["use_length"] = True if args.num_multi_selection_runs is None: sys_timelines = tl_gen.generate_timelines( corpus, [ determine_tl_parameters(tl, use_token_count=use_token_count) for _, tl in timelines ], reference_timelines=list(map(lambda x: x[1], timelines)), query_words=query_words, debug_identifier=debug_identifier) write_results_file(os.path.join(results_dir, corpus_basename + ".txt"), out_timelines_dir, timelines, sys_timelines) else: with open("multirun-results+{}.txt".format(config_basename), "a") as f_out: print(timelines) evaluator = rouge.TimelineRougeEvaluator( measures=["rouge_1", "rouge_2"]) all_run_timelines = tl_gen.generate_timelines( corpus, [ determine_tl_parameters(tl, use_token_count=use_token_count) for _, tl in timelines ], reference_timelines=list(map(lambda x: x[1], timelines)), query_words=query_words, debug_identifier=debug_identifier, num_selection_runs=args.num_multi_selection_runs) for sys_timelines in all_run_timelines: for (timeline_name, gold_timeline), sys_timeline in zip( timelines, sys_timelines): reference_timeline = GroundTruth([gold_timeline]) eval_results = evaluator.evaluate_concat( "TL", sys_timeline, reference_timeline) eval_results_agree = evaluator.evaluate_agreement( "TL", sys_timeline, reference_timeline) eval_results_align = evaluator.evaluate_align_date_content_costs_many_to_one( "TL", sys_timeline, reference_timeline) f_out.write(" ".join( map(str, [ eval_results["rouge_1"]["f_score"], eval_results["rouge_2"]["f_score"], eval_results_agree["rouge_1"]["f_score"], eval_results_agree["rouge_2"]["f_score"], eval_results_align["rouge_1"]["f_score"], eval_results_align["rouge_2"]["f_score"] ]))) f_out.write("\n") f_out.write("--------\n") f_out.write("========\n")
def write_results_file(outfilename, out_timelines_dir, timelines, sys_timelines): evaluator = rouge.TimelineRougeEvaluator(measures=["rouge_1", "rouge_2"]) rouge_1_sum = 0 rouge_1_r_sum = 0 rouge_1_p_sum = 0 rouge_2_sum = 0 rouge_2_r_sum = 0 rouge_2_p_sum = 0 agree_rouge_1_sum = 0 agree_rouge_1_r_sum = 0 agree_rouge_1_p_sum = 0 agree_rouge_2_sum = 0 agree_rouge_2_r_sum = 0 agree_rouge_2_p_sum = 0 align_rouge_1_sum = 0 align_rouge_1_r_sum = 0 align_rouge_1_p_sum = 0 align_rouge_2_sum = 0 align_rouge_2_r_sum = 0 align_rouge_2_p_sum = 0 date_f1_sum = 0 date_f1_r_sum = 0 date_f1_p_sum = 0 with open(outfilename, "w") as f_out: f_out.write( "Timeline \tDate R\tDate P\tDate F1\tR1 R\tR1 P\tR1 F1\tR2 R\tR2 P\tR2 F1\tR1 R\tR1 P\tR1 F1\tR2 R\tR2 P\tR2 F1\tR1 R\tR1 P\tR1 F1\tR2 R\tR2 P\tR2 F1\n" ) for (timeline_name, gold_timeline), sys_timeline in zip(timelines, sys_timelines): with open(os.path.join(out_timelines_dir, timeline_name), "w") as f_tl: f_tl.write(str(sys_timeline)) reference_timeline = GroundTruth([gold_timeline]) eval_results = evaluator.evaluate_concat(sys_timeline, reference_timeline) rouge_1_sum += eval_results["rouge_1"]["f_score"] rouge_1_r_sum += eval_results["rouge_1"]["recall"] rouge_1_p_sum += eval_results["rouge_1"]["precision"] rouge_2_sum += eval_results["rouge_2"]["f_score"] rouge_2_r_sum += eval_results["rouge_2"]["recall"] rouge_2_p_sum += eval_results["rouge_2"]["precision"] eval_results_agree = evaluator.evaluate_agreement( sys_timeline, reference_timeline) agree_rouge_1_sum += eval_results_agree["rouge_1"]["f_score"] agree_rouge_1_r_sum += eval_results_agree["rouge_1"]["recall"] agree_rouge_1_p_sum += eval_results_agree["rouge_1"]["precision"] agree_rouge_2_sum += eval_results_agree["rouge_2"]["f_score"] agree_rouge_2_r_sum += eval_results_agree["rouge_2"]["recall"] agree_rouge_2_p_sum += eval_results_agree["rouge_2"]["precision"] eval_results_align = evaluator.evaluate_align_date_content_costs_many_to_one( sys_timeline, reference_timeline) align_rouge_1_sum += eval_results_align["rouge_1"]["f_score"] align_rouge_1_r_sum += eval_results_align["rouge_1"]["recall"] align_rouge_1_p_sum += eval_results_align["rouge_1"]["precision"] align_rouge_2_sum += eval_results_align["rouge_2"]["f_score"] align_rouge_2_r_sum += eval_results_align["rouge_2"]["recall"] align_rouge_2_p_sum += eval_results_align["rouge_2"]["precision"] print(" ".join( map(lambda x: "{}-{}-{}".format(x.year, x.month, x.day), sorted(sys_timeline)))) print(" ".join( map(lambda x: "{}-{}-{}".format(x.year, x.month, x.day), sorted(gold_timeline)))) date_recall = len(set(sys_timeline) & set(gold_timeline)) / len(gold_timeline) date_precision = len(set(sys_timeline) & set(gold_timeline)) / len(sys_timeline) if date_recall + date_precision > 0: date_f1 = 2 * (date_recall * date_precision) / (date_recall + date_precision) else: date_f1 = 0.0 date_f1_sum += date_f1 date_f1_r_sum += date_recall date_f1_p_sum += date_precision f_out.write( "{:<16}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\n" .format(timeline_name, date_recall, date_precision, date_f1, eval_results["rouge_1"]["recall"], eval_results["rouge_1"]["precision"], eval_results["rouge_1"]["f_score"], eval_results["rouge_2"]["recall"], eval_results["rouge_2"]["precision"], eval_results["rouge_2"]["f_score"], eval_results_agree["rouge_1"]["recall"], eval_results_agree["rouge_1"]["precision"], eval_results_agree["rouge_1"]["f_score"], eval_results_agree["rouge_2"]["recall"], eval_results_agree["rouge_2"]["precision"], eval_results_agree["rouge_2"]["f_score"], eval_results_align["rouge_1"]["recall"], eval_results_align["rouge_1"]["precision"], eval_results_align["rouge_1"]["f_score"], eval_results_align["rouge_2"]["recall"], eval_results_align["rouge_2"]["precision"], eval_results_align["rouge_2"]["f_score"])) f_out.write( "{:<16}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\t{:.4f}\n" .format( "All", date_f1_r_sum / len(timelines), date_f1_p_sum / len(timelines), date_f1_sum / len(timelines), rouge_1_r_sum / len(timelines), rouge_1_p_sum / len(timelines), rouge_1_sum / len(timelines), rouge_2_r_sum / len(timelines), rouge_2_p_sum / len(timelines), rouge_2_sum / len(timelines), agree_rouge_1_r_sum / len(timelines), agree_rouge_1_p_sum / len(timelines), agree_rouge_1_sum / len(timelines), agree_rouge_2_r_sum / len(timelines), agree_rouge_2_p_sum / len(timelines), agree_rouge_2_sum / len(timelines), align_rouge_1_r_sum / len(timelines), align_rouge_1_p_sum / len(timelines), align_rouge_1_sum / len(timelines), align_rouge_2_r_sum / len(timelines), align_rouge_2_p_sum / len(timelines), align_rouge_2_sum / len(timelines))) #print(sys_timeline) print("ROUGE 1", rouge_1_sum / len(timelines)) print("ROUGE 2", rouge_2_sum / len(timelines)) print("Date F1", date_f1_sum / len(timelines))
def evaluate(tls_model, dataset, result_path, trunc_timelines=False, time_span_extension=0, word_mover_stop_words='nltk'): results = [] metric = 'align_date_content_costs_many_to_one' evaluator = tilse_rouge.TimelineRougeEvaluator( measures=["rouge_1", "rouge_2"]) n_topics = len(dataset.collections) for i, collection in enumerate(dataset.collections): ref_timelines = [ TilseTimeline(tl.date_to_summaries) for tl in collection.timelines ] topic = collection.name n_ref = len(ref_timelines) if trunc_timelines: ref_timelines = data.truncate_timelines(ref_timelines, collection) for j, ref_timeline in enumerate(ref_timelines): print( f'topic {i + 1}/{n_topics}: {topic}, ref timeline {j + 1}/{n_ref}' ) tls_model.load(ignored_topics=[collection.name]) ref_dates = sorted(ref_timeline.dates_to_summaries) start, end = data.get_input_time_span(ref_dates, time_span_extension) collection.start = start collection.end = end # utils.plot_date_stats(collection, ref_dates) l = len(ref_dates) k = data.get_average_summary_length(ref_timeline) pred_timeline_ = tls_model.predict( collection, max_dates=l, max_summary_sents=k, ref_tl=ref_timeline # only oracles need this ) print('*** PREDICTED ***') utils.print_tl(pred_timeline_) print('timeline done') pred_timeline = TilseTimeline(pred_timeline_.date_to_summaries) sys_len = len(pred_timeline.get_dates()) ground_truth = TilseGroundTruth([ref_timeline]) rouge_scores = get_scores(metric, pred_timeline, ground_truth, evaluator) date_scores = evaluate_dates(pred_timeline, ground_truth) wm_scores = get_wordmover_score(pred_timeline, ground_truth, word_mover_stop_words, device='cpu') dd_scores = date_dist_scores(pred_timeline, ground_truth) print('sys-len:', sys_len, 'gold-len:', l, 'gold-k:', k) print('Alignment-based ROUGE:') pprint(rouge_scores) print('Date selection:') pprint(date_scores) pprint(dd_scores) print('WordMover scores:') pprint(wm_scores) print('-' * 100) results.append((rouge_scores, date_scores, wm_scores, dd_scores, pred_timeline_.to_dict())) print("Running average:") print(get_average_results(results)) print() avg_results = get_average_results(results) print('Average results:') pprint(avg_results) output = { 'average': avg_results, 'results': results, } utils.write_json(output, result_path)
def evaluate(tls_model, dataset, result_path, trunc_timelines=False, time_span_extension=0): results = [] metric = 'align_date_content_costs_many_to_one' evaluator = rouge.TimelineRougeEvaluator(measures=["rouge_1", "rouge_2"]) n_topics = len(dataset.collections) ave_cluster = 0 for i, collection in enumerate(dataset.collections): ref_timelines = [ TilseTimeline(tl.date_to_summaries) for tl in collection.timelines ] topic = collection.name n_ref = len(ref_timelines) # only for entity if trunc_timelines: ref_timelines = data.truncate_timelines(ref_timelines, collection) for j, ref_timeline in enumerate(ref_timelines): print( f'topic {i+1}/{n_topics}: {topic}, ref timeline {j+1}/{n_ref}') tls_model.load(ignored_topics=[collection.name]) ref_dates = sorted(ref_timeline.dates_to_summaries) #print("data to summaries = {}".format(ref_dates)) start, end = data.get_input_time_span(ref_dates, time_span_extension) collection.start = start collection.end = end print("name = {} start = {} end = {}".format(topic, start, end)) #utils.plot_date_stats(collection, ref_dates) l = len(ref_dates) k = data.get_average_summary_length(ref_timeline) pred_timeline_, n_clusters = tls_model.predict( collection, max_dates=l, max_summary_sents=k, ref_tl=ref_timeline # only oracles need this ) ave_cluster = ave_cluster + n_clusters # print('*** PREDICTED ***') # utils.print_tl(pred_timeline_) print('timeline done') pred_timeline = TilseTimeline(pred_timeline_.date_to_summaries) sys_len = len(pred_timeline.get_dates()) ground_truth = TilseGroundTruth([ref_timeline]) rouge_scores = get_scores(metric, pred_timeline, ground_truth, evaluator) date_scores = evaluate_dates(pred_timeline, ground_truth) print('sys-len:', sys_len, 'gold-len:', l, 'gold-k:', k) print('Alignment-based ROUGE:') pprint(rouge_scores) print('Date selection:') pprint(date_scores) print('-' * 100) results.append( (rouge_scores, date_scores, pred_timeline_.to_dict())) avg_results = get_average_results(results) print('Average results:') pprint(avg_results) output = { 'average_clusters': ave_cluster / len(dataset.collections), 'average': avg_results, 'results': results, } utils.write_json(output, result_path)