def extract_oracle(extract_list, summ_list): """Choose sentence with maximum average rouge.""" # Choose sentence with maximum average rouge. score_accum = [] for e in extract_list: e_trunc = p2s_eval.get_summary_truncated( p2s_eval.get_summary_first_sentence(e), p2s_eval.TRUNC_LEN ) # get_summary_first_sentence may not be necessary accum_rouge_1r_trunc = 0 for s in summ_list: s_e_trunc_score = my_rouge_scorer.score(s, e_trunc) # for computing accumulative rouge accum_rouge_1r_trunc += s_e_trunc_score['rouge1'].recall score_accum.append(accum_rouge_1r_trunc) e_id_o = np.argmax(score_accum) e_o = extract_list[e_id_o] # Compute average rouge for the oracle sentence agg = scoring.BootstrapAggregator() e_o_trunc = p2s_eval.get_summary_truncated( p2s_eval.get_summary_first_sentence(e_o), p2s_eval.TRUNC_LEN) # get_summary_first_sentence may not be necessary for s in summ_list: e_o_trunc_score = my_rouge_scorer.score(s, e_o_trunc) agg.add_scores(e_o_trunc_score) agg_o = agg.aggregate() score_o = { rouge_type: agg_o[rouge_type].mid for rouge_type in agg_o # mid=mean } nwords_o = p2s_eval.count_words(e_o) return (score_o, nwords_o, e_o)
def test_get_summary_truncated(self): s = 'this is a summary and a summary.' self.assertEqual('this is a summary', p2s_eval.get_summary_truncated(s, 4)) s = 'this is a summary!' self.assertEqual('this is a summary!', p2s_eval.get_summary_truncated(s, 4))
def extract_ave(e, summ_list): """Average rouge between ith sentence and human summaries.""" agg = scoring.BootstrapAggregator() e_trunc = p2s_eval.get_summary_truncated( p2s_eval.get_summary_first_sentence(e), p2s_eval.TRUNC_LEN) # get_summary_first_sentence may not be necessary for s in summ_list: s_e_trunc_score = my_rouge_scorer.score(s, e_trunc) agg.add_scores(s_e_trunc_score) agg_ave = agg.aggregate() score_ave = { rouge_type: agg_ave[rouge_type].mid for rouge_type in agg_ave # mid=mean } nwords_e = p2s_eval.count_words(e) return (score_ave, nwords_e)
def human_max(summ_list): """Maximum pairwise rouge between any two human summaries.""" score_max = None rouge_1r_trunc_max = 0 for s1_id, s1 in enumerate(summ_list): for s2_id, s2 in enumerate(summ_list): if s1_id >= s2_id: continue s2_trunc = p2s_eval.get_summary_truncated( p2s_eval.get_summary_first_sentence(s2), p2s_eval.TRUNC_LEN) s1_s2_trunc_score = my_rouge_scorer.score(s1, s2_trunc) if s1_s2_trunc_score['rouge1'].recall >= rouge_1r_trunc_max: score_max = s1_s2_trunc_score rouge_1r_trunc_max = s1_s2_trunc_score['rouge1'].recall nwords_max = np.max([p2s_eval.count_words(s) for s in summ_list]) return (score_max, nwords_max)
def human_ave(summ_list): """Average pairwise rouge between two human summaries.""" agg = scoring.BootstrapAggregator() for s1_id, s1 in enumerate(summ_list): for s2_id, s2 in enumerate(summ_list): if s1_id >= s2_id: # only compute for s1_id < s2_id continue s2_trunc = p2s_eval.get_summary_truncated( p2s_eval.get_summary_first_sentence(s2), p2s_eval.TRUNC_LEN) s1_s2_trunc_score = my_rouge_scorer.score(s1, s2_trunc) agg.add_scores(s1_s2_trunc_score) agg_ave = agg.aggregate() score_ave = { rouge_type: agg_ave[rouge_type].mid for rouge_type in agg_ave # mid=mean } nwords_ave = np.mean([p2s_eval.count_words(s) for s in summ_list]) return (score_ave, nwords_ave)