def _run_for_one(self, t, corpora, topic_to_preprocessed, reference_timelines):
logging.info(t)
corpus = corpora[t]
# train
params = self.train(corpora, topic_to_preprocessed, reference_timelines, t)
results_rouge = {}
results_date_selection = {}
returned_timelines = {}
# predict
for i, timeline in enumerate(reference_timelines[t].timelines):
timeline_properties = self.get_timeline_properties(timeline)
groundtruth = timelines.GroundTruth([timeline])
pred = self.predict(corpus, topic_to_preprocessed[t], timeline_properties, params)
# evaluate
print("topic_to_preprocessed", topic_to_preprocessed)
print("timeline_properties", timeline_properties)
print("groundtruth", groundtruth)
print("pred", pred)
results_rouge[t + "_" + str(i)] = self.rouge.evaluate_all(pred, groundtruth)
results_date_selection[t + "_" + str(i)] = dates.evaluate_dates(pred, groundtruth)
returned_timelines[t + "_" + str(i)] = pred
return results_rouge, results_date_selection, returned_timelines
def get_timeline_properties(self, timeline):
"""
Computes timeline properties for a given timeline.
Params:
timeline (tilse.data.timelines.Timeline): A timeline.
Returns:
A tilse.models.timeline_properties.TimelineProperties object,
with:
* `daily_summary_length` set to the output of `self.asses_length`,
* `num_dates` set to the length of the input timeline in days,
* `num_sentences` set to the length of the input timeline in sentences,
* `start` and `end` set to the first and last days in the input timeline.
"""
groundtruth = timelines.GroundTruth([timeline])
groundtruth_dates = sorted(list(timeline.get_dates()))
desired_timeline_length = len(groundtruth_dates)
summary_length = self.summary_length_assessor(groundtruth)
timeline_properties = TimelineProperties(
summary_length,
desired_timeline_length,
timeline.get_number_of_sentences(),
groundtruth_dates[0],
groundtruth_dates[-1]
)
return timeline_properties
def setUp(self):
self.evaluator = rouge.TimelineRougeEvaluator()
self.ground_truth = timelines.GroundTruth(
[
timelines.Timeline(
{
datetime.date(2010, 1, 1): ["timeline summarization ."],
datetime.date(2010, 1, 2): ["timeline summarization is awesome .",
"coreference resolution is , too ."],
datetime.date(2010, 1, 4): ["alignments are really nice"]
}
),
timelines.Timeline(
{
datetime.date(2010, 1, 2): ["metrics are complicated ."]
}
),
]
)
self.output_same_number_of_dates = timelines.Timeline(
{
datetime.date(2010, 1, 2): ["timeline summarization ."],
datetime.date(2010, 1, 3): ["doing some metric checks ."],
datetime.date(2010, 1, 5): ["checks for alignments ."],
}
)
self.output_same_number_of_dates_scores_higher_with_date_content_costs = timelines.Timeline(
{
datetime.date(2010, 1, 2): ["timeline summarization ."],
datetime.date(2010, 1, 3): ["alignments are really nice"],
datetime.date(2010, 1, 5): ["timeline summarization ."],
}
)
self.output_less_dates = timelines.Timeline(
{
datetime.date(2010, 1, 2): ["timeline summarization ."],
datetime.date(2010, 1, 3): ["doing some metric checks ."],
}
)
self.output_more_dates = timelines.Timeline(
{
datetime.date(2010, 1, 2): ["timeline summarization ."],
datetime.date(2010, 1, 3): ["doing some metric checks ."],
datetime.date(2010, 1, 5): ["checks for alignments ."],
datetime.date(2010, 1, 6): ["asdf"],
}
)
def _run_test(self, test_to_run, ref_tls, names):
results = {}
for name, ref_tl in zip(names, ref_tls):
tl = timelines.Timeline({})
for date, sents in ref_tl.dates_to_summaries.items():
tl.dates_to_summaries[date] = [sent for sent in sents]
test_to_run(tl)
results[name] = self.rouge.evaluate_all(
tl, timelines.GroundTruth([ref_tl]))
return scores.Scores(results)
# filter dataset by keywords
# if keyword_mapping is not None and keyword_mapping[topic] is not None:
# news_corpora[topic] = news_corpora[topic].filter_by_keywords_contained(keyword_mapping[topic])
# read groundtruth timelines
for filename in sorted(
list(os.listdir(raw_directory + "/" + topic + "/timelines/"))):
full_path = raw_directory + "/" + topic + "/timelines/" + filename
temp_reference_timelines[topic].append(
timelines.Timeline.from_file(
codecs.open(full_path, "r", "utf-8", "replace")))
for topic in temp_reference_timelines:
reference_timelines[topic] = timelines.GroundTruth(
temp_reference_timelines[topic])
evaluator = rouge.TimelineRougeEvaluator(measures=["rouge_1", "rouge_2"],
beta=1)
model_avg_scores = {}
for _config in configs:
config_file = modelbase_dir + _config
print(_config)
print(config_file)
config = json.load(open(config_file))
config["rouge_computation"] = "reimpl"
logging.info(config)