class Rouge:
DEFAULT_METRICS = ["rouge-1", "rouge-2", "rouge-l"]
AVAILABLE_METRICS = {
"rouge-1": lambda hyp, ref: rouge_score.rouge_n(hyp, ref, 1),
"rouge-2": lambda hyp, ref: rouge_score.rouge_n(hyp, ref, 2),
"rouge-l":
lambda hyp, ref: rouge_score.rouge_l_summary_level(hyp, ref),
}
DEFAULT_STATS = ["f", "p", "r"]
AVAILABLE_STATS = ["f", "p", "r"]
def __init__(self, metrics=None, stats=None):
self.metrics = metrics if metrics is not None \
else Rouge.DEFAULT_METRICS
self.stats = stats if stats is not None \
else Rouge.DEFAULT_STATS
for m in self.metrics:
if m not in Rouge.AVAILABLE_METRICS:
raise ValueError("Unknown metric '%s'" % m)
for s in self.stats:
if s not in Rouge.AVAILABLE_STATS:
raise ValueError("Unknown stat '%s'" % s)
def get_scores(self, hyps, refs, avg=False, ignore_empty=False):
if isinstance(hyps, six.string_types):
hyps, refs = [hyps], [refs]
if ignore_empty:
# Filter out hyps of 0 length
hyps_and_refs = zip(hyps, refs)
hyps_and_refs = [_ for _ in hyps_and_refs if len(_[0]) > 0]
hyps, refs = zip(*hyps_and_refs)
assert (type(hyps) == type(refs))
assert (len(hyps) == len(refs))
if not avg:
return self._get_scores(hyps, refs)
return self._get_avg_scores(hyps, refs)
def _get_scores(self, hyps, refs):
scores = []
for hyp, ref in zip(hyps, refs):
sen_score = {}
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref)
sen_score[m] = {s: sc[s] for s in self.stats}
scores.append(sen_score)
return scores
def _get_avg_scores(self, hyps, refs):
scores = {m: {s: 0 for s in self.stats} for m in self.metrics}
count = 0
for (hyp, ref) in zip(hyps, refs):
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref)
scores[m] = {s: scores[m][s] + sc[s] for s in sc}
count += 1
scores = {
m: {s: scores[m][s] / count
for s in scores[m]}
for m in scores
}
return scores
sum_rouge_2_p = 0
sum_rouge_2_r = 0
sum_rouge_2_f = 0
sum_rouge_l_p = 0
sum_rouge_l_r = 0
sum_rouge_l_f = 0
for idx in range(size_doc):
score_1 = rouge_n(summarizer.lsa_evaluated_sentences[idx],
summarizer.label_sentences[idx],
n=1)
score_2 = rouge_n(summarizer.lsa_evaluated_sentences[idx],
summarizer.label_sentences[idx])
score_l = rouge_l_summary_level(
summarizer.lsa_evaluated_sentences[idx],
summarizer.label_sentences[idx])
# print(score_1)
# print(score_2)
# print(score_l)
sum_rouge_1_p += score_1['p']
sum_rouge_1_r += score_1['r']
sum_rouge_1_f += score_1['f']
sum_rouge_2_p += score_2['p']
sum_rouge_2_r += score_2['r']
sum_rouge_2_f += score_2['f']
sum_rouge_l_p += score_l['p']
class Rouge:
DEFAULT_METRICS = ["rouge-1", "rouge-2", "rouge-l"]
AVAILABLE_METRICS = {
"rouge-1":
lambda hyp, ref, **k: rouge_score.rouge_n(hyp, ref, 1, **k),
"rouge-2":
lambda hyp, ref, **k: rouge_score.rouge_n(hyp, ref, 2, **k),
"rouge-l":
lambda hyp, ref, **k: rouge_score.rouge_l_summary_level(hyp, ref, **k),
}
DEFAULT_STATS = ["f", "p", "r"]
AVAILABLE_STATS = ["f", "p", "r"]
def __init__(self,
metrics=None,
stats=None,
return_lengths=False,
raw_results=False,
exclusive=False):
self.return_lengths = return_lengths
self.raw_results = raw_results
self.exclusive = exclusive
if metrics is not None:
self.metrics = [m.lower() for m in metrics]
for m in self.metrics:
if m not in Rouge.AVAILABLE_METRICS:
raise ValueError("Unknown metric '%s'" % m)
else:
self.metrics = Rouge.DEFAULT_METRICS
if self.raw_results:
self.stats = ["hyp", "ref", "overlap"]
else:
if stats is not None:
self.stats = [s.lower() for s in stats]
for s in self.stats:
if s not in Rouge.AVAILABLE_STATS:
raise ValueError("Unknown stat '%s'" % s)
else:
self.stats = Rouge.DEFAULT_STATS
def get_scores(self, hyps, refs, avg=False, ignore_empty=False):
# HERE
if isinstance(hyps, six.string_types):
hyps, refs = [hyps], [refs]
if ignore_empty:
# Filter out hyps of 0 length
hyps_and_refs = zip(hyps, refs)
hyps_and_refs = [
_ for _ in hyps_and_refs if len(_[0]) > 0 and len(_[1]) > 0
]
hyps, refs = zip(*hyps_and_refs)
assert (isinstance(hyps, type(refs)))
assert (len(hyps) == len(refs))
if not avg:
return self._get_scores(hyps, refs)
return self._get_avg_scores(hyps, refs)
def _get_scores(self, hyps, refs):
scores = []
for hyp, ref in zip(hyps, refs):
sen_score = {}
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp,
ref,
raw_results=self.raw_results,
exclusive=self.exclusive)
sen_score[m] = {s: sc[s] for s in self.stats}
if self.return_lengths:
lengths = {
"hyp": len(" ".join(hyp).split()),
"ref": len(" ".join(ref).split())
}
sen_score["lengths"] = lengths
scores.append(sen_score)
return scores
def _get_avg_scores(self, hyps, refs):
# FinRouge: MODIFICATIONS MADE HERE TO IMPLEMENT FinRouge
double = False
matches = 0
for word in [
'corp', 'corporation', 'company', 'stock', 'share', 'shares',
'stakeholder', 'financial', '$', 'economy', 'booming', 'bust',
'quarter', 'earnings', 'shareholder', 'appoints', 'appointed',
'reports'
]:
for sent in hyps:
if word in sent.split():
double = True
matches += 1
scores = {m: {s: 0 for s in self.stats} for m in self.metrics}
if self.return_lengths:
scores["lengths"] = {"hyp": 0, "ref": 0}
count = 0
for (hyp, ref) in zip(hyps, refs):
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref, exclusive=self.exclusive)
scores[m] = {s: scores[m][s] + sc[s] for s in self.stats}
if self.return_lengths:
scores["lengths"]["hyp"] += len(" ".join(hyp).split())
scores["lengths"]["ref"] += len(" ".join(ref).split())
count += 1
if not double:
avg_scores = {
m: {s: scores[m][s] / count
for s in self.stats}
for m in self.metrics
}
elif double:
avg_scores = {
m: {s: (scores[m][s] + matches) / count
for s in self.stats}
for m in self.metrics
}
if self.return_lengths and not double:
avg_scores["lengths"] = {
k: scores["lengths"][k] / count
for k in ["hyp", "ref"]
}
elif self.return_lengths and double:
avg_scores["lengths"] = {
k: (scores["lengths"][k] + matches) / count
for k in ["hyp", "ref"]
}
return avg_scores
class Rouge:
DEFAULT_METRICS = ["rouge-1", "rouge-2", "rouge-l"]
AVAILABLE_METRICS = {
"rouge-1":
lambda hyp, ref, **k: rouge_score.rouge_n(hyp, ref, 1, **k),
"rouge-2":
lambda hyp, ref, **k: rouge_score.rouge_n(hyp, ref, 2, **k),
"rouge-l":
lambda hyp, ref, **k: rouge_score.rouge_l_summary_level(hyp, ref, **k),
}
DEFAULT_STATS = ["f", "p", "r"]
AVAILABLE_STATS = ["f", "p", "r"]
def __init__(self,
metrics=None,
stats=None,
return_lengths=False,
raw_results=False,
exclusive=False):
self.return_lengths = return_lengths
self.raw_results = raw_results
self.exclusive = exclusive
if metrics is not None:
self.metrics = [m.lower() for m in metrics]
for m in self.metrics:
if m not in Rouge.AVAILABLE_METRICS:
raise ValueError("Unknown metric '%s'" % m)
else:
self.metrics = Rouge.DEFAULT_METRICS
if self.raw_results:
self.stats = ["hyp", "ref", "overlap"]
else:
if stats is not None:
self.stats = [s.lower() for s in stats]
for s in self.stats:
if s not in Rouge.AVAILABLE_STATS:
raise ValueError("Unknown stat '%s'" % s)
else:
self.stats = Rouge.DEFAULT_STATS
def get_scores(self, hyps, refs, avg=False, ignore_empty=False):
if isinstance(hyps, six.string_types):
hyps, refs = [hyps], [refs]
if ignore_empty:
# Filter out hyps of 0 length
hyps_and_refs = zip(hyps, refs)
hyps_and_refs = [
_ for _ in hyps_and_refs if len(_[0]) > 0 and len(_[1]) > 0
]
hyps, refs = zip(*hyps_and_refs)
assert (isinstance(hyps, type(refs)))
assert (len(hyps) == len(refs))
if not avg:
return self._get_scores(hyps, refs)
return self._get_avg_scores(hyps, refs)
def _get_scores(self, hyps, refs):
scores = []
for hyp, ref in zip(hyps, refs):
sen_score = {}
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp,
ref,
raw_results=self.raw_results,
exclusive=self.exclusive)
sen_score[m] = {s: sc[s] for s in self.stats}
if self.return_lengths:
lengths = {
"hyp": len(" ".join(hyp).split()),
"ref": len(" ".join(ref).split())
}
sen_score["lengths"] = lengths
scores.append(sen_score)
return scores
def _get_avg_scores(self, hyps, refs):
scores = {m: {s: 0 for s in self.stats} for m in self.metrics}
if self.return_lengths:
scores["lengths"] = {"hyp": 0, "ref": 0}
count = 0
for (hyp, ref) in zip(hyps, refs):
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref, exclusive=self.exclusive)
scores[m] = {s: scores[m][s] + sc[s] for s in self.stats}
if self.return_lengths:
scores["lengths"]["hyp"] += len(" ".join(hyp).split())
scores["lengths"]["ref"] += len(" ".join(ref).split())
count += 1
avg_scores = {
m: {s: scores[m][s] / count
for s in self.stats}
for m in self.metrics
}
if self.return_lengths:
avg_scores["lengths"] = {
k: scores["lengths"][k] / count
for k in ["hyp", "ref"]
}
return avg_scores
class Rouge:
DEFAULT_METRICS = ["rouge-1", "rouge-2", "rouge-l"]
AVAILABLE_METRICS = {
"rouge-1": lambda hyp, ref: rouge_score.rouge_n(hyp, ref, 1),
"rouge-2": lambda hyp, ref: rouge_score.rouge_n(hyp, ref, 2),
"rouge-l":
lambda hyp, ref: rouge_score.rouge_l_summary_level(hyp, ref),
}
DEFAULT_STATS = ["f", "p", "r"]
AVAILABLE_STATS = ["f", "p", "r"]
def __init__(self,
metrics=None,
stats=None,
stem=False,
remove_stop=False):
self.metrics = metrics if metrics is not None \
else Rouge.DEFAULT_METRICS
self.stats = stats if stats is not None \
else Rouge.DEFAULT_STATS
self.rouge_calc = RougeCalculator(stopwords=remove_stop,
stemming=stem,
lang="en")
for m in self.metrics:
if m not in Rouge.AVAILABLE_METRICS:
raise ValueError("Unknown metric '%s'" % m)
for s in self.stats:
if s not in Rouge.AVAILABLE_STATS:
raise ValueError("Unknown stat '%s'" % s)
def get_scores(self, hyps, refs, avg=False):
if isinstance(hyps, six.string_types):
hyps, refs = [hyps], [refs]
assert (type(hyps) == type(refs))
assert (len(hyps) == len(refs))
if not avg:
return self._get_scores(hyps, refs)
return self._get_avg_scores(hyps, refs)
def _get_scores(self, hyps, refs):
scores = []
for hyp, ref in zip(hyps, refs):
sen_score = {}
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref)
sen_score[m] = {s: sc[s] for s in self.stats}
scores.append(sen_score)
return scores
def _get_avg_scores(self, hyps, refs):
scores = {m: {s: 0 for s in self.stats} for m in self.metrics}
count = 0
for (hyp, ref) in zip(hyps, refs):
hyp = [
" ".join(_.split()).strip() for _ in hyp.split(" . ")
if len(_) > 0
]
ref = [
" ".join(_.split()).strip() for _ in ref.split(" . ")
if len(_) > 0
]
hyp = [" ".join(self.rouge_calc.tokenize(h, False)) for h in hyp]
ref = [" ".join(self.rouge_calc.tokenize(r, True)) for r in ref]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref)
scores[m] = {s: scores[m][s] + sc[s] for s in sc}
count += 1
scores = {
m: {s: scores[m][s] / count
for s in scores[m]}
for m in scores
}
return scores
class Rouge:
DEFAULT_METRICS = ["rouge-1", "rouge-2", "rouge-l"]
AVAILABLE_METRICS = {
"rouge-1": lambda hyp, ref: rouge_score.rouge_n(hyp, ref, 1),
"rouge-2": lambda hyp, ref: rouge_score.rouge_n(hyp, ref, 2),
"rouge-l":
lambda hyp, ref: rouge_score.rouge_l_summary_level(hyp, ref),
}
DEFAULT_STATS = ["f", "p", "r"]
AVAILABLE_STATS = {"f": 0, "p": 1, "r": 2}
def __init__(self, metrics=None, stats=None):
self.metrics = metrics if metrics is not None else Rouge.DEFAULT_METRICS
self.stats = stats if stats is not None else Rouge.DEFAULT_STATS
for m in self.metrics:
if m not in Rouge.AVAILABLE_METRICS:
raise ValueError("Unknown metric '%s'" % m)
for s in self.stats:
if s not in Rouge.AVAILABLE_STATS:
raise ValueError("Unknown stat '%s'" % s)
def get_scores(self, hyps, refs, avg=False):
if type(hyps) == str:
hyps, refs = [hyps], [refs]
assert (type(hyps) == type(refs))
assert (len(hyps) == len(refs))
if not avg:
return self._get_scores(hyps, refs)
return self._get_avg_scores(hyps, refs)
def _get_scores(self, hyps, refs):
scores = []
for hyp, ref in zip(hyps, refs):
sen_score = {}
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref)
sen_score[m] = {
s: sc[Rouge.AVAILABLE_STATS[s]]
for s in self.stats
}
scores.append(sen_score)
return scores
def _get_avg_scores(self, hyps, refs):
scores = {}
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = [fn(hyp, ref) for hyp, ref in zip(hyps, refs)]
sc = [[sen_sc[Rouge.AVAILABLE_STATS[s]] for s in self.stats]
for sen_sc in sc]
scores[m] = {
s: st
for s, st in zip(self.stats, tuple(map(np.mean, zip(*sc))))
}
return scores
class Rouge:
DEFAULT_METRICS = ["rouge-1", "rouge-2", "rouge-l"]
AVAILABLE_METRICS = {
"rouge-1": lambda hyp, ref, **k: rouge_score.rouge_n(hyp, ref, 1, **k),
"rouge-2": lambda hyp, ref, **k: rouge_score.rouge_n(hyp, ref, 2, **k),
"rouge-l": lambda hyp, ref, **k:
rouge_score.rouge_l_summary_level(hyp, ref, **k),
}
DEFAULT_STATS = ["f", "p", "r"]
AVAILABLE_STATS = ["f", "p", "r"]
def __init__(self, metrics=None, stats=None, return_lengths=False,
raw_results=False, exclusive=False):
self.return_lengths = return_lengths
self.raw_results = raw_results
self.exclusive = exclusive
if metrics is not None:
self.metrics = [m.lower() for m in metrics]
for m in self.metrics:
if m not in Rouge.AVAILABLE_METRICS:
raise ValueError("Unknown metric '%s'" % m)
else:
self.metrics = Rouge.DEFAULT_METRICS
if self.raw_results:
self.stats = ["hyp", "ref", "overlap"]
else:
if stats is not None:
self.stats = [s.lower() for s in stats]
for s in self.stats:
if s not in Rouge.AVAILABLE_STATS:
raise ValueError("Unknown stat '%s'" % s)
else:
self.stats = Rouge.DEFAULT_STATS
def get_scores(self, hyps, refs, avg=False, ignore_empty=False, scoring=None):
if isinstance(hyps, six.string_types):
hyps, refs = [hyps], [refs]
if ignore_empty:
# Filter out hyps of 0 length
hyps_and_refs = zip(hyps, refs)
hyps_and_refs = [_ for _ in hyps_and_refs
if len(_[0]) > 0
and len(_[1]) > 0]
hyps, refs = zip(*hyps_and_refs)
assert(isinstance(hyps, type(refs)))
assert(len(hyps) == len(refs))
if not avg:
return self._get_scores(hyps, refs)
# ave のほうへ一旦追加
return self._get_avg_scores(hyps, refs, scoring)
def _get_scores(self, hyps, refs):
scores = []
for hyp, ref in zip(hyps, refs):
sen_score = {}
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(
hyp,
ref,
raw_results=self.raw_results,
exclusive=self.exclusive)
sen_score[m] = {s: sc[s] for s in self.stats}
if self.return_lengths:
lengths = {
"hyp": len(" ".join(hyp).split()),
"ref": len(" ".join(ref).split())
}
sen_score["lengths"] = lengths
scores.append(sen_score)
return scores
def _get_avg_scores(self, hyps, refs, scoring=None):
scores = {m: {s: 0 for s in self.stats} for m in self.metrics}
if self.return_lengths:
scores["lengths"] = {"hyp": 0, "ref": 0}
count = 0
for (hyp, ref) in zip(hyps, refs):
if scoring in ["A", "B"]:
partial_scores = {m: {s: [] for s in self.stats} for m in self.metrics}
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
partial_ref_length = []
# 正解データが複数<TAB>で分割されている想定
for r in ref.split("\t"):
partial_ref = [" ".join(_.split()) for _ in r.split(".") if len(_) > 0]
partial_ref_length.append(len(" ".join(partial_ref).split()))
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, partial_ref, exclusive=self.exclusive)
for s in self.stats:
partial_scores[m][s].append(sc[s])
# スコアのマージ(ave:A)
if scoring == "A":
for m in self.metrics:
for s in self.stats:
scores[m][s] += sum(partial_scores[m][s]) / len(partial_scores[m][s])
if self.return_lengths:
scores["lengths"]["hyp"] += len(" ".join(hyp).split())
scores["lengths"]["ref"] += sum(partial_ref_length) / len(partial_ref_length)
else:
# スコアのマージ(best:B)
# ベストを選択する場合にソートkey が必要なので一旦
# `rouge-l` の `f` が一番高いものとする
max_val = max(partial_scores["rouge-l"]["f"])
max_idx = partial_scores["rouge-l"]["f"].index(max_val)
for m in self.metrics:
for s in self.stats:
scores[m][s] += partial_scores[m][s][max_idx]
if self.return_lengths:
scores["lengths"]["hyp"] += len(" ".join(hyp).split())
scores["lengths"]["ref"] += partial_ref_length[max_idx]
else:
# scoring 指定なしの場合に
# ref が<TAB>で複数ある場合は先頭を利用する
ref = ref.split("\t")[0]
hyp = [" ".join(_.split()) for _ in hyp.split(".") if len(_) > 0]
ref = [" ".join(_.split()) for _ in ref.split(".") if len(_) > 0]
for m in self.metrics:
fn = Rouge.AVAILABLE_METRICS[m]
sc = fn(hyp, ref, exclusive=self.exclusive)
scores[m] = {s: scores[m][s] + sc[s] for s in self.stats}
if self.return_lengths:
scores["lengths"]["hyp"] += len(" ".join(hyp).split())
scores["lengths"]["ref"] += len(" ".join(ref).split())
count += 1
avg_scores = {
m: {s: scores[m][s] / count for s in self.stats}
for m in self.metrics
}
if self.return_lengths:
avg_scores["lengths"] = {
k: scores["lengths"][k] / count
for k in ["hyp", "ref"]
}
return avg_scores