def compute_individual_metrics(ref, hyp, no_overlap=False, no_skipthoughts=False, no_glove=False):
assert isinstance(hyp, six.string_types)
if isinstance(ref, six.string_types):
ref = ref.split('||<|>||') # special delimiter for backward compatibility
ref = [a.strip() for a in ref]
refs = {0: ref}
ref_list = [ref]
hyps = {0: [hyp.strip()]}
hyp_list = [hyp]
ret_scores = {}
if not no_overlap:
scorers = [
(Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
(Meteor(), "METEOR"),
(Rouge(), "ROUGE_L"),
(Cider(), "CIDEr")
]
for scorer, method in scorers:
score, scores = scorer.compute_score(refs, hyps)
if isinstance(method, list):
for sc, scs, m in zip(score, scores, method):
ret_scores[m] = sc
else:
ret_scores[method] = score
if isinstance(scorer, Meteor):
scorer.close()
del scorers
if not no_skipthoughts:
from nlgeval.skipthoughts import skipthoughts
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
model = skipthoughts.load_model()
encoder = skipthoughts.Encoder(model)
vector_hyps = encoder.encode([h.strip() for h in hyp_list], verbose=False)
ref_list_T = np.array(ref_list).T.tolist()
vector_refs = map(lambda refl: encoder.encode([r.strip() for r in refl], verbose=False), ref_list_T)
cosine_similarity = list(map(lambda refv: cosine_similarity(refv, vector_hyps).diagonal(), vector_refs))
cosine_similarity = np.max(cosine_similarity, axis=0).mean()
ret_scores['SkipThoughtCS'] = cosine_similarity
if not no_glove:
from nlgeval.word2vec.evaluate import eval_emb_metrics
import numpy as np
glove_hyps = [h.strip() for h in hyp_list]
ref_list_T = np.array(ref_list).T.tolist()
glove_refs = map(lambda refl: [r.strip() for r in refl], ref_list_T)
scores = eval_emb_metrics(glove_hyps, glove_refs)
scores = scores.split('\n')
for score in scores:
name, value = score.split(':')
value = float(value.strip())
ret_scores[name] = value
return ret_scores
def load_skipthought_model(self):
from nlgeval.skipthoughts import skipthoughts
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
self.np = np
self.cosine_similarity = cosine_similarity
model = skipthoughts.load_model()
self.skipthought_encoder = skipthoughts.Encoder(model)
def compute_individual_metrics(ref, hyp, no_overlap=False, no_skipthoughts=False, no_glove=False):
assert isinstance(ref, str)
assert isinstance(hyp, str)
ref = ref.split('||<|>||') # special delimiter
ref = [a.strip() for a in ref]
refs = {0: ref}
ref_list = [ref]
hyps = {0: [hyp.strip()]}
hyp_list = [hyp]
ret_scores = {}
if not no_overlap:
scorers = [
(Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
(Meteor(), "METEOR"),
(Rouge(), "ROUGE_L"),
(Cider(), "CIDEr")
]
for scorer, method in scorers:
score, scores = scorer.compute_score(refs, hyps)
if isinstance(method, list):
for sc, scs, m in zip(score, scores, method):
ret_scores[m] = sc
else:
ret_scores[method] = score
if not no_skipthoughts:
from nlgeval.skipthoughts import skipthoughts
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
model = skipthoughts.load_model()
encoder = skipthoughts.Encoder(model)
vector_hyps = encoder.encode([h.strip() for h in hyp_list], verbose=False)
ref_list_T = np.array(ref_list).T.tolist()
vector_refs = map(lambda refl: encoder.encode([r.strip() for r in refl], verbose=False), ref_list_T)
cosine_similarity = map(lambda refv: cosine_similarity(refv, vector_hyps).diagonal(), vector_refs)
cosine_similarity = np.max(cosine_similarity, axis=0).mean()
ret_scores['SkipThoughtCS'] = cosine_similarity
if not no_glove:
from nlgeval.word2vec.evaluate import eval_emb_metrics
import numpy as np
glove_hyps = [h.strip() for h in hyp_list]
ref_list_T = np.array(ref_list).T.tolist()
glove_refs = map(lambda refl: [r.strip() for r in refl], ref_list_T)
scores = eval_emb_metrics(glove_hyps, glove_refs)
scores = scores.split('\n')
for score in scores:
name, value = score.split(':')
value = float(value.strip())
ret_scores[name] = value
return ret_scores
def compute_metrics(hypothesis,
references,
no_overlap=False,
no_skipthoughts=False,
no_glove=False):
with open(hypothesis, 'r') as f:
hyp_list = f.readlines()
ref_list = []
for iidx, reference in enumerate(references):
with open(reference, 'r') as f:
ref_list.append(f.readlines())
ref_list = [list(map(_strip, refs)) for refs in zip(*ref_list)]
refs = {idx: strippedlines for (idx, strippedlines) in enumerate(ref_list)}
hyps = {idx: [lines.strip()] for (idx, lines) in enumerate(hyp_list)}
assert len(refs) == len(hyps)
ret_scores = {}
if not no_overlap:
scorers = [(Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
(Meteor(), "METEOR"), (Rouge(), "ROUGE_L"),
(Cider(), "CIDEr")]
for scorer, method in scorers:
score, scores = scorer.compute_score(refs, hyps)
if isinstance(method, list):
for sc, scs, m in zip(score, scores, method):
print("%s: %0.6f" % (m, sc))
ret_scores[m] = sc
else:
print("%s: %0.6f" % (method, score))
ret_scores[method] = score
if isinstance(scorer, Meteor):
scorer.close()
del scorers
if not no_skipthoughts:
from nlgeval.skipthoughts import skipthoughts
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
model = skipthoughts.load_model()
encoder = skipthoughts.Encoder(model)
vector_hyps = encoder.encode([h.strip() for h in hyp_list],
verbose=False)
ref_list_T = np.array(ref_list).T.tolist()
vector_refs = map(
lambda refl: encoder.encode([r.strip() for r in refl],
verbose=False), ref_list_T)
cosine_similarity = list(
map(lambda refv: cosine_similarity(refv, vector_hyps).diagonal(),
vector_refs))
cosine_similarity = np.max(cosine_similarity, axis=0).mean()
print("SkipThoughtsCosineSimilarity: %0.6f" % (cosine_similarity))
ret_scores['SkipThoughtCS'] = cosine_similarity
del model
if not no_glove:
from nlgeval.word2vec.evaluate import eval_emb_metrics
import numpy as np
glove_hyps = [h.strip() for h in hyp_list]
ref_list_T = np.array(ref_list).T.tolist()
glove_refs = map(lambda refl: [r.strip() for r in refl], ref_list_T)
scores = eval_emb_metrics(glove_hyps, glove_refs)
print(scores)
scores = scores.split('\n')
for score in scores:
name, value = score.split(':')
value = float(value.strip())
ret_scores[name] = value
return ret_scores
def compute_metrics(hypothesis, references, no_overlap=False, no_skipthoughts=False, no_glove=False):
with open(hypothesis, 'r') as f:
hyp_list = f.readlines()
ref_list = []
for iidx, reference in enumerate(references):
with open(reference, 'r') as f:
ref_list.append(f.readlines())
ref_list = [map(str.strip, refs) for refs in zip(*ref_list)]
refs = {idx: strippedlines for (idx, strippedlines) in enumerate(ref_list)}
hyps = {idx: [lines.strip()] for (idx, lines) in enumerate(hyp_list)}
assert len(refs) == len(hyps)
ret_scores = {}
if not no_overlap:
scorers = [
(Bleu(4), ["Bleu_1", "Bleu_2", "Bleu_3", "Bleu_4"]),
(Meteor(), "METEOR"),
(Rouge(), "ROUGE_L"),
(Cider(), "CIDEr")
]
for scorer, method in scorers:
score, scores = scorer.compute_score(refs, hyps)
if isinstance(method, list):
for sc, scs, m in zip(score, scores, method):
print("%s: %0.6f" % (m, sc))
ret_scores[m] = sc
else:
print("%s: %0.6f" % (method, score))
ret_scores[method] = score
if not no_skipthoughts:
from nlgeval.skipthoughts import skipthoughts
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
model = skipthoughts.load_model()
encoder = skipthoughts.Encoder(model)
vector_hyps = encoder.encode([h.strip() for h in hyp_list], verbose=False)
ref_list_T = np.array(ref_list).T.tolist()
vector_refs = map(lambda refl: encoder.encode([r.strip() for r in refl], verbose=False), ref_list_T)
cosine_similarity = map(lambda refv: cosine_similarity(refv, vector_hyps).diagonal(), vector_refs)
cosine_similarity = np.max(cosine_similarity, axis=0).mean()
print("SkipThoughtsCosineSimilairty: %0.6f" % (cosine_similarity))
ret_scores['SkipThoughtCS'] = cosine_similarity
if not no_glove:
from nlgeval.word2vec.evaluate import eval_emb_metrics
import numpy as np
glove_hyps = [h.strip() for h in hyp_list]
ref_list_T = np.array(ref_list).T.tolist()
glove_refs = map(lambda refl: [r.strip() for r in refl], ref_list_T)
scores = eval_emb_metrics(glove_hyps, glove_refs)
print(scores)
scores = scores.split('\n')
for score in scores:
name, value = score.split(':')
value = float(value.strip())
ret_scores[name] = value
return ret_scores
