def compute_score(gts, val_caps, train_imgids, val_imgids, i, j):
res = {}
for imgid in train_imgids:
res[imgid] = [val_caps[val_imgids[i]][j]]
scorer = Rouge()
score, scores = scorer.compute_score(gts, res, train_imgids)
#print(score)
#print(len(scores))
return np.array(scores)
class CaptionEvaluater(object):
def __init__(self, ):
self.blue_scorer = Bleu(4)
self.rouge_scorer = Rouge()
self.cider_scorer = Cider()
self.truth = None
remove = string.punctuation + "、。，．"
self.remove_pattern = r"[{}]".format(remove) # create the pattern
def remove_punctuation(self, line):
#I am not sure how unicode works in python, so just in case.
line = line.replace(u"<unk>", "")
line = line.replace("<unk>", "")
line = line.replace(u"。", "")
line = line.replace('\u3002', "")
return re.sub(self.remove_pattern, "", line)
def trnasform_utf8(self, line):
# return u' '.join(line).encode('utf-8').strip()
return line
def set_ground_truth(self, ground_truth):
'''
ground_truth should be a python dictonary whose shape is;
{"image_identifier": ["a caption", "a similar caption", ...], ...}
"image_identifier" can be either string or number.
'''
for img in ground_truth:
# ground_truth[img]=map(self.trnasform_utf8,ground_truth[img])
ground_truth[img] = map(self.remove_punctuation, ground_truth[img])
self.truth = ground_truth
def evaluate(self, predicetd_captions):
'''
predicetd_captions should be a python dictonary whose shape is;
{"image_identifier": ["the prediced caption"], ...}
"image_identifier" need to be same as used in ground truth.
make sure the number of caption is only one, even though it uses python list.
'''
for img in predicetd_captions:
# predicetd_captions[img]=map(self.trnasform_utf8,predicetd_captions[img])
predicetd_captions[img] = map(self.remove_punctuation,
predicetd_captions[img])
results = {}
for i, score in enumerate(self.get_bleu(predicetd_captions)[0]):
results["bleu-%d" % i] = score
results["rouge"] = self.get_rouge(predicetd_captions)[0]
results["cider"] = self.get_cider(predicetd_captions)[0]
return results
def get_bleu(self, predicetd_captions):
score, scores = self.blue_scorer.compute_score(self.truth,
predicetd_captions)
#output is a python list [bleu-1,bleu-2,bleu-3,bleu-4]
return score, scores
def get_rouge(self, predicetd_captions):
score, scores = self.rouge_scorer.compute_score(
self.truth, predicetd_captions)
return score, scores
def get_cider(self, predicetd_captions):
score, scores = self.cider_scorer.compute_score(
self.truth, predicetd_captions)
return score, scores
