def compute_score(gts, val_caps, train_imgids, val_imgids, i, j): res = {} for imgid in train_imgids: res[imgid] = [val_caps[val_imgids[i]][j]] scorer = Rouge() score, scores = scorer.compute_score(gts, res, train_imgids) #print(score) #print(len(scores)) return np.array(scores)
class CaptionEvaluater(object): def __init__(self, ): self.blue_scorer = Bleu(4) self.rouge_scorer = Rouge() self.cider_scorer = Cider() self.truth = None remove = string.punctuation + "、。,." self.remove_pattern = r"[{}]".format(remove) # create the pattern def remove_punctuation(self, line): #I am not sure how unicode works in python, so just in case. line = line.replace(u"<unk>", "") line = line.replace("<unk>", "") line = line.replace(u"。", "") line = line.replace('\u3002', "") return re.sub(self.remove_pattern, "", line) def trnasform_utf8(self, line): # return u' '.join(line).encode('utf-8').strip() return line def set_ground_truth(self, ground_truth): ''' ground_truth should be a python dictonary whose shape is; {"image_identifier": ["a caption", "a similar caption", ...], ...} "image_identifier" can be either string or number. ''' for img in ground_truth: # ground_truth[img]=map(self.trnasform_utf8,ground_truth[img]) ground_truth[img] = map(self.remove_punctuation, ground_truth[img]) self.truth = ground_truth def evaluate(self, predicetd_captions): ''' predicetd_captions should be a python dictonary whose shape is; {"image_identifier": ["the prediced caption"], ...} "image_identifier" need to be same as used in ground truth. make sure the number of caption is only one, even though it uses python list. ''' for img in predicetd_captions: # predicetd_captions[img]=map(self.trnasform_utf8,predicetd_captions[img]) predicetd_captions[img] = map(self.remove_punctuation, predicetd_captions[img]) results = {} for i, score in enumerate(self.get_bleu(predicetd_captions)[0]): results["bleu-%d" % i] = score results["rouge"] = self.get_rouge(predicetd_captions)[0] results["cider"] = self.get_cider(predicetd_captions)[0] return results def get_bleu(self, predicetd_captions): score, scores = self.blue_scorer.compute_score(self.truth, predicetd_captions) #output is a python list [bleu-1,bleu-2,bleu-3,bleu-4] return score, scores def get_rouge(self, predicetd_captions): score, scores = self.rouge_scorer.compute_score( self.truth, predicetd_captions) return score, scores def get_cider(self, predicetd_captions): score, scores = self.cider_scorer.compute_score( self.truth, predicetd_captions) return score, scores