def compute_score(self, gts, res): """ :param gts: :param res: :return: """ assert(gts.keys() == res.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: hypo = res[id] ref = gts[id] # Sanity check. assert(type(hypo) is list) assert(len(hypo) == 1) assert(type(ref) is list) assert(len(ref) > 0) bleu_scorer += (hypo[0], ref) score, scores = bleu_scorer.compute_score(option='closest', verbose=1) return score, scores
def compute_score(self, gts, res): assert (gts.keys() == res.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: hypo = res[id] ref = gts[id] # Sanity check. assert (type(hypo) is list) #print(len(hypo)) assert (len(hypo) == 1) assert (type(ref) is list) assert (len(ref) >= 1) bleu_scorer += (hypo[0], ref) #score, scores = bleu_scorer.compute_score(option='shortest') score, scores = bleu_scorer.compute_score(option='closest', verbose=1) #score, scores = bleu_scorer.compute_score(option='average', verbose=1) # return (bleu, bleu_info) return score, scores
class Bleu: def __init__(self, n=4): # default compute Blue score up to 4 self._n = n self._hypo_for_image = {} self.ref_for_image = {} self.bleu_scorer = BleuScorer(n=self._n) def compute_score(self, gts, res): self.bleu_scorer.clear() for res_id in res: hypo = res_id['caption'] ref = gts[res_id['image_id']] # Sanity check. assert(type(hypo) is list) assert(len(hypo) == 1) assert(type(ref) is list) assert(len(ref) >= 1) self.bleu_scorer += (hypo[0], ref) #score, scores = bleu_scorer.compute_score(option='shortest') score, scores = self.bleu_scorer.compute_score(option='closest', verbose=0) #score, scores = bleu_scorer.compute_score(option='average', verbose=1) # return (bleu, bleu_info) return score, scores def method(self): return "Bleu"
def compute_score(self, gts, res): """ :param gts: :param res: :return: """ assert (gts.keys() == res.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: hypo = res[id] ref = gts[id] # Sanity check. assert (type(hypo) is list) assert (len(hypo) == 1) assert (type(ref) is list) assert (len(ref) > 0) bleu_scorer += (hypo[0], ref) score, scores = bleu_scorer.compute_score(option='closest', verbose=1) return score, scores
def compute_score_for_consensus2(self, gts, subGts): assert(len(gts.keys()) == 1) assert(gts.keys() == subGts.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: ref = gts[id] subRef = subGts[id] sent_len = len(ref) # Sanity check. #assert(type(hypo) is list) #assert(len(hypo) == 1) assert(type(subRef) is list) assert(type(ref) is list) assert(len(ref) > 1) for r1 in ref: for r2 in subRef: bleu_scorer += (r1, [r2]) #score, scores = bleu_scorer.compute_score(option='shortest') score, scores = bleu_scorer.compute_score(option='closest', verbose=0) #score, scores = bleu_scorer.compute_score(option='average', verbose=1) # return (bleu, bleu_info) scores = np.sum(scores, axis = 0) scores = scores.reshape((len(ref), len(subRef))) scores = np.sum(scores, axis = 1) return score, scores
def compute_score(self, gts, res): assert(gts.keys() == res.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: hypo = res[id] ref = gts[id] # Sanity check. assert(type(hypo) is list) # assert(len(hypo) == 1) assert(type(ref) is list) assert(len(ref) >= 1) # Revise to incorporate paragraph-level situation for hypo_element in hypo: bleu_scorer += (hypo_element, ref) # bleu_scorer += (hypo[0], ref) #score, scores = bleu_scorer.compute_score(option='shortest') score, scores = bleu_scorer.compute_score(option='closest', verbose=1) #score, scores = bleu_scorer.compute_score(option='average', verbose=1) # return (bleu, bleu_info) return score, scores
def compute_score(self, gts, res): assert(gts.keys() == res.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: hypo = res[id] ref = gts[id] # Sanity check. assert(type(hypo) is list) assert(len(hypo) == 1) assert(type(ref) is list) assert(len(ref) >= 1) bleu_scorer += (hypo[0], ref) #score, scores = bleu_scorer.compute_score(option='shortest') score, scores = bleu_scorer.compute_score(option='closest', verbose=1) #score, scores = bleu_scorer.compute_score(option='average', verbose=1) # return (bleu, bleu_info) return score, scores
def compute_score_for_consensus2(self, gts, subGts): assert (len(gts.keys()) == 1) assert (gts.keys() == subGts.keys()) imgIds = gts.keys() bleu_scorer = BleuScorer(n=self._n) for id in imgIds: ref = gts[id] subRef = subGts[id] sent_len = len(ref) # Sanity check. #assert(type(hypo) is list) #assert(len(hypo) == 1) assert (type(subRef) is list) assert (type(ref) is list) assert (len(ref) > 1) for r1 in ref: for r2 in subRef: bleu_scorer += (r1, [r2]) #score, scores = bleu_scorer.compute_score(option='shortest') score, scores = bleu_scorer.compute_score(option='closest', verbose=0) #score, scores = bleu_scorer.compute_score(option='average', verbose=1) # return (bleu, bleu_info) scores = np.sum(scores, axis=0) scores = scores.reshape((len(ref), len(subRef))) scores = np.sum(scores, axis=1) return score, scores
class evalSentence: def __init__(self, coco, useBleu=False, useCider=False): self.coco = coco self.useBleu = useBleu self.useCider = useCider self.params = {'image_id': coco.getImgIds()} imgIds = self.params['image_id'] gts = {} for imgId in imgIds: gts[imgId] = self.coco.imgToAnns[imgId] if self.useBleu: self.b_scorer = BleuScorer() if self.useCider: self.c_scorer = CiderScorer() print('tokenization...') tokenizer = PTBTokenizer() gts = tokenizer.tokenize(gts) for imgId in imgIds: ref = gts[imgId] assert (type(ref) is list) assert (len(ref) > 0) if self.useCider: self.c_scorer += (None, ref) if self.useCider: self.c_scorer.compute_doc_freq() assert (len(self.c_scorer.ctest) >= max( self.c_scorer.document_frequency.values())) def eval_cider(self, test, ref): assert (self.useCider) c_score = self.c_scorer.compute_cider(test, ref) return np.array(c_score) def eval_bleu(self, test, ref): assert (self.useBleu) self.b_scorer.reset_list() for ts, rs in zip(test, ref): self.b_scorer += (ts, rs) b_score, b_scores = self.b_scorer.compute_score() return b_scores[3] # return bleu_4
] result_json.extend([{ "image_id": image_id.item(), "caption": caption, "true_cap": get_sent_from_vocab(clean_sent(refs[0][1:])) } for image_id, caption, refs in zip(ids, pred_sentences, captions)]) for j, refs in enumerate(captions): bleu_scorer += (pred_sentences[j], [ get_sent_from_vocab(clean_sent(ref[1:])) for ref in refs ]) curr_score, _ = bleu_scorer.compute_score(option='closest', verbose=0) print(curr_score) print("Bleu : %.4f" % (sum(curr_score) / 4)) with open(result_file, "w") as f: json.dump(result_json, f) # coco_val_res = coco_val.loadRes(result_file) # coco_eval = COCOEvalCap(coco_val, coco_val_res) # ipdb.set_trace() # coco_eval.params['image_id'] = coco_val_res.getImgIds() # since we use subset of val # coco_eval.evaluate() # # ipdb.set_trace()
def evaluate_stylize(G12, G21, loader, opts, split='test'): """ Evaluates sentence generation from both the generators using BLEU, CIDER, METEOR, ROUGE-L, SPICE """ depleted = False sents_s1_all = [] # GT s1 sents_s2_hat_all = [] # GT s1 -> s2 hat sents_s2_all = [] # GT s2 sents_s1_hat_all = [] # GT s2 -> s1 hat while not depleted: # Sents: batch_size x max_length [w1, w2, ..., <eos>, <pad>, <pad>, ...] # Masks: batch_size x max_length [ 1, 1, ..., 1, 0, 0, ...] if split == 'train': sents_s1, masks_s1, sents_s2, masks_s2, depleted = loader.next_batch_train( ) elif split == 'val': sents_s1, masks_s1, sents_s2, masks_s2, depleted = loader.next_batch_val( ) elif split == 'test': sents_s1, masks_s1, sents_s2, masks_s2, depleted = loader.next_batch_test( ) batch_size = sents_s1.shape[0] # Assuming both styles have same max_length max_length = sents_s1.shape[1] # source input must not contain a start token input_sents_s1 = torch.LongTensor(sents_s1) input_sents_s2 = torch.LongTensor(sents_s2) if use_cuda: input_sents_s1 = input_sents_s1.cuda() input_sents_s2 = input_sents_s2.cuda() input_sents_s1 = Variable(input_sents_s1) input_sents_s2 = Variable(input_sents_s2) # encode the input source sentence input_sents_s1_encoded, hidden_12 = G12.encode(input_sents_s1) input_sents_s2_encoded, hidden_21 = G21.encode(input_sents_s2) # generate the predicted target # initial input must be the start token decoder_input_s2_hat = Variable( torch.LongTensor(np.ones((batch_size, 1)) * opts.start_idx_s2)) decoder_input_s1_hat = Variable( torch.LongTensor(np.ones((batch_size, 1)) * opts.start_idx_s1)) if use_cuda: decoder_input_s2_hat = decoder_input_s2_hat.cuda() decoder_input_s1_hat = decoder_input_s1_hat.cuda() rollouts_s2_hat, _ = G12.decoder_rollout(max_length, decoder_input_s2_hat, hidden_12, input_sents_s1_encoded, opts.alpha) rollouts_s1_hat, _ = G21.decoder_rollout(max_length, decoder_input_s1_hat, hidden_21, input_sents_s2_encoded, opts.alpha) sents_s2_hat = rollouts_s2_hat.data.cpu().numpy().astype(int) sents_s1_hat = rollouts_s1_hat.data.cpu().numpy().astype(int) # computing the string sentences sents_s1_all.extend(get_sentence_from_np(sents_s1, loader, src=True)) sents_s1_hat_all.extend( get_sentence_from_np(sents_s1_hat, loader, src=True)) sents_s2_all.extend(get_sentence_from_np(sents_s2, loader, src=False)) sents_s2_hat_all.extend( get_sentence_from_np(sents_s2_hat, loader, src=False)) # Compute BLEU scores bleu_scorer_G21 = BleuScorer(n=4) bleu_scorer_G12 = BleuScorer(n=4) for i in range(len(sents_s1_all)): bleu_scorer_G21 += (sents_s1_hat_all[i], [sents_s1_all[i]]) bleu_scorer_G12 += (sents_s2_hat_all[i], [sents_s2_all[i]]) bleu_G21, _ = bleu_scorer_G21.compute_score(option='closest') bleu_G12, _ = bleu_scorer_G12.compute_score(option='closest') print( 'BLEU scores for Style 1 to 2 ===> B1: %.3f B2: %.3f B3: %.3f B4: %.3f' % (bleu_G12[0], bleu_G12[1], bleu_G12[2], bleu_G12[3])) print( 'BLEU scores for Style 2 to 1 ===> B1: %.3f B2: %.3f B3: %.3f B4: %.3f' % (bleu_G21[0], bleu_G21[1], bleu_G21[2], bleu_G21[3]))
# from https://github.com/mtanti/coco-caption/blob/master/pycocoevalcap/bleu/bleu_scorer.py from bleu_scorer import BleuScorer true_sentences = [] pred_sentences = [] f = open("eval/preds.txt", "r") for pred in f: pred = pred.split("[SEP]", 1)[0] pred_sentences.append(pred) #print(pred) f.close() g = open("eval/golds.txt", "r") for true in g: true = true.split("[SEP]", 1)[0] true_sentences.append([true]) #print(true) g.close() bleu_scorer = BleuScorer(n=4) # up to 4 gram for true, pred in zip(true_sentences, pred_sentences): bleu_scorer += (pred, true) scores, instance_scores = bleu_scorer.compute_score(option='closest', verbose=0) print("BLEU1 score: ", scores[0]) print("BLEU4 score: ", scores[3])
def bleu(output, ref): scorer = BleuScorer(n=4) scorer += (output.lower(), [ref.lower()]) score, _ = scorer.compute_score() return score