def compute_score(gts, val_caps, train_imgids, val_imgids, i, j):
res = {}
for imgid in train_imgids:
res[imgid] = [val_caps[val_imgids[i]][j]]
scorer = Spice()
score, scores = scorer.compute_score(gts, res, train_imgids)
#print(score)
#print(len(scores))
return np.array(scores)
class SpiceEval():
def __init__(self):
self.evalImgs = []
self.eval = {}
self.imgToEval = {}
self.spice = Spice()
self.tokenizer = PTBTokenizer()
"""
The input have structure
{'123': [{'image_id':123, 'caption': 'xxxxx'}, {'image_id':123, 'caption': 'yyy'}], ...}
"""
def evaluate(self, gts, res):
assert set(gts.keys()) == set(res.keys())
imgIds = gts.keys()
gts = self.tokenizer.tokenize(gts)
res = self.tokenizer.tokenize(res)
# =================================================
# Set up scorers
# =================================================
# =================================================
# Compute scores
# =================================================
print 'computing %s score...' % (self.spice.method())
score, scores = self.spice.compute_score(gts, res)
print "%s: %0.3f" % ("spice", score)
self.eval['spice'] = score
print scores
for imgId, score in zip(sorted(imgIds), scores):
if not imgId in self.imgToEval:
self.imgToEval[imgId] = {}
self.imgToEval[imgId]["image_id"] = imgId
self.imgToEval[imgId]["spice"] = score
return self.eval['spice'], self.imgToEval
# 1. parse sentence using SPICE and save the parsed information into json file (adapted from create coco_sg.py)
data_path = './data/caption_sentences.txt'
sent_list = [item for item in open(data_path, 'r')]
gts = {}
res = {}
img_ids = []
for img_id, this_sent in enumerate(sent_list):
gts[img_id] = []
gts[img_id].append(this_sent)
res[img_id] = []
res[img_id].append('place holder')
img_ids.append(img_id)
scorer = Spice()
score, scores = scorer.compute_score(gts, res)
# 2. extract the parsed triplets from json file (adapted from process_spice_sg.py)
from nltk.stem import WordNetLemmatizer
from functools import partial
def change_word(lem, word_ori):
"""
Lemmatizer a word, like change 'holding' to 'hold' or
'cats' to 'cat'
"""
word_ori = word_ori.lower()
word_change = lem.lemmatize(word_ori)
if word_change == word_ori:
word_change = lem.lemmatize(word_ori, 'v')
def calc_spice(gts, res):
spice = Spice()
score, scores = spice.compute_score(gts, res)
return score, scores