示例#1
0
def compute_score(gts, val_caps, train_imgids, val_imgids, i, j):
    res = {}
    for imgid in train_imgids:
        res[imgid] = [val_caps[val_imgids[i]][j]]

    scorer = Spice()
    score, scores = scorer.compute_score(gts, res, train_imgids)
    #print(score)
    #print(len(scores))
    return np.array(scores)
示例#2
0
class SpiceEval():
    def __init__(self):
        self.evalImgs = []
        self.eval = {}
        self.imgToEval = {}
        self.spice = Spice()
        self.tokenizer = PTBTokenizer()

    """
    The input have structure
    {'123': [{'image_id':123, 'caption': 'xxxxx'}, {'image_id':123, 'caption': 'yyy'}], ...}
    """

    def evaluate(self, gts, res):
        assert set(gts.keys()) == set(res.keys())
        imgIds = gts.keys()
        gts = self.tokenizer.tokenize(gts)
        res = self.tokenizer.tokenize(res)

        # =================================================
        # Set up scorers
        # =================================================

        # =================================================
        # Compute scores
        # =================================================
        print 'computing %s score...' % (self.spice.method())
        score, scores = self.spice.compute_score(gts, res)
        print "%s: %0.3f" % ("spice", score)
        self.eval['spice'] = score
        print scores
        for imgId, score in zip(sorted(imgIds), scores):
            if not imgId in self.imgToEval:
                self.imgToEval[imgId] = {}
                self.imgToEval[imgId]["image_id"] = imgId
            self.imgToEval[imgId]["spice"] = score
        return self.eval['spice'], self.imgToEval
示例#3
0
# 1. parse sentence using SPICE and save the parsed information into json file (adapted from create coco_sg.py)
data_path = './data/caption_sentences.txt'
sent_list = [item for item in open(data_path, 'r')]

gts = {}
res = {}
img_ids = []
for img_id, this_sent in enumerate(sent_list):
    gts[img_id] = []
    gts[img_id].append(this_sent)
    res[img_id] = []
    res[img_id].append('place holder')
    img_ids.append(img_id)

scorer = Spice()
score, scores = scorer.compute_score(gts, res)

# 2. extract the parsed triplets from json file (adapted from process_spice_sg.py)
from nltk.stem import WordNetLemmatizer
from functools import partial


def change_word(lem, word_ori):
    """
    Lemmatizer a word, like change 'holding' to 'hold' or
    'cats' to 'cat'
    """
    word_ori = word_ori.lower()
    word_change = lem.lemmatize(word_ori)
    if word_change == word_ori:
        word_change = lem.lemmatize(word_ori, 'v')
示例#4
0
def calc_spice(gts, res):
    spice = Spice()
    score, scores = spice.compute_score(gts, res)
    return score, scores