def language_eval(preds, test_coco_ids, cache_path):
import sys
sys.path.insert(0, "coco_caption")
# generate target file
annFile = transform_annos(test_coco_ids)
from coco_caption.pycocotools.coco import COCO
from coco_caption.pycocoevalcap.eval import COCOEvalCap
encoder.FLOAT_REPR = lambda o: format(o, '.3f')
coco = COCO(annFile)
valids = coco.getImgIds()
# filter results to only those in MSCOCO validation set (will be about a third)
#preds_filt = [p for p in preds if p['image_id'] in valids]
preds_filt = []
image_id_filt = []
for p in preds:
if p['image_id'] in valids and p['image_id'] not in image_id_filt:
preds_filt.append(p)
image_id_filt.append(p['image_id'])
print('using %d/%d predictions' % (len(preds_filt), len(preds)))
json.dump(preds_filt,
open(cache_path,
'w')) # serialize to temporary json file. Sigh, COCO API...
cocoRes = coco.loadRes(cache_path)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
# create output dictionary
out = {}
for metric, score in cocoEval.eval.items():
out[metric] = score
imgToEval = cocoEval.imgToEval
for p in preds_filt:
image_id, caption = p['image_id'], p['caption']
imgToEval[image_id]['caption'] = caption
with open(cache_path, 'w') as outfile:
json.dump({'overall': out, 'imgToEval': imgToEval}, outfile)
return out
def language_eval(preds, model_id, split):
annFile = 'coco_caption/annotations/captions_val2014.json'
from coco_caption.pycocotools.coco import COCO
from coco_caption.pycocoevalcap.eval import COCOEvalCap
# encoder.FLOAT_REPR = lambda o: format(o, '.3f')
if not os.path.isdir('eval_results'):
os.mkdir('eval_results')
cache_path = os.path.join('eval_results/',
model_id + '_' + split + '.json')
coco = COCO(annFile)
valids = coco.getImgIds()
# filter results to only those in MSCOCO validation set (will be about a third)
preds_filt = [p for p in preds if p['image_id'] in valids]
print(len(preds_filt))
print('using %d/%d predictions' % (len(preds_filt), len(preds)))
json.dump(preds_filt,
open(cache_path,
'w')) # serialize to temporary json file. Sigh, COCO API...
cocoRes = coco.loadRes(cache_path)
cocoEval = COCOEvalCap(coco, cocoRes)
cocoEval.params['image_id'] = cocoRes.getImgIds()
cocoEval.evaluate()
# create output dictionary
out = {}
for metric, score in cocoEval.eval.items():
out[metric] = score
imgToEval = cocoEval.imgToEval
for p in preds_filt:
image_id, caption = p['image_id'], p['caption']
imgToEval[image_id]['caption'] = caption
with open(cache_path, 'w') as outfile:
json.dump({'overall': out, 'imgToEval': imgToEval}, outfile)
return out