def evaluate(prov,
datapath='.',
model_path='model.zip',
batch_size=128,
task=None,
encode_sentences=None,
encode_images=None,
tokenize=words,
split='val'):
model = bundle.load(path=model_path)
model.task.cuda()
model.task.eval()
scaler = model.scaler
batcher = model.batcher
mapper = batcher.mapper
sents = list(prov.iterSentences(split=split))
sents_tok = [tokenize(sent) for sent in sents]
predictions = encode_sentences(model, sents_tok, batch_size=batch_size)
images = list(prov.iterImages(split=split))
img_fs = encode_images(model, [img['feat'] for img in images])
#img_fs = list(scaler.transform([ image['feat'] for image in images ]))
correct_img = numpy.array(
[[sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))]
for i in range(len(sents))])
return ranking(img_fs,
predictions,
correct_img,
ns=(1, 5, 10),
exclude_self=False)
def epoch_eval(mode='image', para=False, flickr=False):
if para: # compatibility
mode = 'para'
if mode == 'para':
return epoch_eval_para(flickr=flickr)
elif mode == 'corr':
return epoch_eval_corr()
elif mode == 'corr_image':
return epoch_eval_corr_image()
# otherwise image
model.task.eval()
task = model.task
scaler = model.scaler
batcher = model.batcher
mapper = batcher.mapper
sents = list(prov.iterSentences(split=eval_config['split']))
sents_tok = [eval_config['tokenize'](sent) for sent in sents]
predictions = eval_config['encode_sentences'](
model, sents_tok, batch_size=eval_config['batch_size'])
images = list(prov.iterImages(split=eval_config['split']))
img_fs = eval_config['encode_images'](model,
[img['feat'] for img in images])
correct_img = numpy.array([[
sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))
] for i in range(len(sents))])
result = ranking(img_fs,
predictions,
correct_img,
ns=(1, 5, 10),
exclude_self=False)
model.task.train()
return result
def retrieval(self, net=None):
img_fs = self.encode_images(net, [ s['feat'] for s in self.images ])
if net is None:
pred = self.pred
else:
with testing(net):
pred = self.encode_sentences(net, self.sentence_data, batch_size=self.config['batch_size'])
result = {}
ret = ranking(img_fs, pred, self.correct_img, ns=(1,5,10), exclude_self=False)
result['recall@1'] = numpy.mean(ret['recall'][1])
result['recall@5'] = numpy.mean(ret['recall'][5])
result['recall@10'] = numpy.mean(ret['recall'][10])
result['medr'] = numpy.median(ret['ranks'])
return result
def epoch_eval():
model.task.eval()
task = model.task
scaler = model.scaler
batcher = model.batcher
mapper = batcher.mapper
sents = list(prov.iterSentences(split=eval_config['split']))
sents_tok = [eval_config['tokenize'](sent) for sent in sents]
predictions = eval_config['encode_sentences'](
model, sents_tok, batch_size=eval_config['batch_size'])
images = list(prov.iterImages(split=eval_config['split']))
img_fs = imaginet.task.encode_images(model,
[img['feat'] for img in images])
correct_img = numpy.array([[
sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))
] for i in range(len(sents))])
return evaluate.ranking(img_fs,
predictions,
correct_img,
ns=(1, 5, 10),
exclude_self=False)