def cmd_eval(dataset='coco',
datapath='.',
scaler_path='scaler.pkl.gz',
input_v='predict_v.npy',
input_r='predict_r.npy',
output='eval.json'):
scaler = pickle.load(gzip.open(scaler_path))
preds_v = numpy.load(input_v)
preds_r = numpy.load(input_r)
prov = dp.getDataProvider(dataset, root=datapath)
sents = list(prov.iterSentences(split='val'))
images = list(prov.iterImages(split='val'))
img_fs = list(scaler.transform([ image['feat'] for image in images ]))
correct_img = numpy.array([ [ sents[i]['imgid']==images[j]['imgid']
for j in range(len(images)) ]
for i in range(len(sents)) ])
correct_para = numpy.array([ [ sents[i]['imgid'] == sents[j]['imgid']
for j in range(len(sents)) ]
for i in range(len(sents)) ])
r_img = evaluate.ranking(img_fs, preds_v, correct_img, ns=(1,5,10), exclude_self=False)
r_para_v = evaluate.ranking(preds_v, preds_v, correct_para, ns=(1,5,10), exclude_self=True)
r_para_r = evaluate.ranking(preds_r, preds_r, correct_para, ns=(1,5,10), exclude_self=True)
r = {'img':r_img, 'para_v':r_para_v,'para_r':r_para_r }
json.dump(r, open(output, 'w'))
for mode in ['img', 'para_v', 'para_r']:
print '{} median_rank'.format(mode), numpy.median(r[mode]['ranks'])
for n in (1,5,10):
print '{} recall@{}'.format(mode, n), numpy.mean(r[mode]['recall'][n])
sys.stdout.flush()
def cmd_eval(dataset='coco',
datapath='.',
scaler_path='scaler.pkl.gz',
input_v='predict_v.npy',
input_r='predict_r.npy',
output='eval.json'):
scaler = pickle.load(gzip.open(scaler_path))
preds_v = numpy.load(input_v)
preds_r = numpy.load(input_r)
prov = dp.getDataProvider(dataset, root=datapath)
sents = list(prov.iterSentences(split='val'))
images = list(prov.iterImages(split='val'))
img_fs = list(scaler.transform([image['feat'] for image in images]))
correct_img = numpy.array(
[[sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))]
for i in range(len(sents))])
correct_para = numpy.array(
[[sents[i]['imgid'] == sents[j]['imgid'] for j in range(len(sents))]
for i in range(len(sents))])
r_img = evaluate.ranking(img_fs,
preds_v,
correct_img,
ns=(1, 5, 10),
exclude_self=False)
r_para_v = evaluate.ranking(preds_v,
preds_v,
correct_para,
ns=(1, 5, 10),
exclude_self=True)
r_para_r = evaluate.ranking(preds_r,
preds_r,
correct_para,
ns=(1, 5, 10),
exclude_self=True)
r = {'img': r_img, 'para_v': r_para_v, 'para_r': r_para_r}
json.dump(r, open(output, 'w'))
for mode in ['img', 'para_v', 'para_r']:
print '{} median_rank'.format(mode), numpy.median(r[mode]['ranks'])
for n in (1, 5, 10):
print '{} recall@{}'.format(mode,
n), numpy.mean(r[mode]['recall'][n])
sys.stdout.flush()
def evaluate(dataset='coco',
datapath='.',
model_path='model.zip',
batch_size=128,
tokenize=phonemes
):
model = visual.load(path=model_path)
task = model.Visual
scaler = model.scaler
batcher = model.batcher
mapper = batcher.mapper
prov = dp.getDataProvider(dataset, root=datapath)
sents_tok = [ tokenize(sent) for sent in prov.iterSentences(split='val') ]
predictions = visual.predict_img(model, sents_tok, batch_size=batch_size)
sents = list(prov.iterSentences(split='val'))
images = list(prov.iterImages(split='val'))
img_fs = list(scaler.transform([ image['feat'] for image in images ]))
correct_img = numpy.array([ [ sents[i]['imgid']==images[j]['imgid']
for j in range(len(images)) ]
for i in range(len(sents)) ] )
return ranking(img_fs, predictions, correct_img, ns=(1,5,10), exclude_self=False)
def evaluate(dataset='coco',
datapath='.',
model_path='model.zip',
batch_size=128,
tokenize=phonemes):
model = visual.load(path=model_path)
task = model.Visual
scaler = model.scaler
batcher = model.batcher
mapper = batcher.mapper
prov = dp.getDataProvider(dataset, root=datapath)
sents_tok = [tokenize(sent) for sent in prov.iterSentences(split='val')]
predictions = visual.predict_img(model, sents_tok, batch_size=batch_size)
sents = list(prov.iterSentences(split='val'))
images = list(prov.iterImages(split='val'))
img_fs = list(scaler.transform([image['feat'] for image in images]))
correct_img = numpy.array(
[[sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))]
for i in range(len(sents))])
return ranking(img_fs,
predictions,
correct_img,
ns=(1, 5, 10),
exclude_self=False)
