Exemple #1
0
def cmd_eval(dataset='coco',
             datapath='.',
             scaler_path='scaler.pkl.gz',
             input_v='predict_v.npy',
             input_r='predict_r.npy',
             output='eval.json'):
    scaler = pickle.load(gzip.open(scaler_path))
    preds_v  = numpy.load(input_v)
    preds_r  = numpy.load(input_r)
    prov   = dp.getDataProvider(dataset, root=datapath)
    sents  = list(prov.iterSentences(split='val'))
    images = list(prov.iterImages(split='val'))
    img_fs = list(scaler.transform([ image['feat'] for image in images ]))
    correct_img = numpy.array([ [ sents[i]['imgid']==images[j]['imgid']
                              for j in range(len(images)) ]
                            for i in range(len(sents)) ])
    correct_para = numpy.array([ [ sents[i]['imgid'] == sents[j]['imgid']
                               for j in range(len(sents)) ]
                            for i in range(len(sents)) ])
    r_img = evaluate.ranking(img_fs, preds_v, correct_img, ns=(1,5,10), exclude_self=False)
    r_para_v = evaluate.ranking(preds_v, preds_v, correct_para, ns=(1,5,10), exclude_self=True)
    r_para_r  = evaluate.ranking(preds_r, preds_r, correct_para, ns=(1,5,10), exclude_self=True)
    r = {'img':r_img, 'para_v':r_para_v,'para_r':r_para_r }
    json.dump(r, open(output, 'w'))
    for mode in ['img', 'para_v', 'para_r']:
        print '{} median_rank'.format(mode), numpy.median(r[mode]['ranks'])
        for n in (1,5,10):
            print '{} recall@{}'.format(mode, n), numpy.mean(r[mode]['recall'][n])
            sys.stdout.flush()
Exemple #2
0
def cmd_eval(dataset='coco',
             datapath='.',
             scaler_path='scaler.pkl.gz',
             input_v='predict_v.npy',
             input_r='predict_r.npy',
             output='eval.json'):
    scaler = pickle.load(gzip.open(scaler_path))
    preds_v = numpy.load(input_v)
    preds_r = numpy.load(input_r)
    prov = dp.getDataProvider(dataset, root=datapath)
    sents = list(prov.iterSentences(split='val'))
    images = list(prov.iterImages(split='val'))
    img_fs = list(scaler.transform([image['feat'] for image in images]))
    correct_img = numpy.array(
        [[sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))]
         for i in range(len(sents))])
    correct_para = numpy.array(
        [[sents[i]['imgid'] == sents[j]['imgid'] for j in range(len(sents))]
         for i in range(len(sents))])
    r_img = evaluate.ranking(img_fs,
                             preds_v,
                             correct_img,
                             ns=(1, 5, 10),
                             exclude_self=False)
    r_para_v = evaluate.ranking(preds_v,
                                preds_v,
                                correct_para,
                                ns=(1, 5, 10),
                                exclude_self=True)
    r_para_r = evaluate.ranking(preds_r,
                                preds_r,
                                correct_para,
                                ns=(1, 5, 10),
                                exclude_self=True)
    r = {'img': r_img, 'para_v': r_para_v, 'para_r': r_para_r}
    json.dump(r, open(output, 'w'))
    for mode in ['img', 'para_v', 'para_r']:
        print '{} median_rank'.format(mode), numpy.median(r[mode]['ranks'])
        for n in (1, 5, 10):
            print '{} recall@{}'.format(mode,
                                        n), numpy.mean(r[mode]['recall'][n])
            sys.stdout.flush()
Exemple #3
0
def evaluate(dataset='coco',
             datapath='.',
             model_path='model.zip',
             batch_size=128,
             tokenize=phonemes
            ):
    model = visual.load(path=model_path)
    task = model.Visual
    scaler = model.scaler
    batcher = model.batcher
    mapper = batcher.mapper
    prov   = dp.getDataProvider(dataset, root=datapath)
    sents_tok =  [ tokenize(sent) for sent in prov.iterSentences(split='val') ]
    predictions = visual.predict_img(model, sents_tok, batch_size=batch_size)
    sents  = list(prov.iterSentences(split='val'))
    images = list(prov.iterImages(split='val'))
    img_fs = list(scaler.transform([ image['feat'] for image in images ]))
    correct_img = numpy.array([ [ sents[i]['imgid']==images[j]['imgid']
                                  for j in range(len(images)) ]
                                for i in range(len(sents)) ] )
    return ranking(img_fs, predictions, correct_img, ns=(1,5,10), exclude_self=False)
Exemple #4
0
def evaluate(dataset='coco',
             datapath='.',
             model_path='model.zip',
             batch_size=128,
             tokenize=phonemes):
    model = visual.load(path=model_path)
    task = model.Visual
    scaler = model.scaler
    batcher = model.batcher
    mapper = batcher.mapper
    prov = dp.getDataProvider(dataset, root=datapath)
    sents_tok = [tokenize(sent) for sent in prov.iterSentences(split='val')]
    predictions = visual.predict_img(model, sents_tok, batch_size=batch_size)
    sents = list(prov.iterSentences(split='val'))
    images = list(prov.iterImages(split='val'))
    img_fs = list(scaler.transform([image['feat'] for image in images]))
    correct_img = numpy.array(
        [[sents[i]['imgid'] == images[j]['imgid'] for j in range(len(images))]
         for i in range(len(sents))])
    return ranking(img_fs,
                   predictions,
                   correct_img,
                   ns=(1, 5, 10),
                   exclude_self=False)