def extract_features(db,
net,
trfs,
pooling='mean',
gemp=3,
detailed=False,
whiten=None,
threads=8,
batch_size=16,
output=None,
dbg=()):
""" Extract features from trained model (network) on a given dataset.
"""
print("\n>> Extracting features...")
try:
query_db = db.get_query_db()
except NotImplementedError:
query_db = None
# extract DB feats
bdescs = []
qdescs = []
trfs_list = [trfs] if isinstance(trfs, str) else trfs
for trfs in trfs_list:
kw = dict(iscuda=net.iscuda,
threads=threads,
batch_size=batch_size,
same_size='Pad' in trfs or 'Crop' in trfs)
bdescs.append(
test.extract_image_features(db, trfs, net, desc="DB", **kw))
# extract query feats
if query_db is not None:
qdescs.append(
bdescs[-1] if db is query_db else test.extract_image_features(
query_db, trfs, net, desc="query", **kw))
# pool from multiple transforms (scales)
bdescs = tonumpy(F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1))
if query_db is not None:
qdescs = tonumpy(F.normalize(pool(qdescs, pooling, gemp), p=2, dim=1))
if whiten is not None:
bdescs = common.whiten_features(bdescs, net.pca, **whiten)
if query_db is not None:
qdescs = common.whiten_features(qdescs, net.pca, **whiten)
mkdir(output, isfile=True)
if query_db is db or query_db is None:
np.save(output, bdescs)
else:
o = osp.splitext(output)
np.save(o[0] + '.qdescs' + o[1], qdescs)
np.save(o[0] + '.dbdescs' + o[1], bdescs)
print('Features extracted.')
def eval_model(db,
net,
trfs,
pooling='mean',
gemp=3,
detailed=False,
whiten=None,
aqe=None,
adba=None,
threads=8,
batch_size=16,
save_feats=None,
load_feats=None,
dbg=()):
""" Evaluate a trained model (network) on a given dataset.
The dataset is supposed to contain the evaluation code.
"""
print("\n>> Evaluation...")
query_db = db.get_query_db()
# extract DB feats
bdescs = []
qdescs = []
if not load_feats:
trfs_list = [trfs] if isinstance(trfs, str) else trfs
for trfs in trfs_list:
kw = dict(iscuda=net.iscuda,
threads=threads,
batch_size=batch_size,
same_size='Pad' in trfs or 'Crop' in trfs)
bdescs.append(
extract_image_features(db, trfs, net, desc="DB", **kw))
# extract query feats
qdescs.append(
bdescs[-1] if db is query_db else extract_image_features(
query_db, trfs, net, desc="query", **kw))
# pool from multiple transforms (scales)
bdescs = F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1)
qdescs = F.normalize(pool(qdescs, pooling, gemp), p=2, dim=1)
else:
bdescs = np.load(os.path.join(load_feats, 'feats.bdescs.npy'))
if query_db is not db:
qdescs = np.load(os.path.join(load_feats, 'feats.qdescs.npy'))
else:
qdescs = bdescs
if save_feats:
mkdir(save_feats)
np.save(os.path.join(save_feats, 'feats.bdescs.npy'),
bdescs.cpu().numpy())
if query_db is not db:
np.save(os.path.join(save_feats, 'feats.qdescs.npy'),
qdescs.cpu().numpy())
if whiten is not None:
bdescs = common.whiten_features(tonumpy(bdescs), net.pca, **whiten)
qdescs = common.whiten_features(tonumpy(qdescs), net.pca, **whiten)
if adba is not None:
bdescs = expand_descriptors(bdescs, **args.adba)
if aqe is not None:
qdescs = expand_descriptors(qdescs, db=bdescs, **args.aqe)
scores = matmul(qdescs, bdescs)
del bdescs
del qdescs
res = {}
try:
aps = [
db.eval_query_AP(q, s)
for q, s in enumerate(tqdm.tqdm(scores, desc='AP'))
]
if not isinstance(aps[0], dict):
aps = [float(e) for e in aps]
if detailed:
res['APs'] = aps
# Queries with no relevants have an AP of -1
res['mAP'] = float(np.mean([e for e in aps if e >= 0]))
else:
modes = aps[0].keys()
for mode in modes:
apst = [float(e[mode]) for e in aps]
if detailed:
res['APs' + '-' + mode] = apst
# Queries with no relevants have an AP of -1
res['mAP' + '-' + mode] = float(
np.mean([e for e in apst if e >= 0]))
except NotImplementedError:
print(" AP not implemented!")
try:
tops = [
db.eval_query_top(q, s)
for q, s in enumerate(tqdm.tqdm(scores, desc='top1'))
]
if detailed:
res['tops'] = tops
for k in tops[0]:
res['top%d' % k] = float(np.mean([top[k] for top in tops]))
except NotImplementedError:
pass
return res
else:
net.pca = None
args.whiten = None
# Evaluate
res = eval_model(dataset,
net,
args.trfs,
pooling=args.pooling,
gemp=args.gemp,
detailed=args.detailed,
threads=args.threads,
dbg=args.dbg,
whiten=args.whiten,
aqe=args.aqe,
adba=args.adba,
save_feats=args.save_feats,
load_feats=args.load_feats)
print(' * ' + '\n * '.join(['%s = %g' % p for p in res.items()]))
if args.out_json:
# write to file
try:
data = json.load(open(args.out_json))
except IOError:
data = {}
data[args.dataset] = res
mkdir(args.out_json)
open(args.out_json, 'w').write(json.dumps(data, indent=1))
print("saved to " + args.out_json)
def eval_model(db, net, trfs, pooling='mean', gemp=3, detailed=False, whiten=None,
aqe=None, adba=None, threads=8, batch_size=16, save_feats=None,
load_feats=None, load_distractors=None, dbg=()):
""" Evaluate a trained model (network) on a given dataset.
The dataset is supposed to contain the evaluation code.
"""
print("\n>> Evaluation...")
query_db = db.get_query_db()
# extract DB feats
bdescs = []
qdescs = []
if not load_feats:
trfs_list = [trfs] if isinstance(trfs, str) else trfs
for trfs in trfs_list:
kw = dict(iscuda=net.iscuda, threads=threads, batch_size=batch_size, same_size='Pad' in trfs or 'Crop' in trfs)
bdescs.append( extract_image_features(db, trfs, net, desc="DB", **kw) )
# extract query feats
qdescs.append( bdescs[-1] if db is query_db else extract_image_features(query_db, trfs, net, desc="query", **kw) )
# pool from multiple transforms (scales)
bdescs = F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1)
qdescs = F.normalize(pool(qdescs, pooling, gemp), p=2, dim=1)
else:
bdescs = np.load(os.path.join(load_feats, 'feats.bdescs.npy'))
qdescs = np.load(os.path.join(load_feats, 'feats.qdescs.npy'))
if save_feats:
mkdir(save_feats, isfile=True)
np.save(save_feats+'.bdescs', bdescs.cpu().numpy())
if query_db is not db:
np.save(save_feats+'.qdescs', qdescs.cpu().numpy())
exit()
if load_distractors:
ddescs = [ np.load(os.path.join(load_distractors, '%d.bdescs.npy' % i)) for i in tqdm.tqdm(range(0,1000), 'Distractors') ]
bdescs = np.concatenate([tonumpy(bdescs)] + ddescs)
qdescs = tonumpy(qdescs) # so matmul below can work
if whiten is not None:
bdescs = common.whiten_features(tonumpy(bdescs), net.pca, **whiten)
qdescs = common.whiten_features(tonumpy(qdescs), net.pca, **whiten)
if adba is not None:
bdescs = expand_descriptors(bdescs, **args.adba)
if aqe is not None:
qdescs = expand_descriptors(qdescs, db=bdescs, **args.aqe)
scores = matmul(qdescs, bdescs)
del bdescs
del qdescs
res = {}
try:
aps = [db.eval_query_AP(q, s) for q,s in enumerate(tqdm.tqdm(scores,desc='AP'))]
if not isinstance(aps[0], dict):
aps = [float(e) for e in aps]
if detailed: res['APs'] = aps
res['mAP'] = float(np.mean([e for e in aps if e>=0])) # Queries with no relevants have an AP of -1
else:
modes = aps[0].keys()
for mode in modes:
apst = [float(e[mode]) for e in aps]
if detailed: res['APs'+'-'+mode] = apst
res['mAP'+'-'+mode] = float(np.mean([e for e in apst if e>=0])) # Queries with no relevants have an AP of -1
if 'ap' in dbg:
pdb.set_trace()
pyplot(globals())
for query in np.argsort(aps):
subplot_grid(20, 1)
pl.imshow(query_db.get_image(query))
qlabel = query_db.get_label(query)
pl.xlabel('#%d %s' % (query, qlabel))
pl_noticks()
ranked = np.argsort(scores[query])[::-1]
gt = db.get_query_groundtruth(query)[ranked]
for i,idx in enumerate(ranked):
if i+2 > 20: break
subplot_grid(20, i+2)
pl.imshow(db.get_image(idx))
pl.xlabel('#%d %s %g' % (idx, 'OK' if label==qlabel else 'BAD', scores[query,idx]))
pl_noticks()
pdb.set_trace()
except NotImplementedError:
print(" AP not implemented!")
try:
tops = [db.eval_query_top(q,s) for q,s in enumerate(tqdm.tqdm(scores,desc='top1'))]
if detailed: res['tops'] = tops
for k in tops[0]:
res['top%d'%k] = float(np.mean([top[k] for top in tops]))
except NotImplementedError:
pass
return res