def eval_model(db, net, trfs, pooling='mean', gemp=3, detailed=False, whiten=None, aqe=None, adba=None, threads=8, batch_size=16, save_feats=None, load_feats=None, dbg=()): """ Evaluate a trained model (network) on a given dataset. The dataset is supposed to contain the evaluation code. """ print("\n>> Evaluation...") query_db = db.get_query_db() # load DB feats bdescs = [] qdescs = [] bdescs = np.load(os.path.join(load_feats, 'feats.bdescs.npy')) qdescs = bdescs if whiten is not None: bdescs = common.whiten_features(tonumpy(bdescs), net.pca, **whiten) qdescs = common.whiten_features(tonumpy(qdescs), net.pca, **whiten) if adba is not None: bdescs = expand_descriptors(bdescs, **args.adba) if aqe is not None: qdescs = expand_descriptors(qdescs, db=bdescs, **args.aqe) scores = matmul(qdescs, bdescs) data_sorted = np.argsort(-scores) del bdescs del qdescs return data_sorted
def expand_descriptors(descs, db=None, alpha=0, k=0): assert k >= 0 and alpha >= 0, 'k and alpha must be non-negative' if k == 0: return descs descs = tonumpy(descs) n = descs.shape[0] db_descs = tonumpy(db if db is not None else descs) sim = matmul(descs, db_descs) if db is None: sim[np.diag_indices(n)] = 0 idx = np.argpartition(sim, int(-k), axis=1)[:, int(-k):] descs_aug = np.zeros_like(descs) for i in range(n): new_q = np.vstack([db_descs[j, :] * sim[i, j]**alpha for j in idx[i]]) new_q = np.vstack([descs[i], new_q]) new_q = np.mean(new_q, axis=0) descs_aug[i] = new_q / np.linalg.norm(new_q) return descs_aug
def eval_model(db, net, trfs, pooling='mean', gemp=3, detailed=False, whiten=None, aqe=None, adba=None, threads=8, batch_size=16, save_feats=None, load_feats=None, dbg=()): """ Evaluate a trained model (network) on a given dataset. The dataset is supposed to contain the evaluation code. """ print("\n>> Evaluation...") query_db = db.get_query_db() # extract DB feats bdescs = [] qdescs = [] if not load_feats: trfs_list = [trfs] if isinstance(trfs, str) else trfs for trfs in trfs_list: kw = dict(iscuda=net.iscuda, threads=threads, batch_size=batch_size, same_size='Pad' in trfs or 'Crop' in trfs) bdescs.append( extract_image_features(db, trfs, net, desc="DB", **kw)) # extract query feats qdescs.append( bdescs[-1] if db is query_db else extract_image_features( query_db, trfs, net, desc="query", **kw)) # pool from multiple transforms (scales) bdescs = F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1) qdescs = F.normalize(pool(qdescs, pooling, gemp), p=2, dim=1) else: bdescs = np.load(os.path.join(load_feats, 'feats.bdescs.npy')) if query_db is not db: qdescs = np.load(os.path.join(load_feats, 'feats.qdescs.npy')) else: qdescs = bdescs if save_feats: mkdir(save_feats) np.save(os.path.join(save_feats, 'feats.bdescs.npy'), bdescs.cpu().numpy()) if query_db is not db: np.save(os.path.join(save_feats, 'feats.qdescs.npy'), qdescs.cpu().numpy()) if whiten is not None: bdescs = common.whiten_features(tonumpy(bdescs), net.pca, **whiten) qdescs = common.whiten_features(tonumpy(qdescs), net.pca, **whiten) if adba is not None: bdescs = expand_descriptors(bdescs, **args.adba) if aqe is not None: qdescs = expand_descriptors(qdescs, db=bdescs, **args.aqe) scores = matmul(qdescs, bdescs) del bdescs del qdescs res = {} try: aps = [ db.eval_query_AP(q, s) for q, s in enumerate(tqdm.tqdm(scores, desc='AP')) ] if not isinstance(aps[0], dict): aps = [float(e) for e in aps] if detailed: res['APs'] = aps # Queries with no relevants have an AP of -1 res['mAP'] = float(np.mean([e for e in aps if e >= 0])) else: modes = aps[0].keys() for mode in modes: apst = [float(e[mode]) for e in aps] if detailed: res['APs' + '-' + mode] = apst # Queries with no relevants have an AP of -1 res['mAP' + '-' + mode] = float( np.mean([e for e in apst if e >= 0])) except NotImplementedError: print(" AP not implemented!") try: tops = [ db.eval_query_top(q, s) for q, s in enumerate(tqdm.tqdm(scores, desc='top1')) ] if detailed: res['tops'] = tops for k in tops[0]: res['top%d' % k] = float(np.mean([top[k] for top in tops])) except NotImplementedError: pass return res
def test(db, net, trfs, pooling='mean', gemp=3, detailed=False, threads=8, batch_size=16): """ Evaluate a trained model (network) on a given dataset. The dataset is supposed to contain the evaluation code. """ print("\n>> Evaluation...") query_db = db.get_query_db() # extract DB feats bdescs = [] qdescs = [] trfs_list = [trfs] if isinstance(trfs, str) else trfs for trfs in trfs_list: kw = dict(iscuda=net.iscuda, threads=threads, batch_size=batch_size, same_size='Pad' in trfs or 'Crop' in trfs) bdescs.append(extract_image_features(db, trfs, net, desc="DB", **kw)) # extract query feats qdescs.append(bdescs[-1] if db is query_db else extract_image_features( query_db, trfs, net, desc="query", **kw)) # pool from multiple transforms (scales) bdescs = F.normalize(pool(bdescs, pooling, gemp), p=2, dim=1) qdescs = F.normalize(pool(qdescs, pooling, gemp), p=2, dim=1) bdescs = tonumpy(bdescs) qdescs = tonumpy(qdescs) scores = matmul(qdescs, bdescs) del bdescs del qdescs res = {} try: aps = [ db.eval_query_AP(q, s) for q, s in enumerate(tqdm.tqdm(scores, desc='AP')) ] if not isinstance(aps[0], dict): aps = [float(e) for e in aps] if detailed: res['APs'] = aps # Queries with no relevants have an AP of -1 res['mAP'] = float(np.mean([e for e in aps if e >= 0])) else: modes = aps[0].keys() for mode in modes: apst = [float(e[mode]) for e in aps] if detailed: res['APs' + '-' + mode] = apst # Queries with no relevants have an AP of -1 res['mAP' + '-' + mode] = float( np.mean([e for e in apst if e >= 0])) except NotImplementedError: print(" AP not implemented!") #writer.add_scalar('mAP', res['mAP'], epoch) return res