def test(datasets, net): print(">> Evaluating network on test datasets...") image_size = 1024 # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta["mean"], std=net.meta["std"]) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening # Lw = None Lw = net.meta["Lw"]["retrieval-SfM-120k"]["ss"] # evaluate on test datasets # datasets = args.test_datasets.split(",") for dataset in datasets: start = time.time() print(">> {}: Extracting...".format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), "test")) images = [cfg["im_fname"](cfg, i) for i in range(cfg["n"])] qimages = [cfg["qim_fname"](cfg, i) for i in range(cfg["nq"])] bbxs = [tuple(cfg["gnd"][i]["bbx"]) for i in range(cfg["nq"])] # extract database and query vectors print(">> {}: database images...".format(dataset)) vecs = extract_vectors(net, images, image_size, transform) print(">> {}: query images...".format(dataset)) qvecs = extract_vectors(net, qimages, image_size, transform, bbxs) print(">> {}: Evaluating...".format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset, ranks, cfg["gnd"]) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw["m"], Lw["P"]) qvecs_lw = whitenapply(qvecs, Lw["m"], Lw["P"]) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset + " + whiten", ranks, cfg["gnd"]) print(">> {}: elapsed time: {}".format(dataset, htime(time.time() - start)))
def cal_ranks(vecs, qvecs, Lw): # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) return scores, ranks
def whiten(params, data): """Apply pre-computed whitening""" dimensions = params.pop("dimensions", None) or None assert not params, params.keys() whitening, names, values = data assert len(names) == len(values) resources = stats.ResourceUsage() time0 = time.time() whitened = whitenapply(values.T, whitening['m'], whitening['P'], dimensions) timing = time.time() - time0 metadata = { "timings": { "whitening_apply": round(timing, 2) }, "resource_usage": resources.take_current_stats().get_resources() } return metadata, names, whitened.T
def test(datasets, net, wandb_enabled=False, epoch=-1): global global_step print('>> Evaluating network on test datasets...') # for testing we use image size of max 1024 image_size = 1024 # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize( mean=net.meta['mean'], std=net.meta['std'] ) transform = transforms.Compose([ transforms.ToTensor(), normalize ]) # compute whitening if args.test_whiten: start = time.time() print('>> {}: Learning whitening...'.format(args.test_whiten)) # loading db db_root = os.path.join(get_data_root(), 'train', args.test_whiten) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))] # extract whitening vectors print('>> {}: Extracting...'.format(args.test_whiten)) wvecs = extract_vectors(net, images, image_size, transform) # implemented with torch.no_grad # learning whitening print('>> {}: Learning...'.format(args.test_whiten)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format(args.test_whiten, htime(time.time()-start))) else: Lw = None # evaluate on test datasets datasets = args.test_datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])] bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, image_size, transform) # implemented with torch.no_grad print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, image_size, transform, bbxs) # implemented with torch.no_grad print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset, ranks, cfg['gnd'], wandb_enabled=wandb_enabled, epoch=epoch, global_step=global_step) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'], wandb_enabled=wandb_enabled, epoch=epoch, global_step=global_step) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main(): args = parser.parse_args() # check if test dataset are downloaded # and download if they are not #download_train(get_data_root()) #download_test(get_data_root()) # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) if args.network_path in PRETRAINED: # pretrained networks (downloaded automatically) state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks')) else: state = torch.load(args.network_path) net = init_network(model=state['meta']['architecture'], pooling=state['meta']['pooling'], whitening=state['meta']['whitening'], mean=state['meta']['mean'], std=state['meta']['std'], pretrained=False) net.load_state_dict(state['state_dict']) # if whitening is precomputed if 'Lw' in state['meta']: net.meta['Lw'] = state['meta']['Lw'] print(">>>> loaded network: ") print(net.meta_repr()) # loading offtheshelf network elif args.network_offtheshelf is not None: offtheshelf = args.network_offtheshelf.split('-') if len(offtheshelf) == 3: if offtheshelf[2] == 'whiten': offtheshelf_whiten = True else: raise (RuntimeError( "Incorrect format of the off-the-shelf network. Examples: resnet101-gem | resnet101-gem-whiten" )) else: offtheshelf_whiten = False print(">> Loading off-the-shelf network:\n>>>> '{}'".format( args.network_offtheshelf)) net = init_network(model=offtheshelf[0], pooling=offtheshelf[1], whitening=offtheshelf_whiten) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters ms = [1] msp = 1 if args.multiscale: ms = [1, 1. / math.sqrt(2), 1. / 2] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format( args.whitening)) if args.multiscale: Lw = net.meta['Lw'][args.whitening]['ms'] else: Lw = net.meta['Lw'][args.whitening]['ss'] else: print('>> {}: Learning whitening...'.format(args.whitening)) # loading db db_root = os.path.join(get_data_root(), 'train', args.whitening) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [ cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids'])) ] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time() - start))) else: Lw = None # evaluate on test datasets datasets = args.datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])] bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset, ranks, cfg['gnd']) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd']) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start)))
def main(): args = parser.parse_args() # check if test dataset are downloaded # and download if they are not download_train(get_data_root()) download_test(get_data_root()) # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) state = torch.load(args.network_path) net = init_network(model=state['meta']['architecture'], pooling=state['meta']['pooling'], whitening=state['meta']['whitening'], mean=state['meta']['mean'], std=state['meta']['std'], pretrained=False) net.load_state_dict(state['state_dict']) print(">>>> loaded network: ") print(net.meta_repr()) # loading offtheshelf network elif args.network_offtheshelf is not None: offtheshelf = args.network_offtheshelf.split('-') if len(offtheshelf) == 3: if offtheshelf[2] == 'whiten': offtheshelf_whiten = True else: raise (RuntimeError( "Incorrect format of the off-the-shelf network. Examples: resnet101-gem | resnet101-gem-whiten" )) else: offtheshelf_whiten = False print(">> Loading off-the-shelf network:\n>>>> '{}'".format( args.network_offtheshelf)) net = init_network(model=offtheshelf[0], pooling=offtheshelf[1], whitening=offtheshelf_whiten) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters ms = [1] msp = 1 if args.multiscale: ms = [1, 1. / math.sqrt(2), 1. / 2] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if args.whitening is not None: start = time.time() print('>> {}: Learning whitening...'.format(args.whitening)) # loading db db_root = os.path.join(get_data_root(), 'train', args.whitening) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [ cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids'])) ] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time() - start))) else: Lw = None datasets = args.datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) if dataset == 'reco': images, qimages = landmark_recognition_dataset() bbxs = [None for x in qimages] elif dataset == 'retr': images, _ = landmark_retrieval_dataset() qimages = [] bbxs = [None for x in qimages] else: # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])] bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] with open('%s_fnames.pkl' % dataset, 'wb') as f: pickle.dump([images, qimages], f) # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) vecs = vecs.numpy() print('>> saving') np.save('{}_vecs.npy'.format(dataset), vecs) if len(qimages) > 0: print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) qvecs = qvecs.numpy() np.save('{}_qvecs.npy'.format(dataset), qvecs) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # TODO print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start)))
def main(): args = parser.parse_args() # check if there are unknown datasets for dataset in args.datasets.split(','): if dataset not in datasets_names: raise ValueError('Unsupported or unknown dataset: {}!'.format(dataset)) # check if test dataset are downloaded # and download if they are not download_train(get_data_root()) download_test(get_data_root()) # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) if args.network_path in PRETRAINED: # pretrained networks (downloaded automatically) state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks')) else: # fine-tuned network from path state = torch.load(args.network_path) # parsing net params from meta # architecture, pooling, mean, std required # the rest has default values, in case that is doesnt exist net_params = {} net_params['architecture'] = state['meta']['architecture'] net_params['pooling'] = state['meta']['pooling'] net_params['local_whitening'] = state['meta'].get('local_whitening', False) net_params['regional'] = state['meta'].get('regional', False) net_params['whitening'] = state['meta'].get('whitening', False) net_params['mean'] = state['meta']['mean'] net_params['std'] = state['meta']['std'] net_params['pretrained'] = False # load network net = init_network(net_params) net.load_state_dict(state['state_dict']) # if whitening is precomputed if 'Lw' in state['meta']: net.meta['Lw'] = state['meta']['Lw'] print(">>>> loaded network: ") print(net.meta_repr()) # loading offtheshelf network elif args.network_offtheshelf is not None: # parse off-the-shelf parameters offtheshelf = args.network_offtheshelf.split('-') net_params = {} net_params['architecture'] = offtheshelf[0] net_params['pooling'] = offtheshelf[1] net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:] net_params['regional'] = 'reg' in offtheshelf[2:] net_params['whitening'] = 'whiten' in offtheshelf[2:] net_params['pretrained'] = True # load off-the-shelf network print(">> Loading off-the-shelf network:\n>>>> '{}'".format(args.network_offtheshelf)) net = init_network(net_params) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters ms = list(eval(args.multiscale)) if len(ms)>1 and net.meta['pooling'] == 'gem' and not net.meta['regional'] and not net.meta['whitening']: msp = net.pool.p.item() print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms)) print(">>>> msp: {}".format(msp)) else: msp = 1 # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize( mean=net.meta['mean'], std=net.meta['std'] ) transform = transforms.Compose([ transforms.ToTensor(), normalize ]) # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening)) if len(ms)>1: Lw = net.meta['Lw'][args.whitening]['ms'] else: Lw = net.meta['Lw'][args.whitening]['ss'] else: # if we evaluate networks from path we should save/load whitening # not to compute it every time if args.network_path is not None: whiten_fn = args.network_path + '_{}_whiten'.format(args.whitening) if len(ms) > 1: whiten_fn += '_ms' whiten_fn += '.pth' else: whiten_fn = None if whiten_fn is not None and os.path.isfile(whiten_fn): print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening)) Lw = torch.load(whiten_fn) else: print('>> {}: Learning whitening...'.format(args.whitening)) # loading db db_root = os.path.join(get_data_root(), 'train', args.whitening) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} # saving whitening if whiten_fn exists if whiten_fn is not None: print('>> {}: Saving to {}...'.format(args.whitening, whiten_fn)) torch.save(Lw, whiten_fn) print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start))) else: Lw = None # evaluate on test datasets datasets = args.datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])] try: bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] except: bbxs = None # for holidaysmanrot and copydays # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) torch.save(vecs, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/data_vect.pt') torch.save(qvecs, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/query_vect.pt') print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset, ranks, cfg['gnd']) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) torch.save(vecs, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/data_lw_vect.pt') torch.save(qvecs_lw, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/query_lw_vect.pt') # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd']) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def testHolidays(net, eConfig, dataset, Lw): print('>> Evaluating network on test dataset: {}'.format(dataset)) # for testing we use image size of max 1024 image_size = 1024 ms = [1] msp = 1 if (eConfig['multiscale']): ms = [1, 1. / math.sqrt(2), 1. / 2] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # read the images and generate paths and queries-positive indexes dbpath = os.path.join(get_data_root(), 'test', 'holidays') ext = 'jpg' if dataset == 'holidays' else 'rjpg' images = sorted(os.listdir(os.path.join(dbpath, ext))) with open(os.path.join(dbpath, 'straight_gnd_holidays.pkl'), 'rb') as f: queries = pickle.load(f) positives = pickle.load(f) qidx = [] pidx = [] for i in range(len(queries)): qidx.append(images.index(queries[i])) aux = [] for j in range(len(positives[i])): aux.append(images.index(positives[i][j])) pidx.append(aux) # extract database and query vectors print('>> {}: database images...'.format(dataset)) X = extract_vectors(net, [os.path.join(dbpath, ext, n) for n in images], image_size, transform, ms=ms, msp=msp) print('>> {}: Evaluating...'.format(dataset)) # rank the similarities X = X.numpy() if (Lw is not None): X = whitenapply(X, Lw['m'], Lw['P']) scores = np.dot(X.T, X) ranks = np.argsort(-scores, axis=1) ranks = ranks[qidx, 1::] APs = [] for i, r in enumerate(ranks): trueRanks = np.isin(r, pidx[i]) trueRanks = np.where(trueRanks == True)[0] APs.append(compute_ap(trueRanks, len(pidx[i]))) mAP = np.mean(APs) print(">> {}: mAP {:.2f}".format(dataset, mAP * 100)) # return the average mAP return (dataset + ('+ multiscale' if eConfig['multiscale'] else ''), mAP)
def main(): args = parser.parse_args() # check if test dataset are downloaded # and download if they are not download_train(get_data_root()) download_test(get_data_root()) # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: net = load_network(args.network_path) # loading offtheshelf network elif args.network_offtheshelf is not None: net = load_offtheshelf(args.network_offtheshelf) # setting up the multi-scale parameters ms = [1] msp = 1 if args.multiscale: ms = [1, 1./math.sqrt(2), 1./2] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize( mean=net.meta['mean'], std=net.meta['std'] ) transform = transforms.Compose([ transforms.ToTensor(), normalize ]) # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening)) if args.multiscale: Lw = net.meta['Lw'][args.whitening]['ms'] else: Lw = net.meta['Lw'][args.whitening]['ss'] else: print('>> {}: Learning whitening...'.format(args.whitening)) if args.whitening == "scores": # special logic for scores database from score_retrieval.exports import ( db, train_images as images, ) else: # loading db db_root = os.path.join(get_data_root(), 'train', args.test_whiten) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start))) else: Lw = None # evaluate on test datasets datasets = args.datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) if dataset == "scores": # Special added logic to handle loading our score dataset from score_retrieval.exports import ( images, qimages, gnd, ) print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, ms=ms, msp=msp) else: # extract ground truth cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) gnd = cfg['gnd'] # prepare config structure for the test dataset images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])] bbxs = [tuple(gnd[i]['bbx']) for i in range(cfg['nq'])] # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) # validation print(">> {}: gnd stats: {}, {}, {}".format( dataset, len(gnd), [len(x["ok"]) for x in gnd[10:]], [len(x["junk"]) for x in gnd[10:]], )) print(">> {}: image stats: {}, {}".format(dataset, len(images), len(qimages))) assert len(gnd) == len(qimages), (len(gnd), len(qimages)) print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() print(">> {}: qvecs.shape: {}".format(dataset, qvecs.shape)) # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) print(">> {}: ranks (shape {}) head: {}".format(dataset, ranks.shape, ranks[10:,10:])) print(">> {}: gnd head: {}".format(dataset, gnd[5:])) # Compute and print metrics compute_acc(ranks, gnd, dataset) compute_mrr(ranks, gnd, dataset) compute_map_and_print(dataset, ranks, gnd) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_acc(ranks, gnd, dataset + " + whiten") compute_mrr(ranks, gnd, dataset + " + whiten") compute_map_and_print(dataset + " + whiten", ranks, gnd) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main(): #def process(network_path, datasets='oxford5k,paris6k', whitening=None, image_size=1024, multiscale = '[1]', query=None): args = parser.parse_args() #args.query = None # check if there are unknown datasets for dataset in args.datasets.split(','): if dataset not in datasets_names: raise ValueError( 'Unsupported or unknown dataset: {}!'.format(dataset)) # check if test dataset are downloaded # and download if they are not #download_train(get_data_root()) #download_test(get_data_root()) # setting up the visible GPU #os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) if args.network_path in PRETRAINED: # pretrained networks (downloaded automatically) state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks')) else: # fine-tuned network from path state = torch.load(args.network_path) # parsing net params from meta # architecture, pooling, mean, std required # the rest has default values, in case that is doesnt exist net_params = {} net_params['architecture'] = state['meta']['architecture'] net_params['pooling'] = state['meta']['pooling'] net_params['local_whitening'] = state['meta'].get( 'local_whitening', False) net_params['regional'] = state['meta'].get('regional', False) net_params['whitening'] = state['meta'].get('whitening', False) net_params['mean'] = state['meta']['mean'] net_params['std'] = state['meta']['std'] net_params['pretrained'] = False # load network net = init_network(net_params) net.load_state_dict(state['state_dict']) # if whitening is precomputed if 'Lw' in state['meta']: net.meta['Lw'] = state['meta']['Lw'] print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters ms = list(eval(args.multiscale)) if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[ 'regional'] and not net.meta['whitening']: msp = net.pool.p.item() print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms)) print(">>>> msp: {}".format(msp)) else: msp = 1 # moving network to gpu and eval mode #net.cuda() #net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format( args.whitening)) if len(ms) > 1: Lw = net.meta['Lw'][args.whitening]['ms'] else: Lw = net.meta['Lw'][args.whitening]['ss'] else: # if we evaluate networks from path we should save/load whitening # not to compute it every time if args.network_path is not None: whiten_fn = args.network_path + '_{}_whiten'.format( args.whitening) if len(ms) > 1: whiten_fn += '_ms' whiten_fn += '.pth' else: whiten_fn = None print(whiten_fn) return if whiten_fn is not None and os.path.isfile(whiten_fn): print('>> {}: Whitening is precomputed, loading it...'.format( args.whitening)) Lw = torch.load(whiten_fn) else: print('>> {}: Learning whitening...'.format(args.whitening)) # loading db db_root = os.path.join(get_data_root(), 'train', args.whitening) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [ cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids'])) ] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} # saving whitening if whiten_fn exists if whiten_fn is not None: print('>> {}: Saving to {}...'.format( args.whitening, whiten_fn)) torch.save(Lw, whiten_fn) print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time() - start))) else: Lw = None # evaluate on test datasets datasets = args.datasets.split(',') # query type for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) #for i in cfg: print(i) #print(cfg['gnd'][0]['bbx']) #return # extract database and query vectors print('>> {}: database images...'.format(dataset)) feas_dir = os.path.join(cfg['dir_data'], 'features') if not os.path.isdir(feas_dir): os.mkdir(feas_dir) feas_sv = os.path.join( feas_dir, dataset + '_' + args.network_path + '_features.pkl') if not os.path.isfile(feas_sv): images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])] vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) with open(feas_sv, 'wb') as f: pickle.dump(vecs, f) else: with open(feas_sv, 'rb') as f: vecs = pickle.load(f) print('>> {}: query images...'.format(dataset)) if args.query is not None: qimages = [args.query] qvecs = extract_vectors(net, qimages, args.image_size, transform, ms=ms, msp=msp) else: qfeas_dir = feas_dir qfeas_sv = os.path.join( qfeas_dir, dataset + '_' + args.network_path + '_qfeatures.pkl') if not os.path.isfile(qfeas_sv): qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])] try: bbxs = [ tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq']) ] except: bbxs = None qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) with open(qfeas_sv, 'wb') as f: pickle.dump(qvecs, f) else: with open(qfeas_sv, 'rb') as f: qvecs = pickle.load(f) print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() #qvecs = qvecs[:, 0].reshape(-1, 1) #args.query = True # search, rank, and print if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranksw = np.argsort(-scores, axis=0) if args.query is None: #compute_map_and_print(dataset + ' + whiten', ranksw, cfg['gnd']) compute_map_and_print1(dataset + ' + whiten', ranksw, cfg['gnd']) scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) # compute_map_and_print(dataset, ranks, cfg['gnd']) compute_map_and_print1(dataset, ranks, cfg['gnd']) else: a = [] for i in ranksw: a.append( os.path.join(cfg['dir_images'], cfg['imlist'][i[0]]) + cfg['ext']) print(a[:10]) result = cfg['dir_data'] + '_result' with open(result + '.pkl', 'wb') as f: pickle.dump(a[:10], f) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start)))
def main(): args = parser.parse_args() # check if there are unknown datasets for dataset in args.datasets.split(','): if dataset not in datasets_names: raise ValueError('Unsupported or unknown dataset: {}!'.format(dataset)) # check if test dataset are downloaded # and download if they are not # download_train(get_data_root()) # download_test(get_data_root()) # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) if args.network_path in PRETRAINED: # pretrained networks (downloaded automatically) state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks')) else: # fine-tuned network from path state = torch.load(args.network_path) # parsing net params from meta # architecture, pooling, mean, std required # the rest has default values, in case that is doesnt exist net_params = {} net_params['architecture'] = state['meta']['architecture'] net_params['pooling'] = state['meta']['pooling'] net_params['local_whitening'] = state['meta'].get('local_whitening', False) net_params['regional'] = state['meta'].get('regional', False) net_params['whitening'] = state['meta'].get('whitening', False) net_params['mean'] = state['meta']['mean'] net_params['std'] = state['meta']['std'] net_params['pretrained'] = False net_params['multi_layer_cat'] = state['meta']['multi_layer_cat'] # load network net = init_network(net_params) net.load_state_dict(state['state_dict']) # if whitening is precomputed if 'Lw' in state['meta']: net.meta['Lw'] = state['meta']['Lw'] print(">>>> loaded network: ") print(net.meta_repr()) # loading offtheshelf network elif args.network_offtheshelf is not None: # parse off-the-shelf parameters offtheshelf = args.network_offtheshelf.split('-') net_params = {} net_params['architecture'] = offtheshelf[0] net_params['pooling'] = offtheshelf[1] net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:] net_params['regional'] = 'reg' in offtheshelf[2:] net_params['whitening'] = 'whiten' in offtheshelf[2:] net_params['pretrained'] = True # load off-the-shelf network print(">> Loading off-the-shelf network:\n>>>> '{}'".format(args.network_offtheshelf)) net = init_network(net_params) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters print(">> image size: {}".format(args.image_size)) ms = list(eval(args.multiscale)) if len(ms)>1 and net.meta['pooling'] == 'gem' and not net.meta['regional'] and not net.meta['whitening']: msp = net.pool.p.item() print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms)) print(">>>> msp: {}".format(msp)) else: msp = 1 print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms)) print(">>>> msp: {}".format(msp)) # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize( mean=net.meta['mean'], std=net.meta['std'] ) transform = transforms.Compose([ transforms.ToTensor(), normalize ]) # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening)) if len(ms)>1: Lw = net.meta['Lw'][args.whitening]['ms'] else: Lw = net.meta['Lw'][args.whitening]['ss'] else: # if we evaluate networks from path we should save/load whitening # not to compute it every time if args.network_path is not None: whiten_fn = args.network_path + '_{}_whiten'.format(args.whitening) if len(ms) > 1: whiten_fn += '_ms' whiten_fn += '.pth' else: whiten_fn = None if whiten_fn is not None and os.path.isfile(whiten_fn): print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening)) Lw = torch.load(whiten_fn) else: print('>> {}: Learning whitening...'.format(args.whitening)) # loading db db_root = os.path.join(get_data_root(), 'train', args.whitening) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} # saving whitening if whiten_fn exists if whiten_fn is not None: print('>> {}: Saving to {}...'.format(args.whitening, whiten_fn)) torch.save(Lw, whiten_fn) print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start))) else: Lw = None # evaluate on test datasets datasets = args.datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])] # bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] print('>> not use bbxs...') bbxs = None # key_url_list = ParseData(os.path.join(get_data_root(), 'index.csv')) # index_image_path = os.path.join(get_data_root(), 'resize_index_image') # images = [os.path.join(index_image_path, key_url_list[i][0]) for i in range(len(key_url_list))] # key_url_list = ParseData(os.path.join(get_data_root(), 'test.csv')) # test_image_path = os.path.join(get_data_root(), 'resize_test_image') # qimages = [os.path.join(test_image_path, key_url_list[i][0]) for i in range(len(key_url_list))] # # bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] # csvfile = open(os.path.join(get_data_root(), 'index_clear.csv'), 'r') # csvreader = csv.reader(csvfile) # images = [line[:1][0] for line in csvreader] # # csvfile = open(os.path.join(get_data_root(), 'test_clear.csv'), 'r') # csvreader = csv.reader(csvfile) # qimages = [line[:1][0] for line in csvreader] # bbxs = None # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # vecs = torch.randn(2048, 5063) # vecs = torch.randn(2048, 4993) # hxq modified # bbxs = None # print('>> set no bbxs...') print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) # hxq modified, test add features map for retrieval # vecs = [vecs[i].numpy() for i in range(len(vecs))] # qvecs_temp = np.zeros((qvecs[0].shape[0], len(qvecs))) # for i in range(len(qvecs)): # qvecs_temp[:, i] = qvecs[i][:, 0].numpy() # qvecs = qvecs_temp # # scores = np.zeros((len(vecs), qvecs.shape[-1])) # for i in range(len(vecs)): # scores[i, :] = np.amax(np.dot(vecs[i].T, qvecs), 0) ranks = np.argsort(-scores, axis=0) mismatched_info = compute_map_and_print(dataset, ranks, cfg['gnd'], kappas=[1, 5, 10, 100]) # hxq added show_false_img = False if show_false_img == True: print('>> Save mismatched image tuple...') for info in mismatched_info: mismatched_img_show_save(info, qimages, images, args, bbxs=bbxs) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) mismatched_info = compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd']) # hxq added # show_false_img = False if show_false_img == True: print('>> Save mismatched image tuple...') for info in mismatched_info: mismatched_img_show_save(info, qimages, images, args, bbxs=bbxs) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def test(datasets, net, noise, image_size): global base print(">> Evaluating network on test datasets...") net.cuda() net.eval() normalize = transforms.Normalize(mean=net.meta["mean"], std=net.meta["std"]) def add_noise(img): n = noise n = F.interpolate(n.unsqueeze(0), mode=MODE, size=tuple(img.shape[-2:]), align_corners=True).squeeze() return torch.clamp(img + n, 0, 1) transform_base = transforms.Compose([transforms.ToTensor(), normalize]) transform_query = transforms.Compose( [transforms.ToTensor(), transforms.Lambda(add_noise), normalize]) if "Lw" in net.meta: Lw = net.meta["Lw"]["retrieval-SfM-120k"]["ss"] else: Lw = None # evaluate on test datasets datasets = args.test_datasets.split(",") attack_result = {} for dataset in datasets: start = time.time() print(">> {}: Extracting...".format(dataset)) cfg = configdataset(dataset, os.path.join(get_data_root(), "test")) images = [cfg["im_fname"](cfg, i) for i in range(cfg["n"])] qimages = [cfg["qim_fname"](cfg, i) for i in range(cfg["nq"])] bbxs = [tuple(cfg["gnd"][i]["bbx"]) for i in range(cfg["nq"])] # extract database and query vectors print(">> {}: database images...".format(dataset)) with torch.no_grad(): if dataset in base and str(image_size) in base[dataset]: vecs = base[dataset][str(image_size)] else: vecs = extract_vectors(net, images, image_size, transform_base) if dataset not in base: base[dataset] = {} base[dataset][str(image_size)] = vecs fname = args.network_path.replace("/", "_") + ".pkl" with open(f"base/{fname}", "wb") as f: pickle.dump(base, f) print(">> {}: query images...".format(dataset)) qvecs = extract_vectors(net, qimages, image_size, transform_query, bbxs) print(">> {}: Evaluating...".format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # whiten the vectors vecs_lw = whitenapply(vecs, Lw["m"], Lw["P"]) qvecs_lw = whitenapply(qvecs, Lw["m"], Lw["P"]) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) r = compute_map_and_print(dataset + " + whiten", ranks, cfg["gnd"]) attack_result[dataset] = r print(">> {}: elapsed time: {}".format(dataset, htime(time.time() - start))) return inv_gfr( attack_result, baseline_result[net.meta["architecture"]][net.meta["pooling"]])
def main(): # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = '0' input_resol = 512 # resolution of input image, will resize to that if larger # input_resol = 1024; # resolution of input image, will resize to that if larger scales = [1, 1 / np.sqrt(2), 1 / 2] # re-scaling factors for multi-scale extraction # sample image img_file = 'sanjuan.jpg' if not path.exists(img_file): os.system( 'wget https://raw.githubusercontent.com/gtolias/tma/master/data/input/' + img_file) img = default_loader(img_file) print("use network trained with gem pooling and FC layer") state = load_url(TRAINED['rSfM120k-tl-resnet101-gem-w'], model_dir=os.path.join(get_data_root(), 'networks')) net = init_network({ 'architecture': state['meta']['architecture'], 'pooling': state['meta']['pooling'], 'whitening': state['meta'].get('whitening') }) net.load_state_dict(state['state_dict']) net.eval() net.cuda() transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=state['meta']['mean'], std=state['meta']['std']) ]) # single-scale extraction vec = extract_ss(net, transform(imresize(img, input_resol)).unsqueeze(0).cuda()) vec = vec.data.cpu().numpy() print(vec) # multi-scale extraction vec = extract_ms(net, transform(imresize(img, input_resol)).unsqueeze(0).cuda(), ms=scales, msp=1.0) vec = vec.data.cpu().numpy() print(vec) print("\n") print( "use network trained with gem pooling, and apply the learned whitening transformation" ) state = load_url(TRAINED['retrievalSfM120k-resnet101-gem'], model_dir=os.path.join(get_data_root(), 'networks')) net = init_network({ 'architecture': state['meta']['architecture'], 'pooling': state['meta']['pooling'], 'whitening': state['meta'].get('whitening') }) net.load_state_dict(state['state_dict']) net.eval() net.cuda() transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=state['meta']['mean'], std=state['meta']['std']) ]) # single-scale extraction vec = extract_ss(net, transform(imresize(img, input_resol)).unsqueeze(0).cuda()) vec = vec.data.cpu().numpy() print(vec) whiten_ss = state['meta']['Lw']['retrieval-SfM-120k']['ss'] vec = whitenapply(vec.reshape(-1, 1), whiten_ss['m'], whiten_ss['P']).reshape(-1) print(vec) # multi-scale extraction vec = extract_ms(net, transform(imresize(img, input_resol)).unsqueeze(0).cuda(), ms=scales, msp=net.pool.p.item()) vec = vec.data.cpu().numpy() print(vec) whiten_ms = state['meta']['Lw']['retrieval-SfM-120k']['ms'] vec = whitenapply(vec.reshape(-1, 1), whiten_ms['m'], whiten_ms['P']).reshape(-1) print(vec) print("\n") print("use pre-trained (on ImageNet) network with appended mac pooling") net = init_network({ 'architecture': 'resnet101', 'pooling': 'mac', 'pretrained': True }) net.eval() net.cuda() transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) ]) # single-scale extraction vec = extract_ss(net, transform(imresize(img, input_resol)).unsqueeze(0).cuda()) vec = vec.data.cpu().numpy() print(vec) # multi-scale extraction vec = extract_ms(net, transform(imresize(img, input_resol)).unsqueeze(0).cuda(), ms=scales, msp=1.0) vec = vec.data.cpu().numpy() print(vec) print("\n")
def test(datasets, net): print('>> Evaluating network on test datasets...') # for testing we use image size of max 1024 image_size = 1024 # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if args.test_whiten: start = time.time() print('>> {}: Learning whitening...'.format(args.test_whiten)) # loading db db_root = os.path.join(get_data_root(), 'train', args.test_whiten) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [ cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids'])) ] # extract whitening vectors print('>> {}: Extracting...'.format(args.test_whiten)) wvecs = extract_vectors(net, images, image_size, transform, print_freq=10, batchsize=20) # implemented with torch.no_grad # learning whitening print('>> {}: Learning...'.format(args.test_whiten)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format(args.test_whiten, htime(time.time() - start))) else: Lw = None # evaluate on test datasets datasets = args.test_datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])] if dataset == 'cdvs_test_retrieval': bbxs = None else: bbxs = None print('>> {}: database images...'.format(dataset)) if args.pool == 'gem': ms = [1, 1 / 2**(1 / 2), 1 / 2] else: ms = [1] if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[ 'regional'] and not net.meta['whitening']: msp = net.pool.p.item() print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms)) print(">>>> msp: {}".format(msp)) else: msp = 1 vecs = extract_vectors(net, images, image_size, transform, bbxs, ms=ms, msp=msp, print_freq=1000, batchsize=1) # implemented with torch.no_grad print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, image_size, transform, bbxs, ms=ms, msp=msp, print_freq=1000, batchsize=1) # implemented with torch.no_grad print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) if dataset == 'cdvs_test_retrieval': compute_map_and_print(dataset, ranks, cfg['gnd_id']) else: compute_map_and_print(dataset, ranks, cfg['gnd']) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd']) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start)))
def main(): args = parser.parse_args() # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) if args.network_path in PRETRAINED: # pretrained networks (downloaded automatically) state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks')) else: # fine-tuned network from path state = torch.load(args.network_path) # parsing net params from meta # architecture, pooling, mean, std required # the rest has default values, in case that is doesnt exist net_params = {} net_params['architecture'] = state['meta']['architecture'] net_params['pooling'] = state['meta']['pooling'] net_params['local_whitening'] = state['meta'].get( 'local_whitening', False) net_params['regional'] = state['meta'].get('regional', False) net_params['whitening'] = state['meta'].get('whitening', False) net_params['mean'] = state['meta']['mean'] net_params['std'] = state['meta']['std'] net_params['pretrained'] = False # load network net = init_network(net_params) net.load_state_dict(state['state_dict']) # if whitening is precomputed if 'Lw' in state['meta']: net.meta['Lw'] = state['meta']['Lw'] print(">>>> loaded network: ") if "epoch" in state: print("Model after {} epochs".format(state["epoch"])) print(net.meta_repr()) # loading offtheshelf network elif args.network_offtheshelf is not None: # parse off-the-shelf parameters offtheshelf = args.network_offtheshelf.split('-') net_params = {} net_params['architecture'] = offtheshelf[0] net_params['pooling'] = offtheshelf[1] net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:] net_params['regional'] = 'reg' in offtheshelf[2:] net_params['whitening'] = 'whiten' in offtheshelf[2:] net_params['pretrained'] = True # load off-the-shelf network print(">> Loading off-the-shelf network:\n>>>> '{}'".format( args.network_offtheshelf)) net = init_network(net_params) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters: test both single scale and multiscale ms_singlescale = [1] msp_singlescale = 1 ms_multiscale = list(eval(args.multiscale)) msp_multiscale = 1 if len(ms_multiscale ) > 1 and net.meta['pooling'] == 'gem' and not net.meta[ 'regional'] and not net.meta['whitening']: msp_multiscale = net.pool.p.item() print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms_multiscale)) print(">>>> msp: {}".format(msp_multiscale)) # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format( args.whitening)) Lw = net.meta['Lw'][args.whitening] else: # if we evaluate networks from path we should save/load whitening # not to compute it every time if args.network_path is not None: whiten_fn = args.network_path + '_{}_whiten'.format( args.whitening) whiten_fn += '.pth' else: whiten_fn = None if whiten_fn is not None and os.path.isfile(whiten_fn): print('>> {}: Whitening is precomputed, loading it...'.format( args.whitening)) Lw = torch.load(whiten_fn) else: Lw = {} for whiten_type, ms, msp in zip( ["ss", "ms"], [ms_singlescale, ms_multiscale], [msp_singlescale, msp_multiscale]): print('>> {0}: Learning whitening {1}...'.format( args.whitening, whiten_type)) # loading db db_root = os.path.join(get_data_root(), 'train', args.whitening) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join( db_root, '{}-whiten.pkl'.format(args.whitening)) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [ cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids'])) ] # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(args.whitening)) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw[whiten_type] = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format( args.whitening, htime(time.time() - start))) # saving whitening if whiten_fn exists if whiten_fn is not None: print('>> {}: Saving to {}...'.format( args.whitening, whiten_fn)) torch.save(Lw, whiten_fn) else: Lw = None # evaluate on test datasets datasets = args.datasets.split(',') for dataset in datasets: start = time.time() for whiten_type, ms, msp in zip(["ss", "ms"], [ms_singlescale, ms_multiscale], [msp_singlescale, msp_multiscale]): print('>> Extracting feature on {0}, whitening {1}'.format( dataset, whiten_type)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])] bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp) print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp) print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) compute_map_and_print(dataset, ranks, cfg['gnd']) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw[whiten_type]['m'], Lw[whiten_type]['P']) qvecs_lw = whitenapply(qvecs, Lw[whiten_type]['m'], Lw[whiten_type]['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) compute_map_and_print( dataset + ' + whiten {}'.format(whiten_type), ranks, cfg['gnd']) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start)))
def main(): args = parser.parse_args() # check if there are unknown datasets for dataset in args.datasets.split(','): if dataset not in datasets_names: raise ValueError( 'Unsupported or unknown dataset: {}!'.format(dataset)) # check if test dataset are downloaded # and download if they are not # download_train(get_data_root()) # download_test(get_data_root()) # setting up the visible GPU os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id # loading network from path if args.network_path is not None: print(">> Loading network:\n>>>> '{}'".format(args.network_path)) if args.network_path in PRETRAINED: # pretrained networks (downloaded automatically) state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks')) else: # fine-tuned network from path state = torch.load(args.network_path) # parsing net params from meta # architecture, pooling, mean, std required # the rest has default values, in case that is doesnt exist net_params = {} net_params['architecture'] = state['meta']['architecture'] net_params['pooling'] = state['meta']['pooling'] net_params['local_whitening'] = state['meta'].get( 'local_whitening', False) net_params['regional'] = state['meta'].get('regional', False) net_params['whitening'] = state['meta'].get('whitening', False) net_params['mean'] = state['meta']['mean'] net_params['std'] = state['meta']['std'] net_params['pretrained'] = False # load network net = init_network(net_params) net.load_state_dict(state['state_dict']) # if whitening is precomputed if 'Lw' in state['meta']: net.meta['Lw'] = state['meta']['Lw'] print(">>>> loaded network: ") print(net.meta_repr()) # loading offtheshelf network elif args.network_offtheshelf is not None: # parse off-the-shelf parameters offtheshelf = args.network_offtheshelf.split('-') net_params = {} net_params['architecture'] = offtheshelf[0] net_params['pooling'] = offtheshelf[1] net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:] net_params['regional'] = 'reg' in offtheshelf[2:] net_params['whitening'] = 'whiten' in offtheshelf[2:] net_params['pretrained'] = True # load off-the-shelf network print(">> Loading off-the-shelf network:\n>>>> '{}'".format( args.network_offtheshelf)) net = init_network(net_params) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters ms = list(eval(args.multiscale)) if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[ 'regional'] and not net.meta['whitening']: msp = net.pool.p.item() print(">> Set-up multiscale:") print(">>>> ms: {}".format(ms)) print(">>>> msp: {}".format(msp)) else: msp = 1 # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # evaluate on test datasets datasets = args.datasets.split(',') for dataset in datasets: start = time.time() print('>> {}: Extracting...'.format(dataset)) print('>> Prepare data information...') index_file_path = os.path.join(get_data_root(), 'index.csv') index_mark_path = os.path.join(get_data_root(), 'index_mark.csv') index_miss_path = os.path.join(get_data_root(), 'index_miss.csv') test_file_path = os.path.join(get_data_root(), 'test.csv') test_mark_path = os.path.join(get_data_root(), 'test_mark.csv') test_mark_add_path = os.path.join(get_data_root(), 'test_mark_add.csv') test_miss_path = os.path.join(get_data_root(), 'test_miss.csv') if dataset == 'google-landmarks-dataset': index_img_path = os.path.join(get_data_root(), 'index') test_img_path = os.path.join(get_data_root(), 'google-landmarks-dataset-test') elif dataset == 'google-landmarks-dataset-resize': index_img_path = os.path.join(get_data_root(), 'resize_index_image') test_img_path = os.path.join(get_data_root(), 'resize_test_image') if not (os.path.isfile(index_mark_path) or os.path.isfile(index_miss_path)): clear_no_exist(index_file_path, index_mark_path, index_miss_path, index_img_path) if not (os.path.isfile(test_mark_path) or os.path.isfile(test_miss_path)): clear_no_exist(test_file_path, test_mark_path, test_miss_path, test_img_path) print('>> load index image path...') retrieval_other_dataset = '/home/iap205/Datasets/google-landmarks-dataset-resize' csvfile = open(index_mark_path, 'r') csvreader = csv.reader(csvfile) images = [] miss, add = 0, 0 for line in csvreader: if line[0] == '1': images.append(os.path.join(index_img_path, line[1] + '.jpg')) elif line[0] == '0': retrieval_img_path = os.path.join(retrieval_other_dataset, 'resize_index_image', line[1] + '.jpg') if os.path.isfile(retrieval_img_path): images.append(retrieval_img_path) add += 1 miss += 1 csvfile.close() print( '>>>> index image miss: {}, supplement: {}, still miss: {}'.format( miss, add, miss - add)) print('>> load query image path...') csvfile = open(test_mark_path, 'r') csvreader = csv.reader(csvfile) savefile = open(test_mark_add_path, 'w') save_writer = csv.writer(savefile) qimages = [] miss, add = 0, 0 for line in csvreader: if line[0] == '1': qimages.append(os.path.join(test_img_path, line[1] + '.jpg')) save_writer.writerow(line) elif line[0] == '0': retrieval_img_path = os.path.join(retrieval_other_dataset, 'resize_test_image', line[1] + '.jpg') if os.path.isfile(retrieval_img_path): qimages.append(retrieval_img_path) save_writer.writerow(['1', line[1]]) add += 1 else: save_writer.writerow(line) miss += 1 csvfile.close() savefile.close() print( '>>>> test image miss: {}, supplement: {}, still miss: {}'.format( miss, add, miss - add)) # extract index vectors print('>> {}: index images...'.format(dataset)) split_num = 6 extract_num = int(len(images) / split_num) num_list = list(range(0, len(images), extract_num)) num_list.append(len(images)) # k = 0 # print('>>>> extract part {} of {}'.format(k, split_num-1)) # vecs = extract_vectors(net, images[num_list[k]:num_list[k+1]], args.image_size, transform, ms=ms, msp=msp) # vecs = vecs.numpy() # print('>>>> save index vecs to pkl...') # vecs_file_path = os.path.join(get_data_root(), 'index_vecs{}_of_{}.pkl'.format(k+1, split_num)) # vecs_file = open(vecs_file_path, 'wb') # pickle.dump(vecs[:, num_list[k]:num_list[k+1]], vecs_file) # vecs_file.close() # print('>>>> index_vecs{}_of_{}.pkl save done...'.format(k+1, split_num)) for i in range(split_num): # vecs_temp = np.loadtxt(open(os.path.join(get_data_root(), 'index_vecs{}_of_{}.csv'.format(i+1, split_num)), "rb"), # delimiter=",", skiprows=0) with open( os.path.join( get_data_root(), 'index_vecs{}_of_{}.pkl'.format(i + 1, split_num)), 'rb') as f: vecs_temp = pickle.load(f) if i == 0: vecs = vecs_temp else: vecs = np.hstack((vecs, vecs_temp[:, :])) del vecs_temp gc.collect() print('\r>>>> index_vecs{}_of_{}.pkl load done...'.format( i + 1, split_num), end='') print('') # extract query vectors print('>> {}: query images...'.format(dataset)) split_num = 1 extract_num = int(len(qimages) / split_num) num_list = list(range(0, len(qimages), extract_num)) num_list.append(len(qimages)) # k = 0 # print('>>>> extract part {} of {}'.format(k, split_num - 1)) # qvecs = extract_vectors(net, qimages[num_list[k]:num_list[k + 1]], args.image_size, transform, ms=ms, msp=msp) # qvecs = qvecs.numpy() # for i in range(split_num): # qvecs_file_path = os.path.join(get_data_root(), 'test_vecs{}_of_{}.pkl'.format(i+1, split_num)) # qvecs_file = open(qvecs_file_path, 'wb') # pickle.dump(qvecs[:, num_list[i]:num_list[i+1]], qvecs_file) # qvecs_file.close() # print('\r>>>> test_vecs{}_of_{}.pkl save done...'.format(i+1, split_num), end='') # print('') for i in range(split_num): # qvecs_temp = np.loadtxt(open(os.path.join(get_data_root(), 'test_vecs{}_of_{}.csv'.format(i+1, split_num)), "rb"), # delimiter=",", skiprows=0) with open( os.path.join( get_data_root(), 'test_vecs{}_of_{}.pkl'.format(i + 1, split_num)), 'rb') as f: qvecs_temp = pickle.load(f) if i == 0: qvecs = qvecs_temp else: qvecs = np.hstack((qvecs, qvecs_temp[:, :])) del qvecs_temp gc.collect() print('\r>>>> test_vecs{}_of_{}.pkl load done...'.format( i + 1, split_num), end='') print('') # vecs = np.zeros((2048, 1093278)) # qvecs = np.zeros((2048, 115921)) # save vecs to csv file # np.savetxt(os.path.join(get_data_root(), 'index_vecs{}_of_{}.csv'.format(k, split_num-1)), vecs, delimiter=',') # np.savetxt(os.path.join(get_data_root(), 'test_vecs{}_of_{}.csv'.format(k, split_num-1)), qvecs, delimiter=',') # compute whitening if args.whitening is not None: start = time.time() if 'Lw' in net.meta and args.whitening in net.meta['Lw']: print('>> {}: Whitening is precomputed, loading it...'.format( args.whitening)) if len(ms) > 1: Lw = net.meta['Lw'][args.whitening]['ms'] else: Lw = net.meta['Lw'][args.whitening]['ss'] else: # if we evaluate networks from path we should save/load whitening # not to compute it every time if args.network_path is not None: whiten_fn = args.network_path + '_{}_whiten'.format( args.whitening) if len(ms) > 1: whiten_fn += '_ms' whiten_fn += '.pth' else: whiten_fn = None if whiten_fn is not None and os.path.isfile(whiten_fn): print('>> {}: Whitening is precomputed, loading it...'. format(args.whitening)) Lw = torch.load(whiten_fn) else: print('>> {}: Learning whitening...'.format( args.whitening)) # extract whitening vectors print('>> {}: Extracting...'.format(args.whitening)) wvecs = vecs # learning whitening print('>> {}: Learning...'.format(args.whitening)) m, P = pcawhitenlearn(wvecs) Lw = {'m': m, 'P': P} # saving whitening if whiten_fn exists if whiten_fn is not None: print('>> {}: Saving to {}...'.format( args.whitening, whiten_fn)) torch.save(Lw, whiten_fn) print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time() - start))) else: Lw = None print('>> apply PCAwhiten...') if Lw is not None: # whiten the vectors and shrinkage vecs = whitenapply(vecs, Lw['m'], Lw['P']) qvecs = whitenapply(qvecs, Lw['m'], Lw['P']) print('>>>> save index PCAwhiten vecs to pkl...') split_num = 6 extract_num = int(len(images) / split_num) num_list = list(range(0, len(images), extract_num)) num_list.append(len(images)) for i in range(split_num): vecs_file_path = os.path.join( get_data_root(), 'index_PCAwhiten_vecs{}_of_{}.pkl'.format(i + 1, split_num)) vecs_file = open(vecs_file_path, 'wb') pickle.dump(vecs[:, num_list[i]:num_list[i + 1]], vecs_file) vecs_file.close() print( '\r>>>> index_PCAwhiten_vecs{}_of_{}.pkl save done...'.format( i + 1, split_num), end='') print('') print('>>>> save test PCAwhiten vecs to pkl...') split_num = 1 extract_num = int(len(qimages) / split_num) num_list = list(range(0, len(qimages), extract_num)) num_list.append(len(images)) for i in range(split_num): qvecs_file_path = os.path.join( get_data_root(), 'test_PCAwhiten_vecs{}_of_{}.pkl'.format(i + 1, split_num)) qvecs_file = open(qvecs_file_path, 'wb') pickle.dump(qvecs[:, num_list[i]:num_list[i + 1]], qvecs_file) qvecs_file.close() print('\r>>>> test_PCAwhiten_vecs{}_of_{}.pkl save done...'.format( i + 1, split_num), end='') print('') print('>>>> load index PCAwhiten vecs from pkl...') for i in range(split_num): with open( os.path.join( get_data_root(), 'index_PCAwhiten_vecs{}_of_{}.pkl'.format( i + 1, split_num)), 'rb') as f: vecs_temp = pickle.load(f) if i == 0: vecs = vecs_temp else: vecs = np.hstack((vecs, vecs_temp[:, :])) del vecs_temp gc.collect() print( '\r>>>> index_PCAwhiten_vecs{}_of_{}.pkl load done...'.format( i + 1, split_num), end='') print('') print('>>>> load test PCAwhiten vecs from pkl...') for i in range(split_num): with open( os.path.join( get_data_root(), 'test_PCAwhiten_vecs{}_of_{}.pkl'.format( i + 1, split_num)), 'rb') as f: qvecs_temp = pickle.load(f) if i == 0: qvecs = qvecs_temp else: qvecs = np.hstack((qvecs, qvecs_temp[:, :])) del qvecs_temp gc.collect() print('\r>>>> test_PCAwhiten_vecs{}_of_{}.pkl load done...'.format( i + 1, split_num), end='') print('') # extract principal components and dimension shrinkage ratio = 0.8 vecs = vecs[:int(vecs.shape[0] * ratio), :] qvecs = vecs[:int(qvecs.shape[0] * ratio), :] print('>> {}: Evaluating...'.format(dataset)) split_num = 50 top_num = 100 vecs_T = np.zeros((vecs.shape[1], vecs.shape[0])).astype('float32') vecs_T[:] = vecs.T[:] QE_iter = 0 QE_weight = (np.arange(top_num, 0, -1) / top_num).reshape( 1, top_num, 1) print('>> find {} nearest neighbour...'.format(top_num)) import faiss # place it in the file top will cause network load so slowly # ranks_top_100 = np.loadtxt(open(os.path.join(get_data_root(), 'ranks_top_{}.csv'.format(top_num)), "rb"), # delimiter=",", skiprows=0).astype('int') for iter in range(0, QE_iter + 1): if iter != 0: # ranks_top_100 = np.ones((100, 115921)).astype('int') print('>> Query expansion iteration {}'.format(iter)) ranks_split = 50 for i in range(ranks_split): ranks_top_100_split = ranks_top_100[:, int(ranks_top_100. shape[1] / ranks_split * i ):int(ranks_top_100 .shape[1] / ranks_split * (i + 1))] top_100_vecs = vecs[:, ranks_top_100_split] # (2048, 100, query_split_size) qvecs_temp = (top_100_vecs * QE_weight).sum(axis=1) qvecs_temp = qvecs_temp / (np.linalg.norm( qvecs_temp, ord=2, axis=0, keepdims=True) + 1e-6) if i == 0: qvecs = qvecs_temp else: qvecs = np.hstack((qvecs, qvecs_temp)) del ranks_top_100_split, top_100_vecs, qvecs_temp gc.collect() print('\r>>>> calculate new query vectors {}/{} done...'. format(i + 1, ranks_split), end='') print('') qe_iter_qvecs_path = os.path.join( get_data_root(), 'QE_iter{}_qvecs.pkl'.format(iter)) qe_iter_qvecs_file = open(qe_iter_qvecs_path, 'wb') pickle.dump(qvecs, qe_iter_qvecs_file) qe_iter_qvecs_file.close() print('>>>> QE_iter{}_qvecs.pkl save done...'.format(iter)) del ranks_top_100 gc.collect() for i in range(split_num): # scores = np.dot(vecs.T, qvecs[:, int(qvecs.shape[1]/split_num*i):int(qvecs.shape[1]/split_num*(i+1))]) # ranks = np.argsort(-scores, axis=0) # kNN search k = top_num index = faiss.IndexFlatL2(vecs.shape[0]) index.add(vecs_T) query_vecs = qvecs[:, int(qvecs.shape[1] / split_num * i):int(qvecs.shape[1] / split_num * (i + 1))] qvecs_T = np.zeros((query_vecs.shape[1], query_vecs.shape[0])).astype('float32') qvecs_T[:] = query_vecs.T[:] _, ranks = index.search(qvecs_T, k) ranks = ranks.T if i == 0: ranks_top_100 = ranks[:top_num, :] else: ranks_top_100 = np.hstack( (ranks_top_100, ranks[:top_num, :])) # del scores, ranks del index, query_vecs, qvecs_T, ranks gc.collect() print('\r>>>> kNN search {} nearest neighbour {}/{} done...'. format(top_num, i + 1, split_num), end='') del qvecs gc.collect() print('') del vecs, vecs_T gc.collect() # save to csv file print(">> save to submission.csv file...") submission_file = open(os.path.join(get_data_root(), 'submission.csv'), 'w') writer = csv.writer(submission_file) test_mark_file = open(test_mark_add_path, 'r') csvreader = csv.reader(test_mark_file) cnt = 0 writer.writerow(['id', 'images']) for index, line in enumerate(csvreader): (flag, img_name) = line[:2] if flag == '1': select = [] for i in range(top_num): select.append(images[int( ranks_top_100[i, cnt])].split('/')[-1].split('.jpg')[0]) cnt += 1 writer.writerow([ img_name.split('/')[-1].split('.jpg')[0], ' '.join(select) ]) else: # random_list = random.sample(range(0, len(images)), top_num) random_list = np.random.choice(len(images), top_num, replace=False) select = [] for i in range(top_num): select.append( images[random_list[i]].split('/')[-1].split('.jpg')[0]) writer.writerow([ img_name.split('/')[-1].split('.jpg')[0], ' '.join(select) ]) if cnt % 10 == 0 or cnt == len(qimages): print('\r>>>> {}/{} done...'.format(cnt, len(qimages)), end='') submission_file.close() test_mark_file.close() print('') print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start)))
def call_benchmark( # must pass one of images or paths images=None, paths=None, **kwargs, ): """Run the given network on the given data and return vectors for it.""" # load params params = default_params.copy() params.update(kwargs) network = params["network"] offtheshelf = params["offtheshelf"] image_size = params["image_size"] gpu = params["gpu"] multiscale = params["multiscale"] whitening = params["whitening"] net_key = (network, offtheshelf, gpu) if net_key in LOADED_NETWORKS: net = LOADED_NETWORKS[net_key] else: # load network if offtheshelf: net = load_offtheshelf(network) else: net = load_network(network) # moving network to gpu and eval mode if gpu: net.cuda() net.eval() # store network in memo dict LOADED_NETWORKS[net_key] = net # setting up the multi-scale parameters ms = [1] msp = 1 if multiscale: ms = [1, 1 / np.sqrt(2), 1 / 2] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # set up the transform normalize = transforms.Normalize( mean=net.meta['mean'], std=net.meta['std'], ) transform = transforms.Compose([ transforms.ToTensor(), normalize, ]) # setting up whitening if whitening is not None: if 'Lw' in net.meta and whitening in net.meta['Lw']: if multiscale: Lw = net.meta['Lw'][whitening]['ms'] else: Lw = net.meta['Lw'][whitening]['ss'] elif whitening == "scores": whiten_key = (network, offtheshelf, image_size, multiscale) Lw = get_scores_whitening(whiten_key, net, transform, ms, msp, image_size, setup_network=False, gpu=gpu) else: raise ValueError( "invalid whitening {} (valid whitenings: {})".format( whitening, list(net.meta['Lw'].keys()))) # process the given data if images is not None: images = np.asarray(images) print("images.shape =", images.shape) vecs = vectors_from_images(net, images, transform, ms=ms, msp=msp, setup_network=False, gpu=gpu) else: vecs = extract_vectors(net, paths, image_size, transform, ms=ms, msp=msp, setup_network=False, gpu=gpu) # convert to numpy vecs = vecs.numpy() # apply whitening if whitening is not None: vecs = whitenapply(vecs, Lw['m'], Lw['P']) # take transpose vecs = vecs.T print("vecs.shape =", vecs.shape) return vecs
Lw = {'m': m, 'P': P} # saving whitening if whiten_fn exists if whiten_fn is not None: print('>> {}: Saving to {}...'.format( whitening, whiten_fn)) # torch.save(Lw, whiten_fn) print('>> {}: elapsed time: {}'.format(whitening, htime(time.time() - start))) else: Lw = None if Lw is not None: for dim in param['pac_dims']: print('>>>> pac_dim: {}'.format(dim)) # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'], dim) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'], dim) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks_lw = np.argsort(-scores, axis=0) qvecs_lw_orig = qvecs_lw ranks_lw_orig = ranks_lw mismatched_info = compute_map_and_print(dataset + ' + whiten', ranks_lw, cfg['gnd'], kappas=[1, 5, 10, 100]) # %% # save vecs, qvecs, ranks to pickle import pickle
def testOxfordParisHolidays(net, eConfig): #datasets = eConfig['test-datasets'].split(',') #results = [] # #for dataset in datasets: # results.append((dataset, np.random.rand(1)[0])) # #return results print('>> Evaluating network on test datasets...') # for testing we use image size of max 1024 image_size = 1024 # setting up the multi-scale parameters ms = [1] msp = 1 if (eConfig['multiscale']): ms = [1, 1. / math.sqrt(2), 1. / 2] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if eConfig['whitening']: start = time.time() print('>> {}: Learning whitening...'.format(eConfig['test-whiten'])) # loading db db_root = os.path.join(get_data_root(), 'train', eConfig['test-whiten']) ims_root = os.path.join(db_root, 'ims') db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(eConfig['test-whiten'])) with open(db_fn, 'rb') as f: db = pickle.load(f) images = [ cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids'])) ] # extract whitening vectors print('>> {}: Extracting...'.format(eConfig['test-whiten'])) wvecs = extract_vectors(net, images, image_size, transform, ms=ms, msp=msp) # learning whitening print('>> {}: Learning...'.format(eConfig['test-whiten'])) wvecs = wvecs.numpy() m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs']) Lw = {'m': m, 'P': P} print('>> {}: elapsed time: {}'.format(eConfig['test-whiten'], htime(time.time() - start))) else: Lw = None # evaluate on test datasets datasets = eConfig['test-datasets'].split(',') results = [] for dataset in datasets: start = time.time() if (dataset != 'holidays' and dataset != 'rholidays'): print('>> {}: Extracting...'.format(dataset)) # prepare config structure for the test dataset cfg = configdataset(dataset, os.path.join(get_data_root(), 'test')) images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])] qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])] bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] if (dataset == 'oxford105k' or dataset == 'paris106k'): images.extend(cfg['distractors']) # extract database and query vectors print('>> {}: database images...'.format(dataset)) vecs = extract_vectors(net, images, image_size, transform, ms=ms, msp=msp) print('>> {}: query images...'.format(dataset)) qvecs = extract_vectors(net, qimages, image_size, transform, bbxs, ms=ms, msp=msp) print('>> {}: Evaluating...'.format(dataset)) # convert to numpy vecs = vecs.numpy() qvecs = qvecs.numpy() # search, rank, and print scores = np.dot(vecs.T, qvecs) ranks = np.argsort(-scores, axis=0) results.append( compute_map_and_print( dataset + ('+ multiscale' if eConfig['multiscale'] else ''), ranks, cfg['gnd'])) if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P']) # search, rank, and print scores = np.dot(vecs_lw.T, qvecs_lw) ranks = np.argsort(-scores, axis=0) results.append( compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])) else: results.append(testHolidays(net, eConfig, dataset, Lw)) print('>> {}: elapsed time: {}'.format(dataset, htime(time.time() - start))) return results
def embed(params, data): net = params.pop("net") imgdir = params.pop("imgdir") whitening = params.pop("whitening", None) whitening_dir = params.pop("whitening_dir", None) image_size = params.pop("image_size", 1024) multiscale = params.pop("multiscale", True) assert not params, params.keys() input_images, bbxs = (data[0], None) if len(data) == 1 else data impaths = [path_join(imgdir, x) for x in input_images] if not data[0]: return ({ "status": "skipped" }, [], []) + (([], ) if whitening_dir else tuple()) # Handle paths assert os.path.exists(net), net # loading network from path print(">> Loading network:\n>>>> '{}'".format(net)) state = torch.load(net) net = init_network({ 'architecture': state['meta']['architecture'], 'pooling': state['meta']['pooling'], 'whitening': state['meta']['whitening'], 'mean': state['meta']['mean'], 'std': state['meta']['std'], 'pretrained': False }) net.load_state_dict(state['state_dict']) print(">>>> loaded network: ") print(net.meta_repr()) # setting up the multi-scale parameters ms = multiscale if not isinstance( multiscale, bool) else [1, 1. / math.sqrt(2), 1. / 2] if multiscale else [1] if net.meta['pooling'] == 'gem' and net.whiten is None: msp = net.pool.p.data.tolist()[0] # moving network to gpu and eval mode net.cuda() net.eval() # set up the transform normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std']) transform = transforms.Compose([transforms.ToTensor(), normalize]) # compute whitening if whitening_dir: whitening_dir = os.path.join( whitening_dir, "%s_%s_%s_%s.lw.pkl" % (whitening, None, image_size, multiscale)) print('>> {}: Loading whitening...'.format(whitening)) with open(whitening_dir, "rb") as handle: Lw = pickle.load(handle) # elif whitening: # Lw, _ = _compute_whitening(whitening, net, image_size, transform, ms, msp) else: Lw = None # extract database and query vectors print('>> Images descriptors...') vecs = extract_vectors(net, impaths, image_size, transform, bbxs=bbxs, ms=ms, msp=msp) print('>> Evaluating...') # convert to numpy vecs = vecs.numpy() if Lw is not None: # whiten the vectors vecs_lw = whitenapply(vecs, Lw['m'], Lw['P']) return {}, input_images, vecs.T, vecs_lw.T return {}, input_images, vecs.T