def test(datasets, net):
print(">> Evaluating network on test datasets...")
image_size = 1024
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta["mean"],
std=net.meta["std"])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
# Lw = None
Lw = net.meta["Lw"]["retrieval-SfM-120k"]["ss"]
# evaluate on test datasets
# datasets = args.test_datasets.split(",")
for dataset in datasets:
start = time.time()
print(">> {}: Extracting...".format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), "test"))
images = [cfg["im_fname"](cfg, i) for i in range(cfg["n"])]
qimages = [cfg["qim_fname"](cfg, i) for i in range(cfg["nq"])]
bbxs = [tuple(cfg["gnd"][i]["bbx"]) for i in range(cfg["nq"])]
# extract database and query vectors
print(">> {}: database images...".format(dataset))
vecs = extract_vectors(net, images, image_size, transform)
print(">> {}: query images...".format(dataset))
qvecs = extract_vectors(net, qimages, image_size, transform, bbxs)
print(">> {}: Evaluating...".format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg["gnd"])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw["m"], Lw["P"])
qvecs_lw = whitenapply(qvecs, Lw["m"], Lw["P"])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + " + whiten", ranks, cfg["gnd"])
print(">> {}: elapsed time: {}".format(dataset,
htime(time.time() - start)))
def cal_ranks(vecs, qvecs, Lw):
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
return scores, ranks
def whiten(params, data):
"""Apply pre-computed whitening"""
dimensions = params.pop("dimensions", None) or None
assert not params, params.keys()
whitening, names, values = data
assert len(names) == len(values)
resources = stats.ResourceUsage()
time0 = time.time()
whitened = whitenapply(values.T, whitening['m'], whitening['P'],
dimensions)
timing = time.time() - time0
metadata = {
"timings": {
"whitening_apply": round(timing, 2)
},
"resource_usage": resources.take_current_stats().get_resources()
}
return metadata, names, whitened.T
def test(datasets, net, wandb_enabled=False, epoch=-1):
global global_step
print('>> Evaluating network on test datasets...')
# for testing we use image size of max 1024
image_size = 1024
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.test_whiten:
start = time.time()
print('>> {}: Learning whitening...'.format(args.test_whiten))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.test_whiten)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.test_whiten))
wvecs = extract_vectors(net, images, image_size, transform) # implemented with torch.no_grad
# learning whitening
print('>> {}: Learning...'.format(args.test_whiten))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.test_whiten, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.test_datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, image_size, transform) # implemented with torch.no_grad
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, image_size, transform, bbxs) # implemented with torch.no_grad
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'], wandb_enabled=wandb_enabled, epoch=epoch, global_step=global_step)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'], wandb_enabled=wandb_enabled, epoch=epoch, global_step=global_step)
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main():
args = parser.parse_args()
# check if test dataset are downloaded
# and download if they are not
#download_train(get_data_root())
#download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
state = torch.load(args.network_path)
net = init_network(model=state['meta']['architecture'],
pooling=state['meta']['pooling'],
whitening=state['meta']['whitening'],
mean=state['meta']['mean'],
std=state['meta']['std'],
pretrained=False)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
offtheshelf = args.network_offtheshelf.split('-')
if len(offtheshelf) == 3:
if offtheshelf[2] == 'whiten':
offtheshelf_whiten = True
else:
raise (RuntimeError(
"Incorrect format of the off-the-shelf network. Examples: resnet101-gem | resnet101-gem-whiten"
))
else:
offtheshelf_whiten = False
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(model=offtheshelf[0],
pooling=offtheshelf[1],
whitening=offtheshelf_whiten)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = [1]
msp = 1
if args.multiscale:
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if args.multiscale:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
state = torch.load(args.network_path)
net = init_network(model=state['meta']['architecture'],
pooling=state['meta']['pooling'],
whitening=state['meta']['whitening'],
mean=state['meta']['mean'],
std=state['meta']['std'],
pretrained=False)
net.load_state_dict(state['state_dict'])
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
offtheshelf = args.network_offtheshelf.split('-')
if len(offtheshelf) == 3:
if offtheshelf[2] == 'whiten':
offtheshelf_whiten = True
else:
raise (RuntimeError(
"Incorrect format of the off-the-shelf network. Examples: resnet101-gem | resnet101-gem-whiten"
))
else:
offtheshelf_whiten = False
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(model=offtheshelf[0],
pooling=offtheshelf[1],
whitening=offtheshelf_whiten)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = [1]
msp = 1
if args.multiscale:
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
if dataset == 'reco':
images, qimages = landmark_recognition_dataset()
bbxs = [None for x in qimages]
elif dataset == 'retr':
images, _ = landmark_retrieval_dataset()
qimages = []
bbxs = [None for x in qimages]
else:
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
with open('%s_fnames.pkl' % dataset, 'wb') as f:
pickle.dump([images, qimages], f)
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
vecs = vecs.numpy()
print('>> saving')
np.save('{}_vecs.npy'.format(dataset), vecs)
if len(qimages) > 0:
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
qvecs = qvecs.numpy()
np.save('{}_qvecs.npy'.format(dataset), qvecs)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# TODO
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError('Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get('local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms)>1 and net.meta['pooling'] == 'gem' and not net.meta['regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
if len(ms)>1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
try:
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
except:
bbxs = None # for holidaysmanrot and copydays
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp)
torch.save(vecs, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/data_vect.pt')
torch.save(qvecs, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/query_vect.pt')
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
torch.save(vecs, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/data_lw_vect.pt')
torch.save(qvecs_lw, '/content/drive/My Drive/Image_retrieval/cnnimageretrieval-pytorch/data/query_lw_vect.pt')
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def testHolidays(net, eConfig, dataset, Lw):
print('>> Evaluating network on test dataset: {}'.format(dataset))
# for testing we use image size of max 1024
image_size = 1024
ms = [1]
msp = 1
if (eConfig['multiscale']):
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# read the images and generate paths and queries-positive indexes
dbpath = os.path.join(get_data_root(), 'test', 'holidays')
ext = 'jpg' if dataset == 'holidays' else 'rjpg'
images = sorted(os.listdir(os.path.join(dbpath, ext)))
with open(os.path.join(dbpath, 'straight_gnd_holidays.pkl'), 'rb') as f:
queries = pickle.load(f)
positives = pickle.load(f)
qidx = []
pidx = []
for i in range(len(queries)):
qidx.append(images.index(queries[i]))
aux = []
for j in range(len(positives[i])):
aux.append(images.index(positives[i][j]))
pidx.append(aux)
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
X = extract_vectors(net, [os.path.join(dbpath, ext, n) for n in images],
image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# rank the similarities
X = X.numpy()
if (Lw is not None):
X = whitenapply(X, Lw['m'], Lw['P'])
scores = np.dot(X.T, X)
ranks = np.argsort(-scores, axis=1)
ranks = ranks[qidx, 1::]
APs = []
for i, r in enumerate(ranks):
trueRanks = np.isin(r, pidx[i])
trueRanks = np.where(trueRanks == True)[0]
APs.append(compute_ap(trueRanks, len(pidx[i])))
mAP = np.mean(APs)
print(">> {}: mAP {:.2f}".format(dataset, mAP * 100))
# return the average mAP
return (dataset + ('+ multiscale' if eConfig['multiscale'] else ''), mAP)
def main():
args = parser.parse_args()
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
net = load_network(args.network_path)
# loading offtheshelf network
elif args.network_offtheshelf is not None:
net = load_offtheshelf(args.network_offtheshelf)
# setting up the multi-scale parameters
ms = [1]
msp = 1
if args.multiscale:
ms = [1, 1./math.sqrt(2), 1./2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
if args.multiscale:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
print('>> {}: Learning whitening...'.format(args.whitening))
if args.whitening == "scores":
# special logic for scores database
from score_retrieval.exports import (
db,
train_images as images,
)
else:
# loading db
db_root = os.path.join(get_data_root(), 'train', args.test_whiten)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
if dataset == "scores":
# Special added logic to handle loading our score dataset
from score_retrieval.exports import (
images,
qimages,
gnd,
)
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, ms=ms, msp=msp)
else:
# extract ground truth
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
gnd = cfg['gnd']
# prepare config structure for the test dataset
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
bbxs = [tuple(gnd[i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp)
# validation
print(">> {}: gnd stats: {}, {}, {}".format(
dataset,
len(gnd),
[len(x["ok"]) for x in gnd[10:]],
[len(x["junk"]) for x in gnd[10:]],
))
print(">> {}: image stats: {}, {}".format(dataset, len(images), len(qimages)))
assert len(gnd) == len(qimages), (len(gnd), len(qimages))
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
print(">> {}: qvecs.shape: {}".format(dataset, qvecs.shape))
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
print(">> {}: ranks (shape {}) head: {}".format(dataset, ranks.shape, ranks[10:,10:]))
print(">> {}: gnd head: {}".format(dataset, gnd[5:]))
# Compute and print metrics
compute_acc(ranks, gnd, dataset)
compute_mrr(ranks, gnd, dataset)
compute_map_and_print(dataset, ranks, gnd)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_acc(ranks, gnd, dataset + " + whiten")
compute_mrr(ranks, gnd, dataset + " + whiten")
compute_map_and_print(dataset + " + whiten", ranks, gnd)
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main():
#def process(network_path, datasets='oxford5k,paris6k', whitening=None, image_size=1024, multiscale = '[1]', query=None):
args = parser.parse_args()
#args.query = None
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError(
'Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
#download_train(get_data_root())
#download_test(get_data_root())
# setting up the visible GPU
#os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get(
'local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
#net.cuda()
#net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if len(ms) > 1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(
args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
print(whiten_fn)
return
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train',
args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
# query type
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
#for i in cfg: print(i)
#print(cfg['gnd'][0]['bbx'])
#return
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
feas_dir = os.path.join(cfg['dir_data'], 'features')
if not os.path.isdir(feas_dir):
os.mkdir(feas_dir)
feas_sv = os.path.join(
feas_dir, dataset + '_' + args.network_path + '_features.pkl')
if not os.path.isfile(feas_sv):
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
with open(feas_sv, 'wb') as f:
pickle.dump(vecs, f)
else:
with open(feas_sv, 'rb') as f:
vecs = pickle.load(f)
print('>> {}: query images...'.format(dataset))
if args.query is not None:
qimages = [args.query]
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
ms=ms,
msp=msp)
else:
qfeas_dir = feas_dir
qfeas_sv = os.path.join(
qfeas_dir,
dataset + '_' + args.network_path + '_qfeatures.pkl')
if not os.path.isfile(qfeas_sv):
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
try:
bbxs = [
tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])
]
except:
bbxs = None
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
with open(qfeas_sv, 'wb') as f:
pickle.dump(qvecs, f)
else:
with open(qfeas_sv, 'rb') as f:
qvecs = pickle.load(f)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
#qvecs = qvecs[:, 0].reshape(-1, 1)
#args.query = True
# search, rank, and print
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranksw = np.argsort(-scores, axis=0)
if args.query is None:
#compute_map_and_print(dataset + ' + whiten', ranksw, cfg['gnd'])
compute_map_and_print1(dataset + ' + whiten', ranksw,
cfg['gnd'])
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
# compute_map_and_print(dataset, ranks, cfg['gnd'])
compute_map_and_print1(dataset, ranks, cfg['gnd'])
else:
a = []
for i in ranksw:
a.append(
os.path.join(cfg['dir_images'], cfg['imlist'][i[0]]) +
cfg['ext'])
print(a[:10])
result = cfg['dir_data'] + '_result'
with open(result + '.pkl', 'wb') as f:
pickle.dump(a[:10], f)
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError('Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
# download_train(get_data_root())
# download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get('local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
net_params['multi_layer_cat'] = state['meta']['multi_layer_cat']
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
print(">> image size: {}".format(args.image_size))
ms = list(eval(args.multiscale))
if len(ms)>1 and net.meta['pooling'] == 'gem' and not net.meta['regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
if len(ms)>1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
# bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
print('>> not use bbxs...')
bbxs = None
# key_url_list = ParseData(os.path.join(get_data_root(), 'index.csv'))
# index_image_path = os.path.join(get_data_root(), 'resize_index_image')
# images = [os.path.join(index_image_path, key_url_list[i][0]) for i in range(len(key_url_list))]
# key_url_list = ParseData(os.path.join(get_data_root(), 'test.csv'))
# test_image_path = os.path.join(get_data_root(), 'resize_test_image')
# qimages = [os.path.join(test_image_path, key_url_list[i][0]) for i in range(len(key_url_list))]
# # bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# csvfile = open(os.path.join(get_data_root(), 'index_clear.csv'), 'r')
# csvreader = csv.reader(csvfile)
# images = [line[:1][0] for line in csvreader]
#
# csvfile = open(os.path.join(get_data_root(), 'test_clear.csv'), 'r')
# csvreader = csv.reader(csvfile)
# qimages = [line[:1][0] for line in csvreader]
# bbxs = None
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# vecs = torch.randn(2048, 5063)
# vecs = torch.randn(2048, 4993)
# hxq modified
# bbxs = None
# print('>> set no bbxs...')
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
# hxq modified, test add features map for retrieval
# vecs = [vecs[i].numpy() for i in range(len(vecs))]
# qvecs_temp = np.zeros((qvecs[0].shape[0], len(qvecs)))
# for i in range(len(qvecs)):
# qvecs_temp[:, i] = qvecs[i][:, 0].numpy()
# qvecs = qvecs_temp
#
# scores = np.zeros((len(vecs), qvecs.shape[-1]))
# for i in range(len(vecs)):
# scores[i, :] = np.amax(np.dot(vecs[i].T, qvecs), 0)
ranks = np.argsort(-scores, axis=0)
mismatched_info = compute_map_and_print(dataset, ranks, cfg['gnd'], kappas=[1, 5, 10, 100])
# hxq added
show_false_img = False
if show_false_img == True:
print('>> Save mismatched image tuple...')
for info in mismatched_info:
mismatched_img_show_save(info, qimages, images, args, bbxs=bbxs)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
mismatched_info = compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
# hxq added
# show_false_img = False
if show_false_img == True:
print('>> Save mismatched image tuple...')
for info in mismatched_info:
mismatched_img_show_save(info, qimages, images, args, bbxs=bbxs)
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def test(datasets, net, noise, image_size):
global base
print(">> Evaluating network on test datasets...")
net.cuda()
net.eval()
normalize = transforms.Normalize(mean=net.meta["mean"],
std=net.meta["std"])
def add_noise(img):
n = noise
n = F.interpolate(n.unsqueeze(0),
mode=MODE,
size=tuple(img.shape[-2:]),
align_corners=True).squeeze()
return torch.clamp(img + n, 0, 1)
transform_base = transforms.Compose([transforms.ToTensor(), normalize])
transform_query = transforms.Compose(
[transforms.ToTensor(),
transforms.Lambda(add_noise), normalize])
if "Lw" in net.meta:
Lw = net.meta["Lw"]["retrieval-SfM-120k"]["ss"]
else:
Lw = None
# evaluate on test datasets
datasets = args.test_datasets.split(",")
attack_result = {}
for dataset in datasets:
start = time.time()
print(">> {}: Extracting...".format(dataset))
cfg = configdataset(dataset, os.path.join(get_data_root(), "test"))
images = [cfg["im_fname"](cfg, i) for i in range(cfg["n"])]
qimages = [cfg["qim_fname"](cfg, i) for i in range(cfg["nq"])]
bbxs = [tuple(cfg["gnd"][i]["bbx"]) for i in range(cfg["nq"])]
# extract database and query vectors
print(">> {}: database images...".format(dataset))
with torch.no_grad():
if dataset in base and str(image_size) in base[dataset]:
vecs = base[dataset][str(image_size)]
else:
vecs = extract_vectors(net, images, image_size, transform_base)
if dataset not in base:
base[dataset] = {}
base[dataset][str(image_size)] = vecs
fname = args.network_path.replace("/", "_") + ".pkl"
with open(f"base/{fname}", "wb") as f:
pickle.dump(base, f)
print(">> {}: query images...".format(dataset))
qvecs = extract_vectors(net, qimages, image_size, transform_query,
bbxs)
print(">> {}: Evaluating...".format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw["m"], Lw["P"])
qvecs_lw = whitenapply(qvecs, Lw["m"], Lw["P"])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
r = compute_map_and_print(dataset + " + whiten", ranks, cfg["gnd"])
attack_result[dataset] = r
print(">> {}: elapsed time: {}".format(dataset,
htime(time.time() - start)))
return inv_gfr(
attack_result,
baseline_result[net.meta["architecture"]][net.meta["pooling"]])
def main():
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
input_resol = 512
# resolution of input image, will resize to that if larger
# input_resol = 1024; # resolution of input image, will resize to that if larger
scales = [1, 1 / np.sqrt(2),
1 / 2] # re-scaling factors for multi-scale extraction
# sample image
img_file = 'sanjuan.jpg'
if not path.exists(img_file):
os.system(
'wget https://raw.githubusercontent.com/gtolias/tma/master/data/input/'
+ img_file)
img = default_loader(img_file)
print("use network trained with gem pooling and FC layer")
state = load_url(TRAINED['rSfM120k-tl-resnet101-gem-w'],
model_dir=os.path.join(get_data_root(), 'networks'))
net = init_network({
'architecture': state['meta']['architecture'],
'pooling': state['meta']['pooling'],
'whitening': state['meta'].get('whitening')
})
net.load_state_dict(state['state_dict'])
net.eval()
net.cuda()
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=state['meta']['mean'],
std=state['meta']['std'])
])
# single-scale extraction
vec = extract_ss(net,
transform(imresize(img, input_resol)).unsqueeze(0).cuda())
vec = vec.data.cpu().numpy()
print(vec)
# multi-scale extraction
vec = extract_ms(net,
transform(imresize(img, input_resol)).unsqueeze(0).cuda(),
ms=scales,
msp=1.0)
vec = vec.data.cpu().numpy()
print(vec)
print("\n")
print(
"use network trained with gem pooling, and apply the learned whitening transformation"
)
state = load_url(TRAINED['retrievalSfM120k-resnet101-gem'],
model_dir=os.path.join(get_data_root(), 'networks'))
net = init_network({
'architecture': state['meta']['architecture'],
'pooling': state['meta']['pooling'],
'whitening': state['meta'].get('whitening')
})
net.load_state_dict(state['state_dict'])
net.eval()
net.cuda()
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=state['meta']['mean'],
std=state['meta']['std'])
])
# single-scale extraction
vec = extract_ss(net,
transform(imresize(img, input_resol)).unsqueeze(0).cuda())
vec = vec.data.cpu().numpy()
print(vec)
whiten_ss = state['meta']['Lw']['retrieval-SfM-120k']['ss']
vec = whitenapply(vec.reshape(-1, 1), whiten_ss['m'],
whiten_ss['P']).reshape(-1)
print(vec)
# multi-scale extraction
vec = extract_ms(net,
transform(imresize(img, input_resol)).unsqueeze(0).cuda(),
ms=scales,
msp=net.pool.p.item())
vec = vec.data.cpu().numpy()
print(vec)
whiten_ms = state['meta']['Lw']['retrieval-SfM-120k']['ms']
vec = whitenapply(vec.reshape(-1, 1), whiten_ms['m'],
whiten_ms['P']).reshape(-1)
print(vec)
print("\n")
print("use pre-trained (on ImageNet) network with appended mac pooling")
net = init_network({
'architecture': 'resnet101',
'pooling': 'mac',
'pretrained': True
})
net.eval()
net.cuda()
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=net.meta['mean'], std=net.meta['std'])
])
# single-scale extraction
vec = extract_ss(net,
transform(imresize(img, input_resol)).unsqueeze(0).cuda())
vec = vec.data.cpu().numpy()
print(vec)
# multi-scale extraction
vec = extract_ms(net,
transform(imresize(img, input_resol)).unsqueeze(0).cuda(),
ms=scales,
msp=1.0)
vec = vec.data.cpu().numpy()
print(vec)
print("\n")
def test(datasets, net):
print('>> Evaluating network on test datasets...')
# for testing we use image size of max 1024
image_size = 1024
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.test_whiten:
start = time.time()
print('>> {}: Learning whitening...'.format(args.test_whiten))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.test_whiten)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.test_whiten))
wvecs = extract_vectors(net,
images,
image_size,
transform,
print_freq=10,
batchsize=20) # implemented with torch.no_grad
# learning whitening
print('>> {}: Learning...'.format(args.test_whiten))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.test_whiten,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.test_datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
if dataset == 'cdvs_test_retrieval':
bbxs = None
else:
bbxs = None
print('>> {}: database images...'.format(dataset))
if args.pool == 'gem':
ms = [1, 1 / 2**(1 / 2), 1 / 2]
else:
ms = [1]
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
vecs = extract_vectors(net,
images,
image_size,
transform,
bbxs,
ms=ms,
msp=msp,
print_freq=1000,
batchsize=1) # implemented with torch.no_grad
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
image_size,
transform,
bbxs,
ms=ms,
msp=msp,
print_freq=1000,
batchsize=1) # implemented with torch.no_grad
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
if dataset == 'cdvs_test_retrieval':
compute_map_and_print(dataset, ranks, cfg['gnd_id'])
else:
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get(
'local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
if "epoch" in state:
print("Model after {} epochs".format(state["epoch"]))
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters: test both single scale and multiscale
ms_singlescale = [1]
msp_singlescale = 1
ms_multiscale = list(eval(args.multiscale))
msp_multiscale = 1
if len(ms_multiscale
) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp_multiscale = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms_multiscale))
print(">>>> msp: {}".format(msp_multiscale))
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
Lw = net.meta['Lw'][args.whitening]
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(
args.whitening)
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
Lw = torch.load(whiten_fn)
else:
Lw = {}
for whiten_type, ms, msp in zip(
["ss", "ms"], [ms_singlescale, ms_multiscale],
[msp_singlescale, msp_multiscale]):
print('>> {0}: Learning whitening {1}...'.format(
args.whitening, whiten_type))
# loading db
db_root = os.path.join(get_data_root(), 'train',
args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(
db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw[whiten_type] = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(
args.whitening, htime(time.time() - start)))
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
for whiten_type, ms, msp in zip(["ss", "ms"],
[ms_singlescale, ms_multiscale],
[msp_singlescale, msp_multiscale]):
print('>> Extracting feature on {0}, whitening {1}'.format(
dataset, whiten_type))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw[whiten_type]['m'],
Lw[whiten_type]['P'])
qvecs_lw = whitenapply(qvecs, Lw[whiten_type]['m'],
Lw[whiten_type]['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(
dataset + ' + whiten {}'.format(whiten_type), ranks,
cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError(
'Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
# download_train(get_data_root())
# download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get(
'local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
print('>> Prepare data information...')
index_file_path = os.path.join(get_data_root(), 'index.csv')
index_mark_path = os.path.join(get_data_root(), 'index_mark.csv')
index_miss_path = os.path.join(get_data_root(), 'index_miss.csv')
test_file_path = os.path.join(get_data_root(), 'test.csv')
test_mark_path = os.path.join(get_data_root(), 'test_mark.csv')
test_mark_add_path = os.path.join(get_data_root(), 'test_mark_add.csv')
test_miss_path = os.path.join(get_data_root(), 'test_miss.csv')
if dataset == 'google-landmarks-dataset':
index_img_path = os.path.join(get_data_root(), 'index')
test_img_path = os.path.join(get_data_root(),
'google-landmarks-dataset-test')
elif dataset == 'google-landmarks-dataset-resize':
index_img_path = os.path.join(get_data_root(),
'resize_index_image')
test_img_path = os.path.join(get_data_root(), 'resize_test_image')
if not (os.path.isfile(index_mark_path)
or os.path.isfile(index_miss_path)):
clear_no_exist(index_file_path, index_mark_path, index_miss_path,
index_img_path)
if not (os.path.isfile(test_mark_path)
or os.path.isfile(test_miss_path)):
clear_no_exist(test_file_path, test_mark_path, test_miss_path,
test_img_path)
print('>> load index image path...')
retrieval_other_dataset = '/home/iap205/Datasets/google-landmarks-dataset-resize'
csvfile = open(index_mark_path, 'r')
csvreader = csv.reader(csvfile)
images = []
miss, add = 0, 0
for line in csvreader:
if line[0] == '1':
images.append(os.path.join(index_img_path, line[1] + '.jpg'))
elif line[0] == '0':
retrieval_img_path = os.path.join(retrieval_other_dataset,
'resize_index_image',
line[1] + '.jpg')
if os.path.isfile(retrieval_img_path):
images.append(retrieval_img_path)
add += 1
miss += 1
csvfile.close()
print(
'>>>> index image miss: {}, supplement: {}, still miss: {}'.format(
miss, add, miss - add))
print('>> load query image path...')
csvfile = open(test_mark_path, 'r')
csvreader = csv.reader(csvfile)
savefile = open(test_mark_add_path, 'w')
save_writer = csv.writer(savefile)
qimages = []
miss, add = 0, 0
for line in csvreader:
if line[0] == '1':
qimages.append(os.path.join(test_img_path, line[1] + '.jpg'))
save_writer.writerow(line)
elif line[0] == '0':
retrieval_img_path = os.path.join(retrieval_other_dataset,
'resize_test_image',
line[1] + '.jpg')
if os.path.isfile(retrieval_img_path):
qimages.append(retrieval_img_path)
save_writer.writerow(['1', line[1]])
add += 1
else:
save_writer.writerow(line)
miss += 1
csvfile.close()
savefile.close()
print(
'>>>> test image miss: {}, supplement: {}, still miss: {}'.format(
miss, add, miss - add))
# extract index vectors
print('>> {}: index images...'.format(dataset))
split_num = 6
extract_num = int(len(images) / split_num)
num_list = list(range(0, len(images), extract_num))
num_list.append(len(images))
# k = 0
# print('>>>> extract part {} of {}'.format(k, split_num-1))
# vecs = extract_vectors(net, images[num_list[k]:num_list[k+1]], args.image_size, transform, ms=ms, msp=msp)
# vecs = vecs.numpy()
# print('>>>> save index vecs to pkl...')
# vecs_file_path = os.path.join(get_data_root(), 'index_vecs{}_of_{}.pkl'.format(k+1, split_num))
# vecs_file = open(vecs_file_path, 'wb')
# pickle.dump(vecs[:, num_list[k]:num_list[k+1]], vecs_file)
# vecs_file.close()
# print('>>>> index_vecs{}_of_{}.pkl save done...'.format(k+1, split_num))
for i in range(split_num):
# vecs_temp = np.loadtxt(open(os.path.join(get_data_root(), 'index_vecs{}_of_{}.csv'.format(i+1, split_num)), "rb"),
# delimiter=",", skiprows=0)
with open(
os.path.join(
get_data_root(),
'index_vecs{}_of_{}.pkl'.format(i + 1, split_num)),
'rb') as f:
vecs_temp = pickle.load(f)
if i == 0:
vecs = vecs_temp
else:
vecs = np.hstack((vecs, vecs_temp[:, :]))
del vecs_temp
gc.collect()
print('\r>>>> index_vecs{}_of_{}.pkl load done...'.format(
i + 1, split_num),
end='')
print('')
# extract query vectors
print('>> {}: query images...'.format(dataset))
split_num = 1
extract_num = int(len(qimages) / split_num)
num_list = list(range(0, len(qimages), extract_num))
num_list.append(len(qimages))
# k = 0
# print('>>>> extract part {} of {}'.format(k, split_num - 1))
# qvecs = extract_vectors(net, qimages[num_list[k]:num_list[k + 1]], args.image_size, transform, ms=ms, msp=msp)
# qvecs = qvecs.numpy()
# for i in range(split_num):
# qvecs_file_path = os.path.join(get_data_root(), 'test_vecs{}_of_{}.pkl'.format(i+1, split_num))
# qvecs_file = open(qvecs_file_path, 'wb')
# pickle.dump(qvecs[:, num_list[i]:num_list[i+1]], qvecs_file)
# qvecs_file.close()
# print('\r>>>> test_vecs{}_of_{}.pkl save done...'.format(i+1, split_num), end='')
# print('')
for i in range(split_num):
# qvecs_temp = np.loadtxt(open(os.path.join(get_data_root(), 'test_vecs{}_of_{}.csv'.format(i+1, split_num)), "rb"),
# delimiter=",", skiprows=0)
with open(
os.path.join(
get_data_root(),
'test_vecs{}_of_{}.pkl'.format(i + 1, split_num)),
'rb') as f:
qvecs_temp = pickle.load(f)
if i == 0:
qvecs = qvecs_temp
else:
qvecs = np.hstack((qvecs, qvecs_temp[:, :]))
del qvecs_temp
gc.collect()
print('\r>>>> test_vecs{}_of_{}.pkl load done...'.format(
i + 1, split_num),
end='')
print('')
# vecs = np.zeros((2048, 1093278))
# qvecs = np.zeros((2048, 115921))
# save vecs to csv file
# np.savetxt(os.path.join(get_data_root(), 'index_vecs{}_of_{}.csv'.format(k, split_num-1)), vecs, delimiter=',')
# np.savetxt(os.path.join(get_data_root(), 'test_vecs{}_of_{}.csv'.format(k, split_num-1)), qvecs, delimiter=',')
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if len(ms) > 1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(
args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.
format(args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(
args.whitening))
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = vecs
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
m, P = pcawhitenlearn(wvecs)
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
print('>> apply PCAwhiten...')
if Lw is not None:
# whiten the vectors and shrinkage
vecs = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs = whitenapply(qvecs, Lw['m'], Lw['P'])
print('>>>> save index PCAwhiten vecs to pkl...')
split_num = 6
extract_num = int(len(images) / split_num)
num_list = list(range(0, len(images), extract_num))
num_list.append(len(images))
for i in range(split_num):
vecs_file_path = os.path.join(
get_data_root(),
'index_PCAwhiten_vecs{}_of_{}.pkl'.format(i + 1, split_num))
vecs_file = open(vecs_file_path, 'wb')
pickle.dump(vecs[:, num_list[i]:num_list[i + 1]], vecs_file)
vecs_file.close()
print(
'\r>>>> index_PCAwhiten_vecs{}_of_{}.pkl save done...'.format(
i + 1, split_num),
end='')
print('')
print('>>>> save test PCAwhiten vecs to pkl...')
split_num = 1
extract_num = int(len(qimages) / split_num)
num_list = list(range(0, len(qimages), extract_num))
num_list.append(len(images))
for i in range(split_num):
qvecs_file_path = os.path.join(
get_data_root(),
'test_PCAwhiten_vecs{}_of_{}.pkl'.format(i + 1, split_num))
qvecs_file = open(qvecs_file_path, 'wb')
pickle.dump(qvecs[:, num_list[i]:num_list[i + 1]], qvecs_file)
qvecs_file.close()
print('\r>>>> test_PCAwhiten_vecs{}_of_{}.pkl save done...'.format(
i + 1, split_num),
end='')
print('')
print('>>>> load index PCAwhiten vecs from pkl...')
for i in range(split_num):
with open(
os.path.join(
get_data_root(),
'index_PCAwhiten_vecs{}_of_{}.pkl'.format(
i + 1, split_num)), 'rb') as f:
vecs_temp = pickle.load(f)
if i == 0:
vecs = vecs_temp
else:
vecs = np.hstack((vecs, vecs_temp[:, :]))
del vecs_temp
gc.collect()
print(
'\r>>>> index_PCAwhiten_vecs{}_of_{}.pkl load done...'.format(
i + 1, split_num),
end='')
print('')
print('>>>> load test PCAwhiten vecs from pkl...')
for i in range(split_num):
with open(
os.path.join(
get_data_root(),
'test_PCAwhiten_vecs{}_of_{}.pkl'.format(
i + 1, split_num)), 'rb') as f:
qvecs_temp = pickle.load(f)
if i == 0:
qvecs = qvecs_temp
else:
qvecs = np.hstack((qvecs, qvecs_temp[:, :]))
del qvecs_temp
gc.collect()
print('\r>>>> test_PCAwhiten_vecs{}_of_{}.pkl load done...'.format(
i + 1, split_num),
end='')
print('')
# extract principal components and dimension shrinkage
ratio = 0.8
vecs = vecs[:int(vecs.shape[0] * ratio), :]
qvecs = vecs[:int(qvecs.shape[0] * ratio), :]
print('>> {}: Evaluating...'.format(dataset))
split_num = 50
top_num = 100
vecs_T = np.zeros((vecs.shape[1], vecs.shape[0])).astype('float32')
vecs_T[:] = vecs.T[:]
QE_iter = 0
QE_weight = (np.arange(top_num, 0, -1) / top_num).reshape(
1, top_num, 1)
print('>> find {} nearest neighbour...'.format(top_num))
import faiss # place it in the file top will cause network load so slowly
# ranks_top_100 = np.loadtxt(open(os.path.join(get_data_root(), 'ranks_top_{}.csv'.format(top_num)), "rb"),
# delimiter=",", skiprows=0).astype('int')
for iter in range(0, QE_iter + 1):
if iter != 0:
# ranks_top_100 = np.ones((100, 115921)).astype('int')
print('>> Query expansion iteration {}'.format(iter))
ranks_split = 50
for i in range(ranks_split):
ranks_top_100_split = ranks_top_100[:,
int(ranks_top_100.
shape[1] /
ranks_split * i
):int(ranks_top_100
.shape[1] /
ranks_split *
(i + 1))]
top_100_vecs = vecs[:,
ranks_top_100_split] # (2048, 100, query_split_size)
qvecs_temp = (top_100_vecs * QE_weight).sum(axis=1)
qvecs_temp = qvecs_temp / (np.linalg.norm(
qvecs_temp, ord=2, axis=0, keepdims=True) + 1e-6)
if i == 0:
qvecs = qvecs_temp
else:
qvecs = np.hstack((qvecs, qvecs_temp))
del ranks_top_100_split, top_100_vecs, qvecs_temp
gc.collect()
print('\r>>>> calculate new query vectors {}/{} done...'.
format(i + 1, ranks_split),
end='')
print('')
qe_iter_qvecs_path = os.path.join(
get_data_root(), 'QE_iter{}_qvecs.pkl'.format(iter))
qe_iter_qvecs_file = open(qe_iter_qvecs_path, 'wb')
pickle.dump(qvecs, qe_iter_qvecs_file)
qe_iter_qvecs_file.close()
print('>>>> QE_iter{}_qvecs.pkl save done...'.format(iter))
del ranks_top_100
gc.collect()
for i in range(split_num):
# scores = np.dot(vecs.T, qvecs[:, int(qvecs.shape[1]/split_num*i):int(qvecs.shape[1]/split_num*(i+1))])
# ranks = np.argsort(-scores, axis=0)
# kNN search
k = top_num
index = faiss.IndexFlatL2(vecs.shape[0])
index.add(vecs_T)
query_vecs = qvecs[:,
int(qvecs.shape[1] / split_num *
i):int(qvecs.shape[1] / split_num *
(i + 1))]
qvecs_T = np.zeros((query_vecs.shape[1],
query_vecs.shape[0])).astype('float32')
qvecs_T[:] = query_vecs.T[:]
_, ranks = index.search(qvecs_T, k)
ranks = ranks.T
if i == 0:
ranks_top_100 = ranks[:top_num, :]
else:
ranks_top_100 = np.hstack(
(ranks_top_100, ranks[:top_num, :]))
# del scores, ranks
del index, query_vecs, qvecs_T, ranks
gc.collect()
print('\r>>>> kNN search {} nearest neighbour {}/{} done...'.
format(top_num, i + 1, split_num),
end='')
del qvecs
gc.collect()
print('')
del vecs, vecs_T
gc.collect()
# save to csv file
print(">> save to submission.csv file...")
submission_file = open(os.path.join(get_data_root(), 'submission.csv'),
'w')
writer = csv.writer(submission_file)
test_mark_file = open(test_mark_add_path, 'r')
csvreader = csv.reader(test_mark_file)
cnt = 0
writer.writerow(['id', 'images'])
for index, line in enumerate(csvreader):
(flag, img_name) = line[:2]
if flag == '1':
select = []
for i in range(top_num):
select.append(images[int(
ranks_top_100[i,
cnt])].split('/')[-1].split('.jpg')[0])
cnt += 1
writer.writerow([
img_name.split('/')[-1].split('.jpg')[0], ' '.join(select)
])
else:
# random_list = random.sample(range(0, len(images)), top_num)
random_list = np.random.choice(len(images),
top_num,
replace=False)
select = []
for i in range(top_num):
select.append(
images[random_list[i]].split('/')[-1].split('.jpg')[0])
writer.writerow([
img_name.split('/')[-1].split('.jpg')[0], ' '.join(select)
])
if cnt % 10 == 0 or cnt == len(qimages):
print('\r>>>> {}/{} done...'.format(cnt, len(qimages)), end='')
submission_file.close()
test_mark_file.close()
print('')
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def call_benchmark(
# must pass one of images or paths
images=None,
paths=None,
**kwargs,
):
"""Run the given network on the given data and return vectors for it."""
# load params
params = default_params.copy()
params.update(kwargs)
network = params["network"]
offtheshelf = params["offtheshelf"]
image_size = params["image_size"]
gpu = params["gpu"]
multiscale = params["multiscale"]
whitening = params["whitening"]
net_key = (network, offtheshelf, gpu)
if net_key in LOADED_NETWORKS:
net = LOADED_NETWORKS[net_key]
else:
# load network
if offtheshelf:
net = load_offtheshelf(network)
else:
net = load_network(network)
# moving network to gpu and eval mode
if gpu:
net.cuda()
net.eval()
# store network in memo dict
LOADED_NETWORKS[net_key] = net
# setting up the multi-scale parameters
ms = [1]
msp = 1
if multiscale:
ms = [1, 1 / np.sqrt(2), 1 / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std'],
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize,
])
# setting up whitening
if whitening is not None:
if 'Lw' in net.meta and whitening in net.meta['Lw']:
if multiscale:
Lw = net.meta['Lw'][whitening]['ms']
else:
Lw = net.meta['Lw'][whitening]['ss']
elif whitening == "scores":
whiten_key = (network, offtheshelf, image_size, multiscale)
Lw = get_scores_whitening(whiten_key,
net,
transform,
ms,
msp,
image_size,
setup_network=False,
gpu=gpu)
else:
raise ValueError(
"invalid whitening {} (valid whitenings: {})".format(
whitening, list(net.meta['Lw'].keys())))
# process the given data
if images is not None:
images = np.asarray(images)
print("images.shape =", images.shape)
vecs = vectors_from_images(net,
images,
transform,
ms=ms,
msp=msp,
setup_network=False,
gpu=gpu)
else:
vecs = extract_vectors(net,
paths,
image_size,
transform,
ms=ms,
msp=msp,
setup_network=False,
gpu=gpu)
# convert to numpy
vecs = vecs.numpy()
# apply whitening
if whitening is not None:
vecs = whitenapply(vecs, Lw['m'], Lw['P'])
# take transpose
vecs = vecs.T
print("vecs.shape =", vecs.shape)
return vecs
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
whitening, whiten_fn))
# torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(whitening,
htime(time.time() - start)))
else:
Lw = None
if Lw is not None:
for dim in param['pac_dims']:
print('>>>> pac_dim: {}'.format(dim))
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'], dim)
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'], dim)
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks_lw = np.argsort(-scores, axis=0)
qvecs_lw_orig = qvecs_lw
ranks_lw_orig = ranks_lw
mismatched_info = compute_map_and_print(dataset + ' + whiten',
ranks_lw,
cfg['gnd'],
kappas=[1, 5, 10, 100])
# %%
# save vecs, qvecs, ranks to pickle
import pickle
def testOxfordParisHolidays(net, eConfig):
#datasets = eConfig['test-datasets'].split(',')
#results = []
#
#for dataset in datasets:
# results.append((dataset, np.random.rand(1)[0]))
#
#return results
print('>> Evaluating network on test datasets...')
# for testing we use image size of max 1024
image_size = 1024
# setting up the multi-scale parameters
ms = [1]
msp = 1
if (eConfig['multiscale']):
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if eConfig['whitening']:
start = time.time()
print('>> {}: Learning whitening...'.format(eConfig['test-whiten']))
# loading db
db_root = os.path.join(get_data_root(), 'train',
eConfig['test-whiten'])
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(eConfig['test-whiten']))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(eConfig['test-whiten']))
wvecs = extract_vectors(net,
images,
image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(eConfig['test-whiten']))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(eConfig['test-whiten'],
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = eConfig['test-datasets'].split(',')
results = []
for dataset in datasets:
start = time.time()
if (dataset != 'holidays' and dataset != 'rholidays'):
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
if (dataset == 'oxford105k' or dataset == 'paris106k'):
images.extend(cfg['distractors'])
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
image_size,
transform,
bbxs,
ms=ms,
msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
results.append(
compute_map_and_print(
dataset +
('+ multiscale' if eConfig['multiscale'] else ''), ranks,
cfg['gnd']))
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
results.append(
compute_map_and_print(dataset + ' + whiten', ranks,
cfg['gnd']))
else:
results.append(testHolidays(net, eConfig, dataset, Lw))
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
return results
def embed(params, data):
net = params.pop("net")
imgdir = params.pop("imgdir")
whitening = params.pop("whitening", None)
whitening_dir = params.pop("whitening_dir", None)
image_size = params.pop("image_size", 1024)
multiscale = params.pop("multiscale", True)
assert not params, params.keys()
input_images, bbxs = (data[0], None) if len(data) == 1 else data
impaths = [path_join(imgdir, x) for x in input_images]
if not data[0]:
return ({
"status": "skipped"
}, [], []) + (([], ) if whitening_dir else tuple())
# Handle paths
assert os.path.exists(net), net
# loading network from path
print(">> Loading network:\n>>>> '{}'".format(net))
state = torch.load(net)
net = init_network({
'architecture': state['meta']['architecture'],
'pooling': state['meta']['pooling'],
'whitening': state['meta']['whitening'],
'mean': state['meta']['mean'],
'std': state['meta']['std'],
'pretrained': False
})
net.load_state_dict(state['state_dict'])
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = multiscale if not isinstance(
multiscale, bool) else [1, 1. / math.sqrt(2), 1. /
2] if multiscale else [1]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if whitening_dir:
whitening_dir = os.path.join(
whitening_dir,
"%s_%s_%s_%s.lw.pkl" % (whitening, None, image_size, multiscale))
print('>> {}: Loading whitening...'.format(whitening))
with open(whitening_dir, "rb") as handle:
Lw = pickle.load(handle)
# elif whitening:
# Lw, _ = _compute_whitening(whitening, net, image_size, transform, ms, msp)
else:
Lw = None
# extract database and query vectors
print('>> Images descriptors...')
vecs = extract_vectors(net,
impaths,
image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
print('>> Evaluating...')
# convert to numpy
vecs = vecs.numpy()
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
return {}, input_images, vecs.T, vecs_lw.T
return {}, input_images, vecs.T
