def __init__(self, params):
# prepare config structure for the test dataset
self.image_size = params.pop("image_size")
self.dataset = params.pop("dataset")
self.transforms = initialize_transforms(params.pop("transforms"), params.pop("mean_std"))
if isinstance(self.dataset, dict):
# Tsv dataset files
assert self.dataset.keys() == {"name", "queries", "db", "imgdir"}
imgdir = self.dataset['imgdir']
with initialize_file_reader(self.dataset['db'], keys=["identifier"]) as reader:
data = reader.get()
self.images = [path_join(imgdir, x) for x in data["identifier"]]
mapping = {x: i for i, x in enumerate(data["identifier"])}
with initialize_file_reader(self.dataset['queries'], keys=["query", "bbx", "ok", "junk"]) as reader:
data = reader.get()
self.qimages = [path_join(imgdir, x) for x in data["query"]]
self.bbxs = [tuple(x) if x else None for x in data["bbx"]]
self.gnd = [{'ok': [mapping[x] for x in ok], 'junk': [mapping[x] for x in junk]} \
for ok, junk in zip(data["ok"], data["junk"])]
self.dataset = self.dataset['name']
else:
# Official cirtorch files
cfg = configdataset(self.dataset, os.path.join(get_data_root(), 'test'))
self.images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
self.qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
self.bbxs = [tuple(cfg['gnd'][i]['bbx']) if cfg['gnd'][i]['bbx'] else None for i in range(cfg['nq'])]
self.gnd = cfg['gnd']
assert not params, params.keys()
def test(datasets, net):
print(">> Evaluating network on test datasets...")
image_size = 1024
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta["mean"],
std=net.meta["std"])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
# Lw = None
Lw = net.meta["Lw"]["retrieval-SfM-120k"]["ss"]
# evaluate on test datasets
# datasets = args.test_datasets.split(",")
for dataset in datasets:
start = time.time()
print(">> {}: Extracting...".format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), "test"))
images = [cfg["im_fname"](cfg, i) for i in range(cfg["n"])]
qimages = [cfg["qim_fname"](cfg, i) for i in range(cfg["nq"])]
bbxs = [tuple(cfg["gnd"][i]["bbx"]) for i in range(cfg["nq"])]
# extract database and query vectors
print(">> {}: database images...".format(dataset))
vecs = extract_vectors(net, images, image_size, transform)
print(">> {}: query images...".format(dataset))
qvecs = extract_vectors(net, qimages, image_size, transform, bbxs)
print(">> {}: Evaluating...".format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg["gnd"])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw["m"], Lw["P"])
qvecs_lw = whitenapply(qvecs, Lw["m"], Lw["P"])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + " + whiten", ranks, cfg["gnd"])
print(">> {}: elapsed time: {}".format(dataset,
htime(time.time() - start)))
def query_ivf(asmk, dataset, desc, globals, logger):
"""The last step of asmk method - querying the ivf"""
metadata, _images, ranks, _scores = asmk.query_ivf(desc['qvecs'],
desc['qimids'])
logger.debug(
f"Average query time (quant+aggr+search) is {metadata['query_avg_time']:.3f}s"
)
gnd = configdataset(dataset, f"{globals['root_path']}/test/")['gnd']
compute_map_and_print(dataset, ranks.T, gnd)
def load_dataset(dataset, data_root=''):
"""Return tuple (image list, query list, bounding boxes, gnd dictionary)"""
if isinstance(dataset, dict):
root = os.path.join(data_root, dataset['image_root'])
images, qimages = None, None
if dataset['database_list'] is not None:
images = [path_join(root, x.strip("\n")) for x in open(dataset['database_list']).readlines()]
if dataset['query_list'] is not None:
qimages = [path_join(root, x.strip("\n")) for x in open(dataset['query_list']).readlines()]
bbxs = None
gnd = None
elif dataset == 'train':
training_set = 'retrieval-SfM-120k'
db_root = os.path.join(data_root, 'train', training_set)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}.pkl'.format(training_set))
with open(db_fn, 'rb') as f:
db = pickle.load(f)['train']
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
qimages = []
bbxs = None
gnd = None
elif dataset == 'val_eccv20':
db_root = os.path.join(data_root, 'train', 'retrieval-SfM-120k')
fn_val_proper = db_root+'/retrieval-SfM-120k-val-eccv2020.pkl' # pos are all with #inl >=3 & <= 10
with open(fn_val_proper, 'rb') as f:
db = pickle.load(f)
ims_root = os.path.join(db_root, 'ims')
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
gnd = db['gnd']
qidx = db['qidx']
qimages = [images[x] for x in qidx]
bbxs = None
elif "/" in dataset:
with open(dataset, 'rb') as handle:
db = pickle.load(handle)
images, qimages, bbxs, gnd = db['imlist'], db['qimlist'], None, db['gnd']
else:
cfg = configdataset(dataset, os.path.join(data_root, 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
if 'bbx' in cfg['gnd'][0].keys():
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
else:
bbxs = None
gnd = cfg['gnd']
return images, qimages, bbxs, gnd
def gen_query_bbx_img():
datasets = ['oxford5k', 'paris6k', 'roxford5k', 'rparis6k']
for dataset in datasets:
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
width = 5
for i in range(len(qimages)):
im = Image.open(qimages[i])
draw = ImageDraw.Draw(im)
(x0, y0, x1, y1) = bbxs[i]
for j in range(width):
draw.rectangle([x0-j, y0-j, x1+j, y1+j], outline='yellow')
im.save('_bbx.jpg'.join(qimages[i].split('.jpg')))
print("{} qurery_bbx generate ok".format(dataset))
def load_dataset(dataset, data_root=''):
"""Return tuple (image list, query list, bounding boxes, gnd dictionary)"""
if dataset == 'mitsubishi_dataset':
global images
global qimages
global bbxs
global gnd
cfg = configdataset()
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
if 'bbx' in cfg['gnd'][0].keys():
#cfg['gnd']=[{'bbx':'','xx':''},{},...{}]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
else:
bbxs = None
gnd=cfg['gnd']
return images, qimages, bbxs, gnd
def eval_datasets(model,
datasets=('oxford5k', 'paris6k', 'roxford5k', 'rparis6k'),
ms=False,
tta_gem_p=1.0,
logger=None):
model = model.eval()
data_root = get_data_root()
scales = [1 / 2**(1 / 2), 1.0, 2**(1 / 2)] if ms else [1.0]
results = dict()
for dataset in datasets:
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(data_root, 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
tqdm_desc = cfg['dataset']
db_feats = extract_vectors(model,
images=images,
bbxs=None,
scales=scales,
tta_gem_p=tta_gem_p,
tqdm_desc=tqdm_desc)
query_feats = extract_vectors(model,
images=qimages,
bbxs=bbxs,
scales=scales,
tta_gem_p=tta_gem_p,
tqdm_desc=tqdm_desc)
scores = np.dot(db_feats, query_feats.T)
ranks = np.argsort(-scores, axis=0)
results[dataset] = compute_map_and_print(dataset,
ranks,
cfg['gnd'],
kappas=[1, 5, 10],
logger=logger)
return results
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError(
'Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network
# pretrained networks (downloaded automatically)
print(">> Loading network:\n>>>> '{}'".format(args.network))
state = load_url(PRETRAINED[args.network],
model_dir=os.path.join(get_data_root(), 'networks'))
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get('local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# network initialization
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
print(">>>> Evaluating scales: {}".format(ms))
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
try:
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
except:
bbxs = None # for holidaysmanrot and copydays
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
top_k = 100
ranks_fnames_qs = []
for q_id in range(len(cfg["qimlist"])):
ranks_q = list(ranks[:top_k, q_id])
ranks_fname_per_q = []
for img_id in ranks_q:
ranks_fname_per_q.append(cfg["imlist"][img_id])
ranks_fnames_qs.append(ranks_fname_per_q)
compute_map_and_print(dataset, ranks, cfg['gnd'])
compute_map_and_print_top_k(dataset, ranks_fnames_qs, cfg['gnd'],
cfg["imlist"])
sys.exit()
with open(dataset + "_gl18_tl_resnet101_gem_w_m.pkl", "wb") as f:
data = {"ranks": ranks, "db_images": images, "q_images": qimages}
pickle.dump(data, f)
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
state = torch.load(args.network_path)
net = init_network(model=state['meta']['architecture'],
pooling=state['meta']['pooling'],
whitening=state['meta']['whitening'],
mean=state['meta']['mean'],
std=state['meta']['std'],
pretrained=False)
net.load_state_dict(state['state_dict'])
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
offtheshelf = args.network_offtheshelf.split('-')
if len(offtheshelf) == 3:
if offtheshelf[2] == 'whiten':
offtheshelf_whiten = True
else:
raise (RuntimeError(
"Incorrect format of the off-the-shelf network. Examples: resnet101-gem | resnet101-gem-whiten"
))
else:
offtheshelf_whiten = False
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(model=offtheshelf[0],
pooling=offtheshelf[1],
whitening=offtheshelf_whiten)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = [1]
msp = 1
if args.multiscale:
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
if dataset == 'reco':
images, qimages = landmark_recognition_dataset()
bbxs = [None for x in qimages]
elif dataset == 'retr':
images, _ = landmark_retrieval_dataset()
qimages = []
bbxs = [None for x in qimages]
else:
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
with open('%s_fnames.pkl' % dataset, 'wb') as f:
pickle.dump([images, qimages], f)
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
vecs = vecs.numpy()
print('>> saving')
np.save('{}_vecs.npy'.format(dataset), vecs)
if len(qimages) > 0:
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
qvecs = qvecs.numpy()
np.save('{}_qvecs.npy'.format(dataset), qvecs)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# TODO
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
print(net.meta_repr())
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# %%
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
# bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
print('>> not use bbxs...')
bbxs = None
# %%
for param in params:
for image_size in param['image_size']:
# setting up the multi-scale parameters
print(">> image size: {}".format(image_size))
if len(param['ms']) > 1 and net.meta['pooling'] == 'gem' \
and not net.meta['regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
def main():
print(">> Retrieval evaluation of attacks\n")
args = parser.parse_args()
# check if unknown dataset
if args.dataset not in datasets_names:
raise ValueError('Unsupported or unknown dataset: {}!'.format(
args.dataset))
# check if test dataset are downloaded and download if they are not
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {
'architecture': offtheshelf[0],
'pooling': offtheshelf[1],
'local_whitening': False,
'regional': False,
'whitening': False,
'pretrained': True
}
# load off-the-shelf network
print(">> Loading off-the-shelf network: '{}'".format(
args.network_offtheshelf))
net = init_network(net_params)
# print(">>>> loaded network: \n'{}'".format(net.meta_repr()))
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# evaluate on test dataset
dataset = args.dataset
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
cfg.update({'qext_a': args.ext_attack})
cfg.update({'dir_data_a': args.dir_attack})
cfg.update({'dir_images_a': cfg['dir_data_a']})
cfg.update({'qim_fname_a': config_qimname_a})
# reduce number of queries for holidays and copydays
if dataset.startswith('holidays') or dataset.startswith('copydays'):
cfg['nq'] = 50
cfg['gnd'] = cfg['gnd'][:cfg['nq']]
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
qimages_a = [cfg['qim_fname_a'](cfg, i) for i in range(cfg['nq'])]
try:
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
except:
bbxs = None # for holidays and copydays
# extract descriptors and cache or load cached ones
print('>> {}: database images...'.format(dataset))
network_fn = args.network_offtheshelf
if args.dir_cache is not None:
vecs_fn = os.path.join(
args.dir_cache,
'{}_{}_{}_{}_vecs.pth'.format(dataset, network_fn, args.image_size,
args.image_resize))
if os.path.isfile(vecs_fn):
vecs = torch.load(vecs_fn)
print('>> loaded cached descriptors from {}'.format(vecs_fn))
else:
vecs = extract_vectors_a(net, images, args.image_size,
args.image_resize, transform)
torch.save(vecs, vecs_fn)
print('>> {}: standard query images...'.format(dataset))
qvecs = extract_vectors_a(net,
qimages,
args.image_size,
args.image_resize,
transform,
bbxs=bbxs)
print('>> {}: attack query images...'.format(dataset))
qvecs_a = extract_vectors_a(net, qimages_a, args.image_size,
args.image_resize, transform)
print('>> {}: evaluating for image resolution {}'.format(
dataset, args.image_resize))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
qvecs_a = qvecs_a.numpy()
qip = (qvecs * qvecs_a).sum(axis=0)
qip_mean, qip_std = qip.mean(), qip.std()
print('>> {}: inner product (target,attack) mean: {:.3f}, std: {:.3f}'.
format(dataset, qip_mean, qip_std))
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
maps, mprs, aps = compute_map_and_print(dataset, ranks, cfg['gnd'])
# attack search, rank, and print
scores = np.dot(vecs.T, qvecs_a)
ranks = np.argsort(-scores, axis=0)
maps_a, mprs_a, aps_a = compute_map_and_print(
dataset + ' attack ({})'.format(args.ext_attack), ranks, cfg['gnd'])
if dataset.startswith('roxford5k') or dataset.startswith('rparis6k'):
r1, r2, r3 = 100 * maps[1], 100 * maps_a[1], 100 * (
maps_a[1] - maps[1]) # medium protocol
else:
r1, r2, r3 = 100 * maps[0], 100 * maps_a[0], 100 * (maps_a[0] -
maps[0])
print(
'\n*** Summary ***\n attack: {}\n test: {}-{}-{} \n mean ip (target,attack): {:.3f}\n mAP: org {:.2f} att {:.2f} dif {:.2f}\n'
.format(
args.dir_attack.split('/')[-2], dataset, args.network_offtheshelf,
args.image_resize, qip_mean, r1, r2, r3))
if args.log is not None:
with open(args.log, 'a') as f:
f.write(
'\n attack: {}\n test: {}-{}-{} \n mean ip (target,attack): {:.3f}\n mAP: org {:.2f} att {:.2f} dif {:.2f}\n\n'
.format(
args.dir_attack.split('/')[-2], dataset,
args.network_offtheshelf, args.image_resize, qip_mean, r1,
r2, r3))
def main():
args = parser.parse_args()
# check if there are unknown datasets
# for dataset in args.datasets.split(','):
# if dataset not in datasets_names:
# raise ValueError('Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
# download_train(get_data_root())
# download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
#datasets = args.datasets.split(',')
for dataset in datasets_names:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join('/data', 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
global_feature_dict, gloabal_feature_size = build_global_feature_dict(
'/home/ubuntu/DELG/extract/roxf5k.delf.global', 'global_features')
# # extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_feature(global_feature_dict, images,
gloabal_feature_size)
print('>> {}: query images...'.format(dataset))
qvecs = extract_feature(global_feature_dict, qimages,
gloabal_feature_size)
print('>> {}: Evaluating...'.format(dataset))
# search, rank, and print
# using cosine similarity of two vectors
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'])
feature_location, feature_descriptor = build_local_feature_dict(
'/home/ubuntu/DELG/extract/roxf5k.delf.local')
new_ranks = [] #np.empty_like(ranks)
# for i, qimage in enumerate(qimages):
# new_ranks[:, i] = RerankByGeometricVerification(i, ranks[:, i], scores[:, i], qimage,
# images, feature_location,
# feature_descriptor, [])
from functools import partial
re_rank_func = partial(RerankByGeometricVerification,
input_ranks=ranks,
initial_scores=scores,
query_name=qimages,
index_names=images,
feature_location=feature_location,
feature_descriptor=feature_descriptor,
junk_ids=[])
new_ranks = p_map(re_rank_func, range(len(qimages)))
new_ranks = np.concatenate(new_ranks, axis=0)
compute_map_and_print(dataset, new_ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def test(datasets, net):
print('>> Evaluating network on test datasets...')
# for testing we use image size of max 1024
image_size = 1024
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.test_whiten:
start = time.time()
print('>> {}: Learning whitening...'.format(args.test_whiten))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.test_whiten)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.test_whiten))
wvecs = extract_vectors(net,
images,
image_size,
transform,
print_freq=10,
batchsize=20) # implemented with torch.no_grad
# learning whitening
print('>> {}: Learning...'.format(args.test_whiten))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.test_whiten,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.test_datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
if dataset == 'cdvs_test_retrieval':
bbxs = None
else:
bbxs = None
print('>> {}: database images...'.format(dataset))
if args.pool == 'gem':
ms = [1, 1 / 2**(1 / 2), 1 / 2]
else:
ms = [1]
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
vecs = extract_vectors(net,
images,
image_size,
transform,
bbxs,
ms=ms,
msp=msp,
print_freq=1000,
batchsize=1) # implemented with torch.no_grad
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
image_size,
transform,
bbxs,
ms=ms,
msp=msp,
print_freq=1000,
batchsize=1) # implemented with torch.no_grad
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
if dataset == 'cdvs_test_retrieval':
compute_map_and_print(dataset, ranks, cfg['gnd_id'])
else:
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def test(datasets, net, wandb_enabled=False, epoch=-1):
global global_step
print('>> Evaluating network on test datasets...')
# for testing we use image size of max 1024
image_size = 1024
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.test_whiten:
start = time.time()
print('>> {}: Learning whitening...'.format(args.test_whiten))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.test_whiten)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.test_whiten))
wvecs = extract_vectors(net, images, image_size, transform) # implemented with torch.no_grad
# learning whitening
print('>> {}: Learning...'.format(args.test_whiten))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.test_whiten, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.test_datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, image_size, transform) # implemented with torch.no_grad
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, image_size, transform, bbxs) # implemented with torch.no_grad
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'], wandb_enabled=wandb_enabled, epoch=epoch, global_step=global_step)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'], wandb_enabled=wandb_enabled, epoch=epoch, global_step=global_step)
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError('Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
# download_train(get_data_root())
# download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get('local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
net_params['multi_layer_cat'] = state['meta']['multi_layer_cat']
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
print(">> image size: {}".format(args.image_size))
ms = list(eval(args.multiscale))
if len(ms)>1 and net.meta['pooling'] == 'gem' and not net.meta['regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
if len(ms)>1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
# bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
print('>> not use bbxs...')
bbxs = None
# key_url_list = ParseData(os.path.join(get_data_root(), 'index.csv'))
# index_image_path = os.path.join(get_data_root(), 'resize_index_image')
# images = [os.path.join(index_image_path, key_url_list[i][0]) for i in range(len(key_url_list))]
# key_url_list = ParseData(os.path.join(get_data_root(), 'test.csv'))
# test_image_path = os.path.join(get_data_root(), 'resize_test_image')
# qimages = [os.path.join(test_image_path, key_url_list[i][0]) for i in range(len(key_url_list))]
# # bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# csvfile = open(os.path.join(get_data_root(), 'index_clear.csv'), 'r')
# csvreader = csv.reader(csvfile)
# images = [line[:1][0] for line in csvreader]
#
# csvfile = open(os.path.join(get_data_root(), 'test_clear.csv'), 'r')
# csvreader = csv.reader(csvfile)
# qimages = [line[:1][0] for line in csvreader]
# bbxs = None
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# vecs = torch.randn(2048, 5063)
# vecs = torch.randn(2048, 4993)
# hxq modified
# bbxs = None
# print('>> set no bbxs...')
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
# hxq modified, test add features map for retrieval
# vecs = [vecs[i].numpy() for i in range(len(vecs))]
# qvecs_temp = np.zeros((qvecs[0].shape[0], len(qvecs)))
# for i in range(len(qvecs)):
# qvecs_temp[:, i] = qvecs[i][:, 0].numpy()
# qvecs = qvecs_temp
#
# scores = np.zeros((len(vecs), qvecs.shape[-1]))
# for i in range(len(vecs)):
# scores[i, :] = np.amax(np.dot(vecs[i].T, qvecs), 0)
ranks = np.argsort(-scores, axis=0)
mismatched_info = compute_map_and_print(dataset, ranks, cfg['gnd'], kappas=[1, 5, 10, 100])
# hxq added
show_false_img = False
if show_false_img == True:
print('>> Save mismatched image tuple...')
for info in mismatched_info:
mismatched_img_show_save(info, qimages, images, args, bbxs=bbxs)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
mismatched_info = compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
# hxq added
# show_false_img = False
if show_false_img == True:
print('>> Save mismatched image tuple...')
for info in mismatched_info:
mismatched_img_show_save(info, qimages, images, args, bbxs=bbxs)
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main():
#def process(network_path, datasets='oxford5k,paris6k', whitening=None, image_size=1024, multiscale = '[1]', query=None):
args = parser.parse_args()
#args.query = None
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError(
'Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
#download_train(get_data_root())
#download_test(get_data_root())
# setting up the visible GPU
#os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get(
'local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
#net.cuda()
#net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if len(ms) > 1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(
args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
print(whiten_fn)
return
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train',
args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
# query type
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
#for i in cfg: print(i)
#print(cfg['gnd'][0]['bbx'])
#return
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
feas_dir = os.path.join(cfg['dir_data'], 'features')
if not os.path.isdir(feas_dir):
os.mkdir(feas_dir)
feas_sv = os.path.join(
feas_dir, dataset + '_' + args.network_path + '_features.pkl')
if not os.path.isfile(feas_sv):
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
with open(feas_sv, 'wb') as f:
pickle.dump(vecs, f)
else:
with open(feas_sv, 'rb') as f:
vecs = pickle.load(f)
print('>> {}: query images...'.format(dataset))
if args.query is not None:
qimages = [args.query]
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
ms=ms,
msp=msp)
else:
qfeas_dir = feas_dir
qfeas_sv = os.path.join(
qfeas_dir,
dataset + '_' + args.network_path + '_qfeatures.pkl')
if not os.path.isfile(qfeas_sv):
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
try:
bbxs = [
tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])
]
except:
bbxs = None
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
with open(qfeas_sv, 'wb') as f:
pickle.dump(qvecs, f)
else:
with open(qfeas_sv, 'rb') as f:
qvecs = pickle.load(f)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
#qvecs = qvecs[:, 0].reshape(-1, 1)
#args.query = True
# search, rank, and print
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranksw = np.argsort(-scores, axis=0)
if args.query is None:
#compute_map_and_print(dataset + ' + whiten', ranksw, cfg['gnd'])
compute_map_and_print1(dataset + ' + whiten', ranksw,
cfg['gnd'])
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
# compute_map_and_print(dataset, ranks, cfg['gnd'])
compute_map_and_print1(dataset, ranks, cfg['gnd'])
else:
a = []
for i in ranksw:
a.append(
os.path.join(cfg['dir_images'], cfg['imlist'][i[0]]) +
cfg['ext'])
print(a[:10])
result = cfg['dir_data'] + '_result'
with open(result + '.pkl', 'wb') as f:
pickle.dump(a[:10], f)
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def test(datasets, net, noise, image_size):
global base
print(">> Evaluating network on test datasets...")
net.cuda()
net.eval()
normalize = transforms.Normalize(mean=net.meta["mean"],
std=net.meta["std"])
def add_noise(img):
n = noise
n = F.interpolate(n.unsqueeze(0),
mode=MODE,
size=tuple(img.shape[-2:]),
align_corners=True).squeeze()
return torch.clamp(img + n, 0, 1)
transform_base = transforms.Compose([transforms.ToTensor(), normalize])
transform_query = transforms.Compose(
[transforms.ToTensor(),
transforms.Lambda(add_noise), normalize])
if "Lw" in net.meta:
Lw = net.meta["Lw"]["retrieval-SfM-120k"]["ss"]
else:
Lw = None
# evaluate on test datasets
datasets = args.test_datasets.split(",")
attack_result = {}
for dataset in datasets:
start = time.time()
print(">> {}: Extracting...".format(dataset))
cfg = configdataset(dataset, os.path.join(get_data_root(), "test"))
images = [cfg["im_fname"](cfg, i) for i in range(cfg["n"])]
qimages = [cfg["qim_fname"](cfg, i) for i in range(cfg["nq"])]
bbxs = [tuple(cfg["gnd"][i]["bbx"]) for i in range(cfg["nq"])]
# extract database and query vectors
print(">> {}: database images...".format(dataset))
with torch.no_grad():
if dataset in base and str(image_size) in base[dataset]:
vecs = base[dataset][str(image_size)]
else:
vecs = extract_vectors(net, images, image_size, transform_base)
if dataset not in base:
base[dataset] = {}
base[dataset][str(image_size)] = vecs
fname = args.network_path.replace("/", "_") + ".pkl"
with open(f"base/{fname}", "wb") as f:
pickle.dump(base, f)
print(">> {}: query images...".format(dataset))
qvecs = extract_vectors(net, qimages, image_size, transform_query,
bbxs)
print(">> {}: Evaluating...".format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw["m"], Lw["P"])
qvecs_lw = whitenapply(qvecs, Lw["m"], Lw["P"])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
r = compute_map_and_print(dataset + " + whiten", ranks, cfg["gnd"])
attack_result[dataset] = r
print(">> {}: elapsed time: {}".format(dataset,
htime(time.time() - start)))
return inv_gfr(
attack_result,
baseline_result[net.meta["architecture"]][net.meta["pooling"]])
net_params = {'local_whitening':False,'regional':False,'whitening':False,'pretrained':True} # default cir network params
train_networks = []
for s in args.modellist.split("+"):
net_params['architecture'] = s.split("-")[0]
train_networks.append(init_network(net_params))
if mode == 'global': train_networks[-1].poolattack = s.split("-")[1:]
for n in train_networks: n.eval(); n.cuda();
# output name
if mode == 'global': exp_name = dataset+"_"+("+".join([n.meta['architecture']+"-"+"-".join(n.poolattack) for n in train_networks]))
else: exp_name = dataset+"_"+("+".join([n.meta['architecture']+"-"+mode for n in train_networks]))
if len(variant): variant= "+"+variant
exp_name+= "+"+str(train_scales).replace(" ","")+"+iter"+str(iters)+"+lr"+str(lr)+"+lam"+str(lam)+"+sigmablur"+str(sigma_blur)+"_"+carrier_fn+variant
# dataset config
cfg = configdataset(dataset, datasets_folder)
if dataset.startswith('holidays') or dataset.startswith('copydays'): cfg['nq'] = 50 # hard code holidays and copydays queries to first 50
if 'bbx' in cfg['gnd'][0].keys(): bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])] # bounding boxes for roxford5k and rparis6k datasets
else: bbxs = None
im_size = {'roxford5k':1024, 'rparis6k':1024, 'holidays':1024, 'copydays':1024} # original image size
scale_factors = [x / im_size[dataset] for x in train_scales] # compute relative re-scaling factors
# log file
log = open(output_folder+"/log_"+exp_name+".txt", 'a')
# save folder for attacks
if not os.path.exists(output_folder+"/"+exp_name):
os.makedirs(output_folder+"/"+exp_name)
# params for rerun if no converge
max_trials, multiply_rate_iters, divide_rate_lr = 10, 2, 5
def main():
args = parser.parse_args()
# check if test dataset are downloaded
# and download if they are not
#download_train(get_data_root())
#download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
state = torch.load(args.network_path)
net = init_network(model=state['meta']['architecture'],
pooling=state['meta']['pooling'],
whitening=state['meta']['whitening'],
mean=state['meta']['mean'],
std=state['meta']['std'],
pretrained=False)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
offtheshelf = args.network_offtheshelf.split('-')
if len(offtheshelf) == 3:
if offtheshelf[2] == 'whiten':
offtheshelf_whiten = True
else:
raise (RuntimeError(
"Incorrect format of the off-the-shelf network. Examples: resnet101-gem | resnet101-gem-whiten"
))
else:
offtheshelf_whiten = False
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(model=offtheshelf[0],
pooling=offtheshelf[1],
whitening=offtheshelf_whiten)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = [1]
msp = 1
if args.multiscale:
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if args.multiscale:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
args.image_size,
transform,
bbxs=bbxs,
ms=ms,
msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
def main():
args = parser.parse_args()
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
net = load_network(args.network_path)
# loading offtheshelf network
elif args.network_offtheshelf is not None:
net = load_offtheshelf(args.network_offtheshelf)
# setting up the multi-scale parameters
ms = [1]
msp = 1
if args.multiscale:
ms = [1, 1./math.sqrt(2), 1./2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
if args.multiscale:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
print('>> {}: Learning whitening...'.format(args.whitening))
if args.whitening == "scores":
# special logic for scores database
from score_retrieval.exports import (
db,
train_images as images,
)
else:
# loading db
db_root = os.path.join(get_data_root(), 'train', args.test_whiten)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.test_whiten))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time()-start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
if dataset == "scores":
# Special added logic to handle loading our score dataset
from score_retrieval.exports import (
images,
qimages,
gnd,
)
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, ms=ms, msp=msp)
else:
# extract ground truth
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
gnd = cfg['gnd']
# prepare config structure for the test dataset
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
bbxs = [tuple(gnd[i]['bbx']) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp)
# validation
print(">> {}: gnd stats: {}, {}, {}".format(
dataset,
len(gnd),
[len(x["ok"]) for x in gnd[10:]],
[len(x["junk"]) for x in gnd[10:]],
))
print(">> {}: image stats: {}, {}".format(dataset, len(images), len(qimages)))
assert len(gnd) == len(qimages), (len(gnd), len(qimages))
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
print(">> {}: qvecs.shape: {}".format(dataset, qvecs.shape))
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
print(">> {}: ranks (shape {}) head: {}".format(dataset, ranks.shape, ranks[10:,10:]))
print(">> {}: gnd head: {}".format(dataset, gnd[5:]))
# Compute and print metrics
compute_acc(ranks, gnd, dataset)
compute_mrr(ranks, gnd, dataset)
compute_map_and_print(dataset, ranks, gnd)
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
compute_acc(ranks, gnd, dataset + " + whiten")
compute_mrr(ranks, gnd, dataset + " + whiten")
compute_map_and_print(dataset + " + whiten", ranks, gnd)
print('>> {}: elapsed time: {}'.format(dataset, htime(time.time()-start)))
def main():
args = parser.parse_args()
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
using_cdvs = float(args.using_cdvs)
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path], model_dir=os.path.join(get_data_root(), 'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get('local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta['regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(
mean=net.meta['mean'],
std=net.meta['std']
)
transform = transforms.Compose([
transforms.ToTensor(),
normalize
])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
if len(ms) > 1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train', args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root, '{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [cid2filename(db['cids'][i], ims_root) for i in range(len(db['cids']))]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening, htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
result_dir=args.network_path[0:-8]
epoch_lun=args.network_path[0:-8].split('/')[-1].replace('model_epoch','')
print(">> Creating directory if it does not exist:\n>> '{}'".format(result_dir))
if not os.path.exists(result_dir):
os.makedirs(result_dir)
for dataset in datasets:
start = time.time()
# search, rank, and print
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
tuple_bbxs_qimlist=None
tuple_bbxs_imlist=None
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
# extract database and query vectors
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=tuple_bbxs_qimlist, ms=ms, msp=msp, batchsize=1)
qvecs = qvecs.numpy()
qvecs = qvecs.astype(np.float32)
np.save(os.path.join(result_dir, "{}_qvecs_ep{}_resize.npy".format(dataset,epoch_lun)), qvecs)
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, bbxs=tuple_bbxs_imlist, msp=msp, batchsize=1)
vecs = vecs.numpy()
vecs = vecs.astype(np.float32)
np.save(os.path.join(result_dir, "{}_vecs_ep{}_resize.npy".format(dataset,epoch_lun)), vecs)
scores = np.dot(vecs.T, qvecs)
if using_cdvs!=0:
print('>> {}: cdvs global descriptor loading...'.format(dataset))
qvecs_global = cfg['qimlist_global']
vecs_global = cfg['imlist_global']
scores_global = np.dot(vecs_global, qvecs_global.T)
scores+=scores_global*using_cdvs
ranks = np.argsort(-scores, axis=0)
if args.ir_remove!='0':
rank_len=10
rank_re = np.loadtxt(os.path.join(result_dir, '{}_ranks_new_relevent.txt'.format(dataset)))
## the max value of rank_len
MAX_RANK_LEN = int((rank_re.shape[0]) ** 0.5)
rank_re=rank_re.reshape(MAX_RANK_LEN,MAX_RANK_LEN,rank_re.shape[1])
for m in range(rank_re.shape[2]):
for i in range(rank_re.shape[0]):
rank_re[i][i][m]=1.0
quanzhong=[1,0.7,0.4]+[0.1]*(MAX_RANK_LEN-3)
for m in range(rank_re.shape[2]):
#if adaption, then change the rank_len to a adaption params according to the rank_re_q, q_aer, cons_n
if args.ir_adaption:
using_local_query=True
cons_n = 5
q_aer = float(args.ir_adaption)
if using_local_query:
## using local feature scores, please don't forget note the query_q belong to deep
rank_re_q = np.loadtxt(os.path.join(result_dir, '{}_ranks_new_query.txt'.format(dataset)))
query_q = rank_re_q[:, m]
else:
## using deep feature scores
query_q = scores[ranks[:, m], m]
rank_len=0
jishu=0
for idx in range(min(len(query_q),MAX_RANK_LEN)-cons_n):
if jishu<cons_n:
if query_q[idx]>q_aer:
rank_len=idx+1
else:
jishu+=1
else:
break
max_dim = min(rank_len, MAX_RANK_LEN)
print (max_dim)
if max_dim>2:
#put the image to the MAX_RANK_LEN2 location if equals max_dim then re rank in the maxdim length
list2 = []
list_hou = []
MAX_RANK_LEN2 = max_dim
for i in range(MAX_RANK_LEN2):
if i < max_dim:
fenshu = 0
for j in range(max_dim):
fenshu+=rank_re[min(i,j)][max(i,j)][m]*quanzhong[j]
fenshu = fenshu / (max_dim - 1)
if fenshu > float(args.ir_remove):
list2.append(ranks[i][m])
else:
list_hou.append(ranks[i][m])
else:
list2.append(ranks[i][m])
ranks[0:MAX_RANK_LEN2, m] = list2 + list_hou
np.savetxt(os.path.join(result_dir, "{}_ranks.txt".format(dataset)), ranks.astype(np.int))
if dataset == 'cdvs_test_retrieval':
compute_map_and_print(dataset, ranks, cfg['gnd_id'])
else:
compute_map_and_print(dataset, ranks, cfg['gnd'])
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError(
'Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
#download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get(
'local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if len(ms) > 1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(
args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train',
args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
"""
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg,i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg,i) for i in range(cfg['nq'])]
try:
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
except:
bbxs = None # for holidaysmanrot and copydays
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net, images, args.image_size, transform, ms=ms, msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net, qimages, args.image_size, transform, bbxs=bbxs, ms=ms, msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
print (vecs.shape)
print (qvecs.shape)
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
print (scores.shape)
# to save scores (single query)
# oxford
#f = 'oxf_single.npy'
# paris
#f = 'par_single.npy'
# roxford
#f = 'roxf_single.npy'
# rparis
f = 'rpar_single.npy'
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset, ranks, cfg['gnd'])
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
# save
np.save(f, scores)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
"""
############################################################
# Test
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
print(qimages)
# to load scores
# oxford
#f = 'oxf_single.npy'
#f = 'oxf_mq_avg.npy'
#f = 'oxf_mq_max.npy'
#f = 'oxf_sc_imf.npy'
# paris
#f = 'par_single.npy'
#f = 'par_mq_avg.npy'
#f = 'par_mq_max.npy'
f = 'par_sc_imf.npy'
# roxford
#f = 'roxf_single.npy'
#f = 'roxf_mq_avg.npy'
#f = 'roxf_mq_max.npy'
#f = 'roxf_sc_imf.npy'
# rparis
#f = 'rpar_single.npy'
#f = 'rpar_mq_avg.npy'
#f = 'rpar_mq_max.npy'
#f = 'rpar_sc_imf.npy'
# load
scores = np.load(f)
ranks = np.argsort(-scores, axis=0)
compute_map_and_print(dataset + ' + whiten', ranks, cfg['gnd'])
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get('local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
#
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
#config groundtruth
cfg = configdataset('oxford5k', os.path.join(get_data_root(), 'test'))
#parameter
normalize = transforms.Normalize(mean=net.meta['mean'], std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
ms = list(eval('[1, 1/2**(1/2), 1/2]'))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
def testOxfordParisHolidays(net, eConfig):
#datasets = eConfig['test-datasets'].split(',')
#results = []
#
#for dataset in datasets:
# results.append((dataset, np.random.rand(1)[0]))
#
#return results
print('>> Evaluating network on test datasets...')
# for testing we use image size of max 1024
image_size = 1024
# setting up the multi-scale parameters
ms = [1]
msp = 1
if (eConfig['multiscale']):
ms = [1, 1. / math.sqrt(2), 1. / 2]
if net.meta['pooling'] == 'gem' and net.whiten is None:
msp = net.pool.p.data.tolist()[0]
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if eConfig['whitening']:
start = time.time()
print('>> {}: Learning whitening...'.format(eConfig['test-whiten']))
# loading db
db_root = os.path.join(get_data_root(), 'train',
eConfig['test-whiten'])
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(eConfig['test-whiten']))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(eConfig['test-whiten']))
wvecs = extract_vectors(net,
images,
image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(eConfig['test-whiten']))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
print('>> {}: elapsed time: {}'.format(eConfig['test-whiten'],
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = eConfig['test-datasets'].split(',')
results = []
for dataset in datasets:
start = time.time()
if (dataset != 'holidays' and dataset != 'rholidays'):
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
if (dataset == 'oxford105k' or dataset == 'paris106k'):
images.extend(cfg['distractors'])
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images,
image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages,
image_size,
transform,
bbxs,
ms=ms,
msp=msp)
print('>> {}: Evaluating...'.format(dataset))
# convert to numpy
vecs = vecs.numpy()
qvecs = qvecs.numpy()
# search, rank, and print
scores = np.dot(vecs.T, qvecs)
ranks = np.argsort(-scores, axis=0)
results.append(
compute_map_and_print(
dataset +
('+ multiscale' if eConfig['multiscale'] else ''), ranks,
cfg['gnd']))
if Lw is not None:
# whiten the vectors
vecs_lw = whitenapply(vecs, Lw['m'], Lw['P'])
qvecs_lw = whitenapply(qvecs, Lw['m'], Lw['P'])
# search, rank, and print
scores = np.dot(vecs_lw.T, qvecs_lw)
ranks = np.argsort(-scores, axis=0)
results.append(
compute_map_and_print(dataset + ' + whiten', ranks,
cfg['gnd']))
else:
results.append(testHolidays(net, eConfig, dataset, Lw))
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
return results
def main():
args = parser.parse_args()
# check if there are unknown datasets
for dataset in args.datasets.split(','):
if dataset not in datasets_names:
raise ValueError(
'Unsupported or unknown dataset: {}!'.format(dataset))
# check if test dataset are downloaded
# and download if they are not
download_train(get_data_root())
download_test(get_data_root())
# setting up the visible GPU
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu_id
# loading network from path
if args.network_path is not None:
print(">> Loading network:\n>>>> '{}'".format(args.network_path))
if args.network_path in PRETRAINED:
# pretrained networks (downloaded automatically)
state = load_url(PRETRAINED[args.network_path],
model_dir=os.path.join(get_data_root(),
'networks'))
else:
# fine-tuned network from path
state = torch.load(args.network_path)
# parsing net params from meta
# architecture, pooling, mean, std required
# the rest has default values, in case that is doesnt exist
net_params = {}
net_params['architecture'] = state['meta']['architecture']
net_params['pooling'] = state['meta']['pooling']
net_params['local_whitening'] = state['meta'].get(
'local_whitening', False)
net_params['regional'] = state['meta'].get('regional', False)
net_params['whitening'] = state['meta'].get('whitening', False)
net_params['mean'] = state['meta']['mean']
net_params['std'] = state['meta']['std']
net_params['pretrained'] = False
# load network
net = init_network(net_params)
net.load_state_dict(state['state_dict'])
# if whitening is precomputed
if 'Lw' in state['meta']:
net.meta['Lw'] = state['meta']['Lw']
print(">>>> loaded network: ")
print(net.meta_repr())
# loading offtheshelf network
elif args.network_offtheshelf is not None:
# parse off-the-shelf parameters
offtheshelf = args.network_offtheshelf.split('-')
net_params = {}
net_params['architecture'] = offtheshelf[0]
net_params['pooling'] = offtheshelf[1]
net_params['local_whitening'] = 'lwhiten' in offtheshelf[2:]
net_params['regional'] = 'reg' in offtheshelf[2:]
net_params['whitening'] = 'whiten' in offtheshelf[2:]
net_params['pretrained'] = True
# load off-the-shelf network
print(">> Loading off-the-shelf network:\n>>>> '{}'".format(
args.network_offtheshelf))
net = init_network(net_params)
print(">>>> loaded network: ")
print(net.meta_repr())
# setting up the multi-scale parameters
ms = list(eval(args.multiscale))
if len(ms) > 1 and net.meta['pooling'] == 'gem' and not net.meta[
'regional'] and not net.meta['whitening']:
msp = net.pool.p.item()
print(">> Set-up multiscale:")
print(">>>> ms: {}".format(ms))
print(">>>> msp: {}".format(msp))
else:
msp = 1
# moving network to gpu and eval mode
net.cuda()
net.eval()
# set up the transform
normalize = transforms.Normalize(mean=net.meta['mean'],
std=net.meta['std'])
transform = transforms.Compose([transforms.ToTensor(), normalize])
# compute whitening
if args.whitening is not None:
start = time.time()
if 'Lw' in net.meta and args.whitening in net.meta['Lw']:
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
if len(ms) > 1:
Lw = net.meta['Lw'][args.whitening]['ms']
else:
Lw = net.meta['Lw'][args.whitening]['ss']
else:
# if we evaluate networks from path we should save/load whitening
# not to compute it every time
if args.network_path is not None:
whiten_fn = args.network_path + '_{}_whiten'.format(
args.whitening)
if len(ms) > 1:
whiten_fn += '_ms'
whiten_fn += '.pth'
else:
whiten_fn = None
if whiten_fn is not None and os.path.isfile(whiten_fn):
print('>> {}: Whitening is precomputed, loading it...'.format(
args.whitening))
Lw = torch.load(whiten_fn)
else:
print('>> {}: Learning whitening...'.format(args.whitening))
# loading db
db_root = os.path.join(get_data_root(), 'train',
args.whitening)
ims_root = os.path.join(db_root, 'ims')
db_fn = os.path.join(db_root,
'{}-whiten.pkl'.format(args.whitening))
with open(db_fn, 'rb') as f:
db = pickle.load(f)
images = [
cid2filename(db['cids'][i], ims_root)
for i in range(len(db['cids']))
]
# extract whitening vectors
print('>> {}: Extracting...'.format(args.whitening))
wvecs = extract_vectors(net,
images,
args.image_size,
transform,
ms=ms,
msp=msp)
# learning whitening
print('>> {}: Learning...'.format(args.whitening))
wvecs = wvecs.numpy()
m, P = whitenlearn(wvecs, db['qidxs'], db['pidxs'])
Lw = {'m': m, 'P': P}
# saving whitening if whiten_fn exists
if whiten_fn is not None:
print('>> {}: Saving to {}...'.format(
args.whitening, whiten_fn))
torch.save(Lw, whiten_fn)
print('>> {}: elapsed time: {}'.format(args.whitening,
htime(time.time() - start)))
else:
Lw = None
# evaluate on test datasets
datasets = args.datasets.split(',')
for dataset in datasets:
start = time.time()
print('>> {}: Extracting...'.format(dataset))
# prepare config structure for the test dataset
cfg = configdataset(dataset, os.path.join(get_data_root(), 'test'))
images = [cfg['im_fname'](cfg, i) for i in range(cfg['n'])]
qimages = [cfg['qim_fname'](cfg, i) for i in range(cfg['nq'])]
try:
bbxs = [tuple(cfg['gnd'][i]['bbx']) for i in range(cfg['nq'])]
except:
bbxs = None # for holidaysmanrot and copydays
#自定义的抽取
names = []
images_path = []
qimages_path = []
#运行时候要更改的参数
L = 2
# prepath="E:\\PycharmProjects\\image-retrieval\\data\\test\\rparis6k\\jpg\\"
prepath = "E:\\PycharmProjects\\image-retrieval\\ox1\\"
for i, img_path in enumerate(images):
img_name = os.path.split(img_path)[1]
print(img_name)
if L > 1:
img_name = os.path.splitext(img_name)[0]
for i in range(1, L + 1):
for j in range(1, L + 1):
names.append(img_name + "_" + str(i - 1) + str(j - 1) +
".jpg")
images_path.append(prepath + img_name + "_" +
str(i - 1) + str(j - 1) + ".jpg")
else:
names.append(img_name)
images_path.append(prepath + img_name)
qnames = []
print("-----------query images-----------")
for i, img_path in enumerate(qimages):
img_name = os.path.split(img_path)[1]
print(img_name)
if L > 1:
img_name = os.path.splitext(img_name)[0]
for i in range(1, L + 1):
for j in range(1, L + 1):
qnames.append(img_name + "_" + str(i - 1) +
str(j - 1) + ".jpg")
qimages_path.append(prepath + img_name + "_" +
str(i - 1) + str(j - 1) + ".jpg")
else:
qnames.append(img_name)
qimages_path.append(prepath + img_name)
# extract database and query vectors
print('>> {}: database images...'.format(dataset))
vecs = extract_vectors(net,
images_path,
args.image_size,
transform,
ms=ms,
msp=msp)
print('>> {}: query images...'.format(dataset))
qvecs = extract_vectors(net,
qimages_path,
args.image_size,
transform,
ms=ms,
msp=msp)
# convert to numpy
vecs = vecs.numpy()
vecs = vecs.T
qvecs = qvecs.numpy()
qvecs = qvecs.T
print("--------------------------------------------------")
print(" writing feature extraction results ...")
print("--------------------------------------------------")
output = "gem_res_rox_2.h5"
h5f = h5py.File(output, 'w')
h5f.create_dataset('dataset_1', data=vecs)
h5f.create_dataset('dataset_2', data=np.string_(names))
h5f.create_dataset('dataset_3', data=qvecs)
h5f.create_dataset('dataset_4', data=np.string_(qnames))
h5f.close()
print('>> {}: elapsed time: {}'.format(dataset,
htime(time.time() - start)))
