def evaluate(qf,ql,qc,gf,gl,gc,type='euclidean'):
qf = qf[np.newaxis,:]
# qf.shape = (1,2048)
# gf.shape = (19732,2048)
score = compute_dist(qf, gf, type=type) # cosine
# predict index
score = np.squeeze(score,axis=0) # score.shape = (19732,)
index = np.argsort(score) # from small to large
if type == 'cosine':
index = index[::-1] # from large to small
# good index
query_index = np.argwhere(gl==ql)
camera_index = np.argwhere(gc==qc)
good_index = np.setdiff1d(query_index, camera_index, assume_unique=True)
junk_index1 = np.argwhere(gl==-1) # junk bboxs index
junk_index2 = np.intersect1d(query_index, camera_index)
junk_index = np.append(junk_index2, junk_index1)
CMC_tmp = compute_mAP(index, good_index, junk_index)
return CMC_tmp
def eval(self,
normalize_feat=True,
to_re_rank=True,
pool_type='average',
verbose=True):
"""Evaluate using metric CMC and mAP.
Args:
normalize_feat: whether to normalize features before computing distance
to_re_rank: whether to also report re-ranking scores
pool_type: 'average' or 'max', only for multi-query case
verbose: whether to print the intermediate information
"""
with measure_time('Extracting feature...', verbose=verbose):
feat, ids, cams, im_names, marks = self.extract_feat(
normalize_feat, verbose)
# query, gallery, multi-query indices
q_inds = marks == 0
g_inds = marks == 1
mq_inds = marks == 2
# A helper function just for avoiding code duplication.
def compute_score(dist_mat,
query_ids=ids[q_inds],
gallery_ids=ids[g_inds],
query_cams=cams[q_inds],
gallery_cams=cams[g_inds]):
# Compute mean AP
mAP = mean_ap(distmat=dist_mat,
query_ids=query_ids,
gallery_ids=gallery_ids,
query_cams=query_cams,
gallery_cams=gallery_cams)
# Compute CMC scores
cmc_scores = cmc(distmat=dist_mat,
query_ids=query_ids,
gallery_ids=gallery_ids,
query_cams=query_cams,
gallery_cams=gallery_cams,
separate_camera_set=self.separate_camera_set,
single_gallery_shot=self.single_gallery_shot,
first_match_break=self.first_match_break,
topk=10)
return mAP, cmc_scores
def print_scores(mAP, cmc_scores):
print(
'[mAP: {:5.2%}], [cmc1: {:5.2%}], [cmc5: {:5.2%}], [cmc10: {:5.2%}]'
.format(mAP, *cmc_scores[[0, 4, 9]]))
################
# Single Query #
################
with measure_time('Computing distance...', verbose=verbose):
# query-gallery distance
q_g_dist = compute_dist(feat[q_inds],
feat[g_inds],
type='euclidean')
with measure_time('Computing scores...', verbose=verbose):
mAP, cmc_scores = compute_score(q_g_dist)
print('{:<30}'.format('Single Query:'), end='')
print_scores(mAP, cmc_scores)
###############
# Multi Query #
###############
mq_mAP, mq_cmc_scores = None, None
if any(mq_inds):
mq_ids = ids[mq_inds]
mq_cams = cams[mq_inds]
mq_feat = feat[mq_inds]
unique_mq_ids_cams = defaultdict(list)
for ind, (id, cam) in enumerate(zip(mq_ids, mq_cams)):
unique_mq_ids_cams[(id, cam)].append(ind)
keys = unique_mq_ids_cams.keys()
assert pool_type in ['average', 'max']
pool = np.mean if pool_type == 'average' else np.max
mq_feat = np.stack(
[pool(mq_feat[unique_mq_ids_cams[k]], axis=0) for k in keys])
with measure_time('Multi Query, Computing distance...',
verbose=verbose):
# multi_query-gallery distance
mq_g_dist = compute_dist(mq_feat,
feat[g_inds],
type='euclidean')
with measure_time('Multi Query, Computing scores...',
verbose=verbose):
mq_mAP, mq_cmc_scores = compute_score(
mq_g_dist,
query_ids=np.array(zip(*keys)[0]),
gallery_ids=ids[g_inds],
query_cams=np.array(zip(*keys)[1]),
gallery_cams=cams[g_inds])
print('{:<30}'.format('Multi Query:'), end='')
print_scores(mq_mAP, mq_cmc_scores)
if to_re_rank:
##########################
# Re-ranked Single Query #
##########################
with measure_time('Re-ranking distance...', verbose=verbose):
# query-query distance
q_q_dist = compute_dist(feat[q_inds],
feat[q_inds],
type='euclidean')
# gallery-gallery distance
g_g_dist = compute_dist(feat[g_inds],
feat[g_inds],
type='euclidean')
# re-ranked query-gallery distance
re_r_q_g_dist = re_ranking(q_g_dist, q_q_dist, g_g_dist)
with measure_time('Computing scores for re-ranked distance...',
verbose=verbose):
mAP, cmc_scores = compute_score(re_r_q_g_dist)
print('{:<30}'.format('Re-ranked Single Query:'), end='')
print_scores(mAP, cmc_scores)
#########################
# Re-ranked Multi Query #
#########################
if any(mq_inds):
with measure_time('Multi Query, Re-ranking distance...',
verbose=verbose):
# multi_query-multi_query distance
mq_mq_dist = compute_dist(mq_feat,
mq_feat,
type='euclidean')
# re-ranked multi_query-gallery distance
re_r_mq_g_dist = re_ranking(mq_g_dist, mq_mq_dist,
g_g_dist)
with measure_time(
'Multi Query, Computing scores for re-ranked distance...',
verbose=verbose):
mq_mAP, mq_cmc_scores = compute_score(
re_r_mq_g_dist,
query_ids=np.array(zip(*keys)[0]),
gallery_ids=ids[g_inds],
query_cams=np.array(zip(*keys)[1]),
gallery_cams=cams[g_inds])
print('{:<30}'.format('Re-ranked Multi Query:'), end='')
print_scores(mq_mAP, mq_cmc_scores)
return mAP, cmc_scores, mq_mAP, mq_cmc_scores
def test(sess, rerank=True):
def print_scores(mAP, cmc_scores):
print(
'[mAP: {:5.2%}], [cmc1: {:5.2%}], [cmc5: {:5.2%}], [cmc10: {:5.2%}]'
.format(mAP, *cmc_scores[[0, 4, 9]]))
q_featfile = q_testfeat
g_featfile = g_testfeat
if args.feat_ext:
feature_extract(sess, q_featfile, g_featfile)
q_info = np.genfromtxt(q_featfile, delimiter=',', skip_header=1)
g_info = np.genfromtxt(g_featfile, delimiter=',', skip_header=1)
print('shape of q_info2: ', np.shape(q_info[:, 3:]))
print('shape of q_info2: ', np.shape(g_info[:, 3:]))
if args.normalize_feat:
q_feat = normalize(q_info[:, 3:], axis=1)
g_feat = normalize(g_info[:, 3:], axis=1)
else:
q_feat = q_info[:, 3:]
g_feat = g_info[:, 3:]
################
# Single Query #
################
# query-gallery distance
print('Computing distance...')
st = time.time()
q_g_dist = compute_dist(q_feat, g_feat, type='euclidean')
print('Done, {:.2f}s, shape of q_g_dist: {}'.format(
time.time() - st, np.shape(q_g_dist)))
print('Computing scores...')
st = time.time()
mAP, cmc_scores = compute_score(q_g_dist,
query_ids=q_info[:, 1],
gallery_ids=g_info[:, 1],
query_cams=q_info[:, 2],
gallery_cams=g_info[:, 2])
print('Done, {:.2f}s'.format(time.time() - st))
print('{:<30}'.format('Single Query:'), end='')
print_scores(mAP, cmc_scores)
if not rerank:
return mAP, cmc_scores
##########################
# Re-ranked Single Query #
##########################
print('Computing Re-ranking distance...')
st = time.time()
# query-query distance
q_q_dist = compute_dist(q_feat, q_feat, type='euclidean')
print('shape of q_q_dist: ', np.shape(q_q_dist))
# gallery-gallery distance
g_g_dist = compute_dist(g_feat, g_feat, type='euclidean')
print('shape of g_g_dist: ', np.shape(g_g_dist))
# re-ranked query-gallery distance
re_r_q_g_dist = re_ranking(q_g_dist, q_q_dist, g_g_dist)
print('shape of re_r_q_g_dist: ', np.shape(re_r_q_g_dist))
print('Done, {:.2f}s'.format(time.time() - st))
print('Computing scores for re-ranked distance...')
st = time.time()
remAP, recmc_scores = compute_score(re_r_q_g_dist,
query_ids=q_info[:, 1],
gallery_ids=g_info[:, 1],
query_cams=q_info[:, 2],
gallery_cams=g_info[:, 2])
print('Done, {:.2f}s'.format(time.time() - st))
print('{:<30}'.format('Re-ranked Single Query:'), end='')
print_scores(remAP, recmc_scores)
return mAP, cmc_scores, remAP, recmc_scores
def SV(feature_path, result_path, plot=False):
queryfeatpath = feature_path + '/query.csv'
query = readfeatures(queryfeatpath, args.normalize_feat)
q_feat = query[:, 3:]
q_ids = query[:, 1].astype(np.int)
galleryfeatpath = feature_path + '/gallery.csv'
gallery = readfeatures(galleryfeatpath, args.normalize_feat)
g_feat = gallery[:, 3:]
g_ids = gallery[:, 1].astype(np.int)
resultfile = result_path
if not os.path.exists(os.path.dirname(resultfile)):
os.makedirs(os.path.dirname(resultfile))
f = open(resultfile, "w", newline='')
w = csv.writer(f)
head = ["Threshold", "TTR", "FTR"]
w.writerow(head)
ingalleryID = list(np.unique(g_ids))
q_g_dist = compute_dist(q_feat, g_feat, type='euclidean')
if args.rerank:
##########################
# Re-ranked Single Query #
##########################
print('Computing Re-ranking distance...')
# st = time.time()
# gallery-gallery distance
g_g_dist = compute_dist(g_feat, g_feat, type='cosine')
# print('shape of g_g_dist: ', np.shape(g_g_dist))
for i in range(len(q_feat)):
temp_q_feat = np.expand_dims(q_feat[i], axis=0)
q_q_dist = compute_dist(temp_q_feat, temp_q_feat, type='euclidean')
# print('shape of q_q_dist: ', np.shape(q_q_dist))
# re-ranked query-gallery distance
q_g_dist[i] = re_ranking(np.expand_dims(q_g_dist[i], axis=0), q_q_dist, g_g_dist, k1=10, k2=5)
# query-query distance
# similar_idx = np.argmin(q_g_dist, axis=1)
similar_dist = np.min(q_g_dist, axis=1)
# ed = time.time()
# print("%f" % (ed-st))
TTR_list = []
FTR_list = []
TTR_to_FTR = np.zeros(6, dtype=np.float)
print("Start SV evaluating: {}".format(os.path.dirname(queryfeatpath)))
for t in range(0, 2000):
threshold = t / 1000.0
match_mask = similar_dist <= threshold
# ed = time.clock()
# print("{:.6f}".format(ed - st))
inidmask = np.in1d(q_ids, ingalleryID)
TQ = float(len(q_ids[inidmask]))
NTQ = float(len(q_ids[np.logical_not(inidmask)]))
matched_q = q_ids[match_mask]
matached_inidmask = np.in1d(matched_q, ingalleryID)
TTQ = float(list(matached_inidmask).count(True))
FNTQ = float(list(matached_inidmask).count(False))
TTR = TTQ / TQ * 100.0
FTR = FNTQ / NTQ * 100.0
# print("Threshold: {}, TTR: {:.3f}, FTR: {:.3f}".format(threshold, TTR, FTR))
w.writerow([threshold, TTR, FTR])
f.flush()
TTR_list.append(TTR)
FTR_list.append(FTR)
if FTR <= 0.1:
TTR_to_FTR[0] = TTR
elif FTR <= 1.0:
TTR_to_FTR[1] = TTR
elif FTR <= 5.0:
TTR_to_FTR[2] = TTR
elif FTR <= 10.0:
TTR_to_FTR[3] = TTR
elif FTR <= 20.0:
TTR_to_FTR[4] = TTR
elif FTR <= 30.0:
TTR_to_FTR[5] = TTR
f.close()
if plot:
graphname = 'gallery_' + args.dataset + "_SV_%d" % (i)
plotcurve(graphname, TTR_list, FTR_list)
print()
print("Test SV: %s" % (resultfile))
print("==================================")
print("FTR:\t0.1,\t1,\t5,\t10,\t20,\t30")
print("TTR:\t{:.2f},\t{:.2f},\t{:.2f}, \t{:.2f},\t{:.2f},\t{:.2f}\n\n".format(*TTR_to_FTR.tolist()))
return TTR_to_FTR
def IV(feature_path, result_path, plot=False):
queryfeatpath = feature_path + '/query.csv'
query = readfeatures(queryfeatpath, args.normalize_feat)
q_feat = query[:, 3:]
q_ids = query[:, 1].astype(np.int)
galleryfeatpath = feature_path + '/gallery.csv'
gallery = readfeatures(galleryfeatpath, args.normalize_feat)
g_feat = gallery[:, 3:]
g_ids = gallery[:, 1].astype(np.int)
resultfile = result_path
if not os.path.exists(os.path.dirname(resultfile)):
os.makedirs(os.path.dirname(resultfile))
f = open(resultfile, "w", newline='')
w = csv.writer(f)
head = ["Threshold", "TTR", "FTR"]
w.writerow(head)
ingalleryID = list(np.unique(g_ids))
q_g_dist = compute_dist(q_feat, g_feat, type='euclidean')
if args.rerank:
##########################
# Re-ranked Single Query #
##########################
print('Computing Re-ranking distance...')
# st = time.time()
# gallery-gallery distance
g_g_dist = compute_dist(g_feat, g_feat, type='euclidean')
# print('shape of g_g_dist: ', np.shape(g_g_dist))
for i in range(len(q_feat)):
temp_q_feat = np.expand_dims(q_feat[i], axis=0)
q_q_dist = compute_dist(temp_q_feat, temp_q_feat, type='euclidean')
# print('shape of q_q_dist: ', np.shape(q_q_dist))
# re-ranked query-gallery distance
q_g_dist[i] = re_ranking(np.expand_dims(q_g_dist[i], axis=0), q_q_dist, g_g_dist, k1=10, k2=5)
# query-query distance
TTR_list = []
FTR_list = []
TTR_to_FTR = np.zeros(6, dtype=np.float)
print("Start IV evaluating: {}".format(os.path.dirname(queryfeatpath)))
for t in range(0, 2000):
threshold = t / 1000.0
iv_TTR = np.zeros(len(ingalleryID), dtype=np.float)
iv_FTR = np.zeros(len(ingalleryID), dtype=np.float)
for j, id in enumerate(ingalleryID):
g_mask = g_ids == id
onlyg_q_dist = q_g_dist[:, g_mask]
similar_dist = np.min(onlyg_q_dist, axis=1)
similar_idx = np.argmin(onlyg_q_dist, axis=1)
ingallerymask = q_ids == id
TQ = float(list(ingallerymask).count(True))
NTQ = float(list(ingallerymask).count(False))
ing_dist = similar_dist[ingallerymask]
match_dist = ing_dist <= threshold
TTQ = float(list(match_dist).count(True))
not_ing_dist = similar_dist[np.logical_not(ingallerymask)]
# not_ing_idx = similar_idx[np.logical_not(ingallerymask)]
not_ing_match_dist = not_ing_dist <= threshold
# not_ing_match_dist = np.bitwise_and(not_ing_dist <= threshold, not_ing_pre_idx == id)
FNTQ = float(list(not_ing_match_dist).count(True))
iv_TTR[j] = TTQ / TQ * 100.0
iv_FTR[j] = FNTQ / NTQ * 100.0
TTR = np.mean(iv_TTR)
FTR = np.mean(iv_FTR)
TTR_list.append(TTR)
FTR_list.append(FTR)
# print("Threshold: {}, TTR: {:.3f}, FTR: {:.3f}".format(threshold, TTR, FTR))
w.writerow([threshold, TTR, FTR])
f.flush()
if FTR <= 0.1:
TTR_to_FTR[0] = TTR
elif FTR <= 1.0:
TTR_to_FTR[1] = TTR
elif FTR <= 5.0:
TTR_to_FTR[2] = TTR
elif FTR <= 10.0:
TTR_to_FTR[3] = TTR
elif FTR <= 20.0:
TTR_to_FTR[4] = TTR
elif FTR <= 30.0:
TTR_to_FTR[5] = TTR
f.close()
if plot:
graphname = 'gallery_' + args.dataset + "_IV_%d" % (i)
plotcurve(graphname, TTR_list, FTR_list)
print()
print("Test IV: %s" % (resultfile))
print("==================================")
print("FTR:\t0.1,\t1,\t5,\t10,\t20,\t30")
print("TTR:\t{:.2f},\t{:.2f},\t{:.2f}, \t{:.2f},\t{:.2f},\t{:.2f}\n\n".format(*TTR_to_FTR.tolist()))
return TTR_to_FTR#, np.asarray(TTR_list), np.asarray(FTR_list)
def main():
cfg = Config()
# Redirect logs to both console and file.
if cfg.log_to_file:
ReDirectSTD(cfg.stdout_file, 'stdout', False)
ReDirectSTD(cfg.stderr_file, 'stderr', False)
TVT, TMO = set_devices(cfg.sys_device_ids)
# Dump the configurations to log.
import pprint
print('-' * 60)
print('cfg.__dict__')
pprint.pprint(cfg.__dict__)
print('-' * 60)
###########
# Dataset #
###########
test_set = create_dataset(**cfg.test_set_kwargs)
#########
# Model #
#########
model = Model(cfg.net,
path_to_predefined='',
pretrained=False,
last_conv_stride=cfg.last_conv_stride)
model.cuda()
r'''
This is compeletly useless, but since I used apex, and its optimization level
has different effect on each layer of networ and optimizer is mandatory argument, I created this optimizer.
'''
optimizer = optim.Adam(model.parameters())
model, optimizer = amp.initialize(
model,
optimizer,
opt_level=cfg.opt_level,
#loss_scale=cfg.loss_scale
)
print(model)
# Model wrapper
model_w = DataParallel(model)
# May Transfer Model to Specified Device.
TMO([model])
#####################
# Load Model Weight #
#####################
# To first load weights to CPU
map_location = (lambda storage, loc: storage)
used_file = cfg.model_weight_file or cfg.ckpt_file
loaded = torch.load(used_file, map_location=map_location)
if cfg.model_weight_file == '':
loaded = loaded['state_dicts'][0]
load_state_dict(model, loaded)
print('Loaded model weights from {}'.format(used_file))
###################
# Extract Feature #
###################
test_set.set_feat_func(ExtractFeature(model_w, TVT))
with measure_time('Extracting feature...', verbose=True):
feat, ids, cams, im_names, marks = test_set.extract_feat(True,
verbose=True)
#######################
# Select Query Images #
#######################
# Fix some query images, so that the visualization for different models can
# be compared.
# Sort in the order of image names
inds = np.argsort(im_names)
feat, ids, cams, im_names, marks = \
feat[inds], ids[inds], cams[inds], im_names[inds], marks[inds]
# query, gallery index mask
is_q = marks == 0
is_g = marks == 1
prng = np.random.RandomState(2)
# selected query indices
sel_q_inds = prng.permutation(range(np.sum(is_q)))[:cfg.num_queries]
q_ids = ids[is_q][sel_q_inds]
q_cams = cams[is_q][sel_q_inds]
q_feat = feat[is_q][sel_q_inds]
q_im_names = im_names[is_q][sel_q_inds]
####################
# Compute Distance #
####################
# query-gallery distance
q_g_dist = compute_dist(q_feat, feat[is_g], type='euclidean')
###########################
# Save Rank List as Image #
###########################
q_im_paths = list()
for n in q_im_names:
if isinstance(n, bytes):
n = n.decode("utf-8")
q_im_paths.append(ospj(test_set.im_dir, n))
save_paths = list()
for n in q_im_names:
if isinstance(n, bytes):
n = n.decode("utf-8")
save_paths.append(ospj(cfg.exp_dir, 'rank_lists', n))
g_im_paths = list()
for n in im_names[is_g]:
if isinstance(n, bytes):
n = n.decode("utf-8")
g_im_paths.append(ospj(test_set.im_dir, n))
for dist_vec, q_id, q_cam, q_im_path, save_path in zip(
q_g_dist, q_ids, q_cams, q_im_paths, save_paths):
rank_list, same_id = get_rank_list(dist_vec, q_id, q_cam, ids[is_g],
cams[is_g], cfg.rank_list_size)
save_rank_list_to_im(rank_list, same_id, q_im_path, g_im_paths,
save_path)
lr_2 = piecewise_constant_lr(lr_decay_steps, lr_decay)
lr = lr_1 + lr_2
train_net = TrainStepWrap(loss_net, lr, config.momentum, is_train=False)
load_checkpoint("checkpoints/40.ckpt", net=train_net)
q_feats, q_labels, g_feats, g_labels = [], [], [], []
for data, gt_classes, theta in query_dataset:
output = train_net(data, gt_classes, theta)
output = output.asnumpy()
label = gt_classes.asnumpy()
q_feats.append(output)
q_labels.append(label)
q_feats = np.vstack(q_feats)
q_labels = np.hstack(q_labels)
for data, gt_classes, theta in gallery_dataset:
output = train_net(data, gt_classes, theta)
output = output.asnumpy()
label = gt_classes.asnumpy()
g_feats.append(output)
g_labels.append(label)
g_feats = np.vstack(g_feats)
g_labels = np.hstack(g_labels)
q_g_dist = compute_dist(q_feats, g_feats, dis_type='cosine')
mAP, cmc_scores = compute_score(q_g_dist, q_labels, g_labels)
print(mAP, cmc_scores)