def test_duplicate_ids(self):
distmat = np.tile(np.arange(4), (4, 1))
query_ids = [0, 0, 1, 1]
gallery_ids = [0, 0, 1, 1]
ret = cmc(distmat, query_ids=query_ids, gallery_ids=gallery_ids, topk=4,
separate_camera_set=False, single_gallery_shot=False)
self.assertTrue(np.all(ret == [0.5, 0.5, 1, 1]))
def test_only_distmat(self):
distmat = np.array([[0, 1, 2, 3, 4],
[1, 0, 2, 3, 4],
[0, 1, 2, 3, 4],
[0, 1, 2, 3, 4],
[1, 2, 3, 4, 0]])
ret = cmc(distmat)
self.assertTrue(np.all(ret[:5] == [0.6, 0.6, 0.8, 1.0, 1.0]))
def evaluate_all(distmat, query=None, gallery=None,
query_ids=None, gallery_ids=None,
query_cams=None, gallery_cams=None,
cmc_topk=(1, 5, 10)):
if query is not None and gallery is not None:
query_ids = [pid for _, pid, _ in query]
gallery_ids = [pid for _, pid, _ in gallery]
query_cams = [cam for _, _, cam in query]
gallery_cams = [cam for _, _, cam in gallery]
else:
assert (query_ids is not None and gallery_ids is not None
and query_cams is not None and gallery_cams is not None)
# Compute mean AP
mAP = mean_ap(distmat, query_ids, gallery_ids, query_cams, gallery_cams)
print('Mean AP: {:4.1%}'.format(mAP))
# Compute all kinds of CMC scores
cmc_configs = {
'allshots': dict(separate_camera_set=False,
single_gallery_shot=False,
first_match_break=False),
'cuhk03': dict(separate_camera_set=True,
single_gallery_shot=True,
first_match_break=False),
'market1501': dict(separate_camera_set=False,
single_gallery_shot=False,
first_match_break=True)}
cmc_scores = {name: cmc(distmat, query_ids, gallery_ids,
query_cams, gallery_cams, **params)
for name, params in cmc_configs.items()}
print('CMC Scores{:>12}{:>12}{:>12}'
.format('allshots', 'cuhk03', 'market1501'))
for k in cmc_topk:
print(' top-{:<4}{:12.1%}{:12.1%}{:12.1%}'
.format(k, cmc_scores['allshots'][k - 1],
cmc_scores['cuhk03'][k - 1],
cmc_scores['market1501'][k - 1]))
# Use the allshots cmc top-1 score for validation criterion
return cmc_scores['allshots'][0]
def evaluate(self, data_loader, query, gallery, metric=None, **kwargs):
self.model.eval()
query_ps = [path for path, _, _ in query]
gallery_ps = [path for path, _, _ in gallery]
query_ids = [pid for _, pid, _ in query]
gallery_ids = [pid for _, pid, _ in gallery]
query_cams = [cam for _, _, cam in query]
gallery_cams = [cam for _, _, cam in gallery]
query_ids = np.asarray(query_ids)
gallery_ids = np.asarray(gallery_ids)
query_cams = np.asarray(query_cams)
gallery_cams = np.asarray(gallery_cams)
features = extract_features(self.model, data_loader)[0]
assert len(features) != 0
res = {}
# final = kwargs.get('final', False)
if 'prefix' in kwargs:
prefix = kwargs.get('prefix', '') + '/'
else:
prefix = ''
logging.info(f'prefix is {prefix}')
if self.args['rerank']:
rerank_range = [True, False]
else:
rerank_range = [
False,
]
for rerank in rerank_range:
lz.timer.since_last_check(
f'start eval whether to use rerank : {rerank}')
## todo dump feature only
# msgpack_dump([features, query, gallery, ], work_path + 't.pk')
# return 'ok'
if self.conf == 'market1501':
# try:
# ## use faiss
#
# import faiss
#
# xx = torch.cat([features[f].unsqueeze(0) for f, _, _ in query], 0)
# yy = torch.cat([features[f].unsqueeze(0) for f, _, _ in gallery], 0)
# xx = to_numpy(xx)
# yy = to_numpy(yy)
# index = faiss.IndexFlatL2(xx.shape[1])
# index.add(yy)
# _, ranklist = index.search(xx, yy.shape[0])
# # _, ranklist = index.search(xx, 11)
# mAP, all_cmc, all_AP = eval_market1501_wrap(
# np.asarray(ranklist), query_ids, gallery_ids, query_cams,
# gallery_cams,
# max_rank=10, faiss=True)
# lz.timer.since_last_check(f'faiss {mAP}')
# except ImportError as e:
## use NN
distmat, xx, yy = pairwise_distance(features,
query,
gallery,
metric=metric,
rerank=rerank)
print(f'facing {distmat.shape}')
distmat = distmat.astype(np.float16)
mAP, all_cmc, all_AP = eval_market1501_wrap(distmat,
query_ids,
gallery_ids,
query_cams,
gallery_cams,
max_rank=10)
# nmi = NMI(xx, query_ids, n_cluster=200)
# print('mni is ', nmi)
nmi = nmi2(query_ids, xx)
print('mni is ', nmi)
lz.timer.since_last_check(f'NN {mAP}')
msgpack_dump([distmat, xx, yy, query, gallery, all_AP],
work_path + 't.pk') # todo for plot
if rerank:
res = lz.dict_concat([
res, {
'mAP.rk': mAP,
'top-1.rk': all_cmc[0],
'top-5.rk': all_cmc[4],
'top-10.rk': all_cmc[9]
}
])
else:
res = lz.dict_concat([
res,
{
'mAP': mAP
},
{f'top-{ind+1}': v
for ind, v in enumerate(all_cmc)},
])
else:
## use NN & mkt
# distmat, xx, yy = pairwise_distance(features, query, gallery, metric=metric, rerank=rerank)
# distmat = distmat.astype(np.float16)
# mAP, all_cmc, all_AP = eval_market1501_wrap(
# distmat, query_ids, gallery_ids, query_cams, gallery_cams,
# max_rank=10)
# print('--> Mean AP: {:4.1%}'.format(mAP), all_cmc )
## use faiss & mkt
# import faiss
# xx = torch.cat([features[f].unsqueeze(0) for f, _, _ in query], 0)
# yy = torch.cat([features[f].unsqueeze(0) for f, _, _ in gallery], 0)
# xx = to_numpy(xx)
# yy = to_numpy(yy)
# index = faiss.IndexFlatL2(xx.shape[1])
# index.add(yy)
# _, ranklist = index.search(xx, yy.shape[0])
# # _, ranklist = index.search(xx, 11)
# mAP, all_cmc, all_AP = eval_market1501_wrap(
# np.asarray(ranklist), query_ids, gallery_ids, query_cams,
# gallery_cams,
# max_rank=10, faiss=True)
# lz.timer.since_last_check(f'faiss {mAP}')
## use NN & cu03 in fact cu03 should use this
distmat, xx, yy = pairwise_distance(features,
query,
gallery,
metric=metric,
rerank=rerank)
print(f'facing {distmat.shape}')
mAP = mean_ap(distmat, query_ids, gallery_ids, query_cams,
gallery_cams)
cmc_configs = {
'cuhk03':
dict(separate_camera_set=True,
single_gallery_shot=True,
first_match_break=False),
}
cmc_configs = {
k: v
for k, v in cmc_configs.items() if k == self.conf
}
cmc_scores = {
name: cmc(distmat, query_ids, gallery_ids, query_cams,
gallery_cams, **params)
for name, params in cmc_configs.items()
}
print(
f'--> mAP {mAP} cmc-1 {self.conf} {cmc_scores[self.conf][0]} '
)
if rerank:
res = lz.dict_concat([
res, {
'mAP.rk': mAP,
'top-1.rk': cmc_scores[self.conf][0],
'top-5.rk': cmc_scores[self.conf][4],
'top-10.rk': cmc_scores[self.conf][9],
}
])
else:
res = lz.dict_concat([
res, {
'mAP': mAP,
'top-1': cmc_scores[self.conf][0],
'top-5': cmc_scores[self.conf][4],
'top-10': cmc_scores[self.conf][9],
}
])
timer.since_start()
# json_dump(res, self.args.logs_dir + '/res.json')
logging.info(f'eval finish res {res}')
return res
def evaluate_vid(self, queryloader, galleryloader, metric=None, **kwargs):
self.model.eval()
res = {}
self.timer.since_last_check('start eval')
with torch.no_grad():
qf, q_pids, q_camids = [], [], []
for batch_idx, data in enumerate(queryloader):
(imgs, pids, camids) = data['img'], data['pid'], data['cid']
imgs = imgs.cuda()
b, s, c, h, w = imgs.size()
imgs = imgs.view(b * s, c, h, w)
features = self.model(imgs)[0]
features = features.view(b, s, -1)
features = torch.mean(features, 1) # use avg
features = features.data.cpu()
qf.append(features)
q_pids.extend(pids)
q_camids.extend(camids)
qf = torch.cat(qf, 0)
q_pids = np.asarray(q_pids)
q_camids = np.asarray(q_camids)
print("Extracted features for query set, obtained {}-by-{} matrix".
format(qf.size(0), qf.size(1)))
self.timer.since_last_check('extract query')
gf, g_pids, g_camids = [], [], []
for batch_idx, data in enumerate(galleryloader):
(imgs, pids, camids) = data['img'], data['pid'], data['cid']
imgs = imgs.cuda()
b, s, c, h, w = imgs.size()
imgs = imgs.view(b * s, c, h, w)
features = self.model(imgs)[0]
features = features.view(b, s, -1)
features = torch.mean(features, 1)
features = features.data.cpu()
gf.append(features)
g_pids.extend(pids)
g_camids.extend(camids)
gf = torch.cat(gf, 0)
g_pids = np.asarray(g_pids)
g_camids = np.asarray(g_camids)
print(
"Extracted features for gallery set, obtained {}-by-{} matrix".
format(gf.size(0), gf.size(1)))
self.timer.since_last_check('extract gallery')
print("Computing distance matrix")
rerank = False # todo
print('!!rerank is ', rerank)
if rerank:
xx = to_numpy(qf)
yy = to_numpy(gf)
# lz.msgpack_dump([xx, yy], work_path + 'tmp.mp')
distmat = re_ranking(xx, yy)
else:
m, n = qf.size(0), gf.size(0)
distmat = torch.pow(qf, 2).sum(dim=1, keepdim=True).expand(m, n) + \
torch.pow(gf, 2).sum(dim=1, keepdim=True).expand(n, m).t()
distmat.addmm_(1, -2, qf, gf.t())
distmat = distmat.numpy()
if self.conf == 'market1501':
self.timer.since_last_check('distmat ')
# distmat = np.array(distmat, np.float16)
print(f'facing {distmat.shape}')
mAP, all_cmc = eval_market1501_wrap(distmat, q_pids, g_pids,
q_camids, g_camids, 10)
self.timer.since_last_check('map cmc ok')
res = {
'mAP': mAP,
'top-1': all_cmc[0],
'top-5': all_cmc[4],
'top-10': all_cmc[9]
}
else:
self.timer.since_last_check('distmat ')
print("Computing CMC and mAP")
mAP = mean_ap(distmat, q_pids, g_pids, q_camids, g_camids)
self.timer.since_last_check('mAP ok ')
print('Mean AP: {:4.1%}'.format(mAP))
cmc_configs = {
'cuhk03':
dict(separate_camera_set=True,
single_gallery_shot=True,
first_match_break=False),
'market1501':
dict(
separate_camera_set=False, # hard
single_gallery_shot=False, # hard
first_match_break=True),
'allshots':
dict(
separate_camera_set=False, # hard
single_gallery_shot=False, # hard
first_match_break=False),
}
cmc_configs = {
k: v
for k, v in cmc_configs.items() if k == self.conf
}
cmc_scores = {
name: cmc(distmat, q_pids, g_pids, q_camids, g_camids,
**params)
for name, params in cmc_configs.items()
}
print(f'cmc-1 {self.conf} {cmc_scores[self.conf][0]} ')
if rerank:
res = lz.dict_concat([
res, {
'mAP.rk': mAP,
'top-1.rk': cmc_scores[self.conf][0],
'top-5.rk': cmc_scores[self.conf][4],
'top-10.rk': cmc_scores[self.conf][9],
}
])
else:
res = lz.dict_concat([
res, {
'mAP': mAP,
'top-1': cmc_scores[self.conf][0],
'top-5': cmc_scores[self.conf][4],
'top-10': cmc_scores[self.conf][9],
}
])
json_dump(res, self.args.logs_dir + '/res.json', 'w')
self.timer.since_last_check('cmc ok')
return res