def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
pred_logits = comm.gather(self.pred_logits)
pred_logits = sum(pred_logits, [])
labels = comm.gather(self.labels)
labels = sum(labels, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
pred_logits = self.pred_logits
labels = self.labels
pred_logits = torch.cat(pred_logits, dim=0)
labels = torch.stack(labels)
# measure accuracy and record loss
acc1, = accuracy(pred_logits, labels, topk=(1, ))
self._results = OrderedDict()
self._results["Acc@1"] = acc1
self._results["metric"] = acc1
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
features = []
pids = []
# camids = []
for prediction in predictions:
features.append(prediction['feats'])
pids.append(prediction['pids'])
# camids.append(prediction['camids'])
features = torch.cat(features, dim=0)
pids = torch.cat(pids, dim=0).numpy()
rerank_dist = compute_jaccard_distance(
features,
k1=self.cfg.CLUSTER.JACCARD.K1,
k2=self.cfg.CLUSTER.JACCARD.K2,
)
pseudo_labels = self.cluster.fit_predict(rerank_dist)
contingency_matrix = metrics.cluster.contingency_matrix(
pids, pseudo_labels)
purity = np.sum(np.amax(contingency_matrix,
axis=0)) / np.sum(contingency_matrix)
return purity
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
features = self.features
features = torch.cat(features, dim=0)
features = F.normalize(features, p=2, dim=1).numpy()
self._results = OrderedDict()
tpr, fpr, accuracy, best_thresholds = evaluate(features, self.labels)
self._results["Accuracy"] = accuracy.mean() * 100
self._results["Threshold"] = best_thresholds.mean()
self._results["metric"] = accuracy.mean() * 100
buf = gen_plot(fpr, tpr)
roc_curve = Image.open(buf)
PathManager.mkdirs(self._output_dir)
roc_curve.save(
os.path.join(self._output_dir, self.dataset_name + "_roc.png"))
return copy.deepcopy(self._results)
def _distributed_worker(
local_rank, main_func, world_size, num_gpus_per_machine, machine_rank, dist_url, args
):
assert torch.cuda.is_available(), "cuda is not available. Please check your installation."
global_rank = machine_rank * num_gpus_per_machine + local_rank
try:
dist.init_process_group(
backend="NCCL", init_method=dist_url, world_size=world_size, rank=global_rank
)
except Exception as e:
logger = logging.getLogger(__name__)
logger.error("Process group URL: {}".format(dist_url))
raise e
# synchronize is needed here to prevent a possible timeout after calling init_process_group
# See: https://github.com/facebookresearch/maskrcnn-benchmark/issues/172
comm.synchronize()
assert num_gpus_per_machine <= torch.cuda.device_count()
torch.cuda.set_device(local_rank)
# Setup the local process group (which contains ranks within the same machine)
assert comm._LOCAL_PROCESS_GROUP is None
num_machines = world_size // num_gpus_per_machine
for i in range(num_machines):
ranks_on_i = list(range(i * num_gpus_per_machine, (i + 1) * num_gpus_per_machine))
pg = dist.new_group(ranks_on_i)
if i == machine_rank:
comm._LOCAL_PROCESS_GROUP = pg
main_func(*args)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
features = []
pids = []
# camids = []
for prediction in predictions:
features.append(prediction['feats'])
pids.append(prediction['pids'])
# camids.append(prediction['camids'])
features = torch.cat(features, dim=0)
pids = torch.cat(pids, dim=0).numpy()
rerank_dist = compute_jaccard_distance(
features,
k1=self.cfg.CLUSTER.JACCARD.K1,
k2=self.cfg.CLUSTER.JACCARD.K2,
)
pseudo_labels = self.cluster.fit_predict(rerank_dist)
ARI_score = metrics.adjusted_rand_score(pids, pseudo_labels)
return ARI_score
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process(): return {}
else:
predictions = self._predictions
pred_logits = []
labels = []
for prediction in predictions:
pred_logits.append(prediction['logits'])
labels.append(prediction['labels'])
pred_logits = torch.cat(pred_logits, dim=0)
labels = torch.cat(labels, dim=0)
# measure accuracy and record loss
acc1, = accuracy(pred_logits, labels, topk=(1, ))
self._results = OrderedDict()
self._results["Acc@1"] = acc1
self._results["metric"] = acc1
return copy.deepcopy(self._results)
def after_epoch(self):
next_epoch = self.trainer.epoch + 1
is_final = next_epoch == self.trainer.max_epoch
if is_final or (self._period > 0 and next_epoch % self._period == 0):
self._do_eval()
# Evaluation may take different time among workers.
# A barrier make them start the next iteration together.
comm.synchronize()
def init_pretrained_weights(key):
"""Initializes model with pretrained weights.
Layers that don't match with pretrained layers in name or size are kept unchanged.
"""
import os
import errno
import gdown
def _get_torch_home():
ENV_TORCH_HOME = 'TORCH_HOME'
ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME'
DEFAULT_CACHE_DIR = '~/.cache'
torch_home = os.path.expanduser(
os.getenv(
ENV_TORCH_HOME,
os.path.join(
os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR), 'torch'
)
)
)
return torch_home
torch_home = _get_torch_home()
model_dir = os.path.join(torch_home, 'checkpoints')
try:
os.makedirs(model_dir)
except OSError as e:
if e.errno == errno.EEXIST:
# Directory already exists, ignore.
pass
else:
# Unexpected OSError, re-raise.
raise
filename = model_urls[key].split('/')[-1]
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
if comm.is_main_process():
gdown.download(model_urls[key], cached_file, quiet=False)
comm.synchronize()
logger.info(f"Loading pretrained model from {cached_file}")
state_dict = torch.load(cached_file, map_location=torch.device('cpu'))
return state_dict
def after_step(self):
next_iter = self.trainer.iter + 1
is_final = next_iter == self.trainer.max_iter
if is_final or (self._period > 0 and next_iter % self._period == 0):
results = self._do_eval()
for task in results.keys():
if results[task]['R-1'] > self.best_top1[task]:
self.best_top1[task] = results[task]['R-1']
logger = logging.getLogger(__name__)
logger.info('----------------------------------------')
logger.info("Print Best_top1 in csv format:")
for task, res in self.best_top1.items():
logger.info(f"{task}: " + ", ".join(["{:.1%}".format(res)]))
logger.info('----------------------------------------')
# Evaluation may take different time among workers.
# A barrier make them start the next iteration together.
comm.synchronize()
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
labels = comm.gather(self.labels)
labels = sum(labels, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
features = self.features
labels = self.labels
features = torch.cat(features, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_labels = np.asarray(labels[:self._num_query])
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(labels[self._num_query:])
self._results = OrderedDict()
if self._num_query == len(features):
cmc = recall_at_ks(query_features,
query_labels,
self.recalls,
cosine=True)
else:
cmc = recall_at_ks(query_features,
query_labels,
self.recalls,
gallery_features,
gallery_pids,
cosine=True)
for r in self.recalls:
self._results['Recall@{}'.format(r)] = cmc[r]
self._results["metric"] = cmc[self.recalls[0]]
return copy.deepcopy(self._results)
def _do_eval(self):
results = self._func()
if results:
assert isinstance(
results, dict
), "Eval function must return a dict. Got {} instead.".format(results)
flattened_results = flatten_results_dict(results)
for k, v in flattened_results.items():
try:
v = float(v)
except Exception:
raise ValueError(
"[EvalHook] eval_function should return a nested dict of float. "
"Got '{}: {}' instead.".format(k, v)
)
self.trainer.storage.put_scalars(**flattened_results, smoothing_hint=False)
# Evaluation may take different time among workers.
# A barrier make them start the next iteration together.
comm.synchronize()
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
pred_logits = comm.gather(self.pred_logits)
pred_logits = sum(pred_logits, [])
gt_labels = comm.gather(self.gt_labels)
gt_labels = sum(gt_labels, [])
if not comm.is_main_process():
return {}
else:
pred_logits = self.pred_logits
gt_labels = self.gt_labels
pred_logits = torch.stack(pred_logits, dim=0).numpy()
gt_labels = torch.stack(gt_labels, dim=0).numpy()
# Pedestrian attribute metrics
thres = self.cfg.TEST.THRES
self._results = self.get_attr_metrics(gt_labels, pred_logits, thres)
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process(): return {}
else:
predictions = self._predictions
total_correct_num = 0
total_samples = 0
for prediction in predictions:
total_correct_num += prediction["num_correct"]
total_samples += prediction["num_samples"]
acc1 = total_correct_num / total_samples * 100
self._results = OrderedDict()
self._results["Acc@1"] = acc1
self._results["metric"] = acc1
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
pids = comm.gather(self.pids)
pids = sum(pids, [])
camids = comm.gather(self.camids)
camids = sum(camids, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
features = self.features
pids = self.pids
camids = self.camids
features = torch.cat(features, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = np.asarray(pids[:self._num_query])
query_camids = np.asarray(camids[:self._num_query])
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(pids[self._num_query:])
gallery_camids = np.asarray(camids[self._num_query:])
self._results = OrderedDict()
if self.cfg.TEST.AQE.ENABLED:
logger.info("Test with AQE setting")
qe_time = self.cfg.TEST.AQE.QE_TIME
qe_k = self.cfg.TEST.AQE.QE_K
alpha = self.cfg.TEST.AQE.ALPHA
query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha)
dist = build_dist(query_features, gallery_features, self.cfg.TEST.METRIC)
if self.cfg.TEST.RERANK.ENABLED:
logger.info("Test with rerank setting")
k1 = self.cfg.TEST.RERANK.K1
k2 = self.cfg.TEST.RERANK.K2
lambda_value = self.cfg.TEST.RERANK.LAMBDA
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
rerank_dist = build_dist(query_features, gallery_features, metric="jaccard", k1=k1, k2=k2)
dist = rerank_dist * (1 - lambda_value) + dist * lambda_value
cmc, all_AP, all_INP = evaluate_rank(dist, query_pids, gallery_pids, query_camids, gallery_camids)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
self._results['Rank-{}'.format(r)] = cmc[r - 1]
self._results['mAP'] = mAP
self._results['mINP'] = mINP
self._results["metric"] = (mAP + cmc[0]) / 2
if self.cfg.TEST.ROC_ENABLED:
scores, labels = evaluate_roc(dist, query_pids, gallery_pids, query_camids, gallery_camids)
fprs, tprs, thres = metrics.roc_curve(labels, scores)
for fpr in [1e-4, 1e-3, 1e-2]:
ind = np.argmin(np.abs(fprs - fpr))
self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind]
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
pids = comm.gather(self.pids)
pids = sum(pids, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
features = self.features
pids = self.pids
features = torch.cat(features, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = np.asarray(pids[:self._num_query])
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(pids[self._num_query:])
if self.cfg.TEST.AQE.ENABLED:
logger.info("Test with AQE setting")
qe_time = self.cfg.TEST.AQE.QE_TIME
qe_k = self.cfg.TEST.AQE.QE_K
alpha = self.cfg.TEST.AQE.ALPHA
query_features, gallery_features = aqe(query_features,
gallery_features, qe_time,
qe_k, alpha)
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
dist = build_dist(query_features, gallery_features,
self.cfg.TEST.METRIC)
if self.cfg.TEST.RERANK.ENABLED:
logger.info("Test with rerank setting")
k1 = self.cfg.TEST.RERANK.K1
k2 = self.cfg.TEST.RERANK.K2
lambda_value = self.cfg.TEST.RERANK.LAMBDA
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
rerank_dist = build_dist(query_features,
gallery_features,
metric="jaccard",
k1=k1,
k2=k2)
dist = rerank_dist * (1 - lambda_value) + dist * lambda_value
if self.cfg.TEST.SAVE_DISTMAT:
np.save(os.path.join(self.cfg.OUTPUT_DIR, "distmat.npy"), dist)
results = defaultdict(list)
topk_indices = partition_arg_topK(dist, K=200, axis=1)
for i in range(topk_indices.shape[0]):
results[query_pids[i]].extend(gallery_pids[topk_indices[i]])
with open(os.path.join(self.cfg.OUTPUT_DIR, "submit.json"), 'w') as f:
json.dump(results, f)
return {}
def init_pretrained_weights(model, key=''):
"""Initializes model with pretrained weights.
Layers that don't match with pretrained layers in name or size are kept unchanged.
"""
import os
import errno
import gdown
from collections import OrderedDict
import warnings
import logging
logger = logging.getLogger(__name__)
def _get_torch_home():
ENV_TORCH_HOME = 'TORCH_HOME'
ENV_XDG_CACHE_HOME = 'XDG_CACHE_HOME'
DEFAULT_CACHE_DIR = '~/.cache'
torch_home = os.path.expanduser(
os.getenv(
ENV_TORCH_HOME,
os.path.join(os.getenv(ENV_XDG_CACHE_HOME, DEFAULT_CACHE_DIR),
'torch')))
return torch_home
torch_home = _get_torch_home()
model_dir = os.path.join(torch_home, 'checkpoints')
try:
os.makedirs(model_dir)
except OSError as e:
if e.errno == errno.EEXIST:
# Directory already exists, ignore.
pass
else:
# Unexpected OSError, re-raise.
raise
filename = key + '_imagenet.pth'
cached_file = os.path.join(model_dir, filename)
if not os.path.exists(cached_file):
if comm.is_main_process():
gdown.download(model_urls[key], cached_file, quiet=False)
comm.synchronize()
state_dict = torch.load(cached_file, map_location=torch.device('cpu'))
model_dict = model.state_dict()
new_state_dict = OrderedDict()
matched_layers, discarded_layers = [], []
for k, v in state_dict.items():
if k.startswith('module.'):
k = k[7:] # discard module.
if k in model_dict and model_dict[k].size() == v.size():
new_state_dict[k] = v
matched_layers.append(k)
else:
discarded_layers.append(k)
model_dict.update(new_state_dict)
model.load_state_dict(model_dict)
if len(matched_layers) == 0:
warnings.warn('The pretrained weights from "{}" cannot be loaded, '
'please check the key names manually '
'(** ignored and continue **)'.format(cached_file))
else:
logger.info(
'Successfully loaded imagenet pretrained weights from "{}"'.format(
cached_file))
if len(discarded_layers) > 0:
logger.info('** The following layers are discarded '
'due to unmatched keys or layer size: {}'.format(
discarded_layers))
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
pids = comm.gather(self.pids)
pids = sum(pids, [])
camids = comm.gather(self.camids)
camids = sum(camids, [])
names = comm.gather(self.names)
names = sum(names, [])
if not comm.is_main_process():
return {}
else:
features = self.features
pids = self.pids
camids = self.camids
names = self.names
features = torch.cat(features, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = np.asarray(pids[:self._num_query])
query_camids = np.asarray(camids[:self._num_query])
query_names = names[:self._num_query]
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(pids[self._num_query:])
gallery_camids = np.asarray(camids[self._num_query:])
gallery_names = names[self._num_query:]
# save_dicts = {'q_feat': query_features.numpy(),
# 'q_id': query_pids,
# 'q_cam': query_camids,
# 'q_name': query_names,
# 'g_feat': gallery_features.numpy(),
# 'g_id': gallery_pids,
# 'g_cam': gallery_camids,
# 'g_name': gallery_names}
# import pickle
# print('Saving features ...')
# with open('feats/pcl_filter_flip_aug.pkl', 'wb') as f:
# pickle.dump(save_dicts, f)
self._results = OrderedDict()
if self.cfg.TEST.AQE.ENABLED:
logger.info("Test with AQE setting")
qe_time = self.cfg.TEST.AQE.QE_TIME
qe_k = self.cfg.TEST.AQE.QE_K
alpha = self.cfg.TEST.AQE.ALPHA
query_features, gallery_features = aqe(query_features,
gallery_features, qe_time,
qe_k, alpha)
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
dist = self.cal_dist(self.cfg.TEST.METRIC, query_features,
gallery_features)
if self.cfg.TEST.RERANK.ENABLED:
logger.info("Test with rerank setting")
k1 = self.cfg.TEST.RERANK.K1
k2 = self.cfg.TEST.RERANK.K2
lambda_value = self.cfg.TEST.RERANK.LAMBDA
q_q_dist = self.cal_dist(self.cfg.TEST.METRIC, query_features,
query_features)
g_g_dist = self.cal_dist(self.cfg.TEST.METRIC, gallery_features,
gallery_features)
re_dist = re_ranking(dist, q_q_dist, g_g_dist, k1, k2,
lambda_value)
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
cmc, all_AP, all_INP = evaluate_rank(re_dist,
query_features,
gallery_features,
query_pids,
gallery_pids,
query_camids,
gallery_camids,
use_distmat=True)
else:
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
cmc, all_AP, all_INP = evaluate_rank(dist,
query_features,
gallery_features,
query_pids,
gallery_pids,
query_camids,
gallery_camids,
use_distmat=False)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
self._results['Rank-{}'.format(r)] = cmc[r - 1]
self._results['mAP'] = mAP
self._results['mINP'] = mINP
if self.cfg.TEST.ROC_ENABLED:
scores, labels = evaluate_roc(dist, query_features,
gallery_features, query_pids,
gallery_pids, query_camids,
gallery_camids)
fprs, tprs, thres = metrics.roc_curve(labels, scores)
for fpr in [1e-4, 1e-3, 1e-2]:
ind = np.argmin(np.abs(fprs - fpr))
self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind]
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
spatial_features = comm.gather(self.spatial_features)
spatial_features = sum(spatial_features, [])
scores = comm.gather(self.scores)
scores = sum(scores, [])
pids = comm.gather(self.pids)
pids = sum(pids, [])
camids = comm.gather(self.camids)
camids = sum(camids, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
features = self.features
spatial_features = self.spatial_features
scores = self.scores
pids = self.pids
camids = self.camids
features = torch.cat(features, dim=0)
spatial_features = torch.cat(spatial_features, dim=0).numpy()
scores = torch.cat(scores, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = np.asarray(pids[:self._num_query])
query_camids = np.asarray(camids[:self._num_query])
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(pids[self._num_query:])
gallery_camids = np.asarray(camids[self._num_query:])
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
dist = 1 - torch.mm(query_features, gallery_features.t()).numpy()
self._results = OrderedDict()
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
if self.cfg.TEST.DSR.ENABLED:
logger.info("Testing with DSR setting")
dsr_dist = compute_dsr_dist(spatial_features[:self._num_query],
spatial_features[self._num_query:],
dist, scores[:self._num_query])
max_value = 0
k = 0
for i in range(0, 101):
lamb = 0.01 * i
dist1 = (1 - lamb) * dist + lamb * dsr_dist
cmc, all_AP, all_INP = evaluate_rank(dist1, query_pids,
gallery_pids,
query_camids,
gallery_camids)
if (cmc[0] > max_value):
k = lamb
max_value = cmc[0]
dist1 = (1 - k) * dist + k * dsr_dist
cmc, all_AP, all_INP = evaluate_rank(dist1, query_pids,
gallery_pids, query_camids,
gallery_camids)
else:
cmc, all_AP, all_INP = evaluate_rank(dist, query_pids,
gallery_pids, query_camids,
gallery_camids)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
self._results['Rank-{}'.format(r)] = cmc[r - 1] * 100
self._results['mAP'] = mAP * 100
self._results['mINP'] = mINP * 100
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
features = []
pids = []
camids = []
for prediction in predictions:
features.append(prediction['feats'])
pids.append(prediction['pids'])
camids.append(prediction['camids'])
features = torch.cat(features, dim=0)
pids = torch.cat(pids, dim=0).numpy()
camids = torch.cat(camids, dim=0).numpy()
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = pids[:self._num_query]
query_camids = camids[:self._num_query]
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = pids[self._num_query:]
gallery_camids = camids[self._num_query:]
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
self._results = OrderedDict()
if self.cfg.TEST.AQE.ENABLED:
logger.info("Test with AQE setting")
qe_time = self.cfg.TEST.AQE.QE_TIME
qe_k = self.cfg.TEST.AQE.QE_K
alpha = self.cfg.TEST.AQE.ALPHA
query_features, gallery_features = aqe(query_features,
gallery_features, qe_time,
qe_k, alpha)
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
dist = build_dist(query_features, gallery_features,
self.cfg.TEST.METRIC)
if self.cfg.TEST.RERANK.ENABLED:
logger.info("Test with rerank setting")
k1 = self.cfg.TEST.RERANK.K1
k2 = self.cfg.TEST.RERANK.K2
lambda_value = self.cfg.TEST.RERANK.LAMBDA
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
rerank_dist = build_dist(query_features,
gallery_features,
metric="jaccard",
k1=k1,
k2=k2)
dist = rerank_dist * (1 - lambda_value) + dist * lambda_value
cmc, all_AP, all_INP = evaluate_rank(dist, query_pids, gallery_pids,
query_camids, gallery_camids)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
self._results['Rank-{}'.format(r)] = cmc[r - 1] * 100
self._results['mAP'] = mAP * 100
self._results['mINP'] = mINP * 100
self._results["metric"] = (mAP + cmc[0]) / 2 * 100
if self.cfg.TEST.ROC.ENABLED:
scores, labels = evaluate_roc(dist, query_pids, gallery_pids,
query_camids, gallery_camids)
fprs, tprs, thres = metrics.roc_curve(labels, scores)
for fpr in [1e-4, 1e-3, 1e-2]:
ind = np.argmin(np.abs(fprs - fpr))
self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind]
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
pids = comm.gather(self.pids)
pids = sum(pids, [])
camids = comm.gather(self.camids)
camids = sum(camids, [])
img_paths = comm.gather(self.img_paths)
img_paths = sum(img_paths, [])
if not comm.is_main_process():
return {}
else:
features = self.features
pids = self.pids
camids = self.camids
img_paths = self.img_paths
features = torch.cat(features, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = np.asarray(pids[:self._num_query])
query_camids = np.asarray(camids[:self._num_query])
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(pids[self._num_query:])
gallery_camids = np.asarray(camids[self._num_query:])
self._results = OrderedDict()
if self.cfg.TEST.AQE.ENABLED:
logger.info("Test with AQE setting")
qe_time = self.cfg.TEST.AQE.QE_TIME
qe_k = self.cfg.TEST.AQE.QE_K
alpha = self.cfg.TEST.AQE.ALPHA
query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha)
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
dist = self.cal_dist(self.cfg.TEST.METRIC, query_features, gallery_features)
if self.cfg.TEST.RERANK.ENABLED:
logger.info("Test with rerank setting")
k1 = self.cfg.TEST.RERANK.K1
k2 = self.cfg.TEST.RERANK.K2
lambda_value = self.cfg.TEST.RERANK.LAMBDA
# q_q_dist = self.cal_dist(self.cfg.TEST.METRIC, query_features, query_features)
# g_g_dist = self.cal_dist(self.cfg.TEST.METRIC, gallery_features, gallery_features)
# re_dist = re_ranking(dist, q_q_dist, g_g_dist, k1, k2, lambda_value)
# Luo rerank
re_dist = re_ranking(query_features, gallery_features, k1, k2, lambda_value)
print('re_dist',re_dist.shape)
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
# gen json
eval_json(img_paths,self._num_query,re_dist,query_features,gallery_features,self._output_dir,use_distmat=True)
return
#
cmc, all_AP, all_INP = evaluate_rank(re_dist, query_features, gallery_features,
query_pids, gallery_pids, query_camids,
gallery_camids, use_distmat=True)
else:
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
# gen json
eval_json(img_paths,self._num_query,dist,query_features,gallery_features,self._output_dir,use_distmat=False)
return
#
cmc, all_AP, all_INP = evaluate_rank(dist, query_features, gallery_features,
query_pids, gallery_pids, query_camids, gallery_camids,
use_distmat=False)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
self._results['Rank-{}'.format(r)] = cmc[r - 1]
self._results['mAP'] = mAP
self._results['mINP'] = mINP
if self.cfg.TEST.ROC_ENABLED:
scores, labels = evaluate_roc(dist, query_features, gallery_features,
query_pids, gallery_pids, query_camids, gallery_camids)
fprs, tprs, thres = metrics.roc_curve(labels, scores)
for fpr in [1e-4, 1e-3, 1e-2]:
ind = np.argmin(np.abs(fprs - fpr))
self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind]
return copy.deepcopy(self._results)
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
predictions = comm.gather(self._predictions, dst=0)
predictions = list(itertools.chain(*predictions))
if not comm.is_main_process():
return {}
else:
predictions = self._predictions
features = []
pids = []
# camids = []
for prediction in predictions:
features.append(prediction['feats'])
pids.append(prediction['pids'])
# camids.append(prediction['camids'])
features = torch.cat(features, dim=0)
pids = torch.cat(pids, dim=0).numpy()
rerank_dist = compute_jaccard_distance(
features,
k1=self.cfg.CLUSTER.JACCARD.K1,
k2=self.cfg.CLUSTER.JACCARD.K2,
)
pseudo_labels = self.cluster.fit_predict(rerank_dist)
ARI_score = metrics.adjusted_rand_score(pids, pseudo_labels)
contingency_matrix = metrics.cluster.contingency_matrix(
pids, pseudo_labels)
purity = np.sum(np.amax(contingency_matrix,
axis=0)) / np.sum(contingency_matrix)
return ARI_score, purity
# def extract_cnn_feature(model, inputs, return_branches=False):
# inputs = to_torch(inputs).cuda()
# if return_branches:
# outputs, branches_lists = model(inputs, return_branches=True)
# outputs = outputs.data.cpu()
# for i in range(len(branches_lists)):
# branches_lists[i] = branches_lists[i].data.cpu()
# return outputs, branches_lists
# else:
# outputs = model(inputs)
# outputs = outputs.data.cpu()
# return outputs
# def extract_features(model, data_loader, print_freq=50, with_mem_idx=False, return_branches=False):
# model.eval()
# batch_time = AverageMeter()
# data_time = AverageMeter()
# features = OrderedDict()
# labels = OrderedDict()
# end = time.time()
# with torch.no_grad():
# for i, data in enumerate(data_loader):
# if with_mem_idx:
# imgs, fnames, pids, _, _ = data
# else:
# imgs, fnames, pids, _ = data
# data_time.update(time.time() - end)
# if return_branches:
# outputs, branches_lists = extract_cnn_feature(model, imgs, return_branches=True)
# for i_batch, (fname, output, pid) in enumerate(zip(fnames, outputs, pids)):
# features[fname] = {
# 'feat': output,
# 'branches_list': [branches_lists[j_branch][i_batch] for j_branch in range(len(branches_lists))]
# }
# labels[fname] = pid
# else:
# outputs = extract_cnn_feature(model, imgs)
# for fname, output, pid in zip(fnames, outputs, pids):
# features[fname] = output
# labels[fname] = pid
# batch_time.update(time.time() - end)
# end = time.time()
# if (i + 1) % print_freq == 0:
# print('Extract Features: [{}/{}]\t'
# 'Time {:.3f} ({:.3f})\t'
# 'Data {:.3f} ({:.3f})\t'
# .format(i + 1, len(data_loader),
# batch_time.val, batch_time.avg,
# data_time.val, data_time.avg))
# return features, labels
def evaluate(self):
if comm.get_world_size() > 1:
comm.synchronize()
features = comm.gather(self.features)
features = sum(features, [])
spatial_features = comm.gather(self.spatial_features)
spatial_features = sum(spatial_features, [])
scores = comm.gather(self.scores)
scores = sum(scores, [])
pids = comm.gather(self.pids)
pids = sum(pids, [])
camids = comm.gather(self.camids)
camids = sum(camids, [])
# fmt: off
if not comm.is_main_process(): return {}
# fmt: on
else:
features = self.features
spatial_features = self.spatial_features
scores = self.scores
pids = self.pids
camids = self.camids
features = torch.cat(features, dim=0)
spatial_features = torch.cat(spatial_features, dim=0).numpy()
scores = torch.cat(scores, dim=0)
# query feature, person ids and camera ids
query_features = features[:self._num_query]
query_pids = np.asarray(pids[:self._num_query])
query_camids = np.asarray(camids[:self._num_query])
# gallery features, person ids and camera ids
gallery_features = features[self._num_query:]
gallery_pids = np.asarray(pids[self._num_query:])
gallery_camids = np.asarray(camids[self._num_query:])
if self.cfg.TEST.METRIC == "cosine":
query_features = F.normalize(query_features, dim=1)
gallery_features = F.normalize(gallery_features, dim=1)
dist = 1 - torch.mm(query_features, gallery_features.t()).numpy()
self._results = OrderedDict()
query_features = query_features.numpy()
gallery_features = gallery_features.numpy()
if self.cfg.TEST.DSR.ENABLED:
logger.info("Testing with DSR setting")
dist = compute_dsr_dist(spatial_features[:self._num_query],
spatial_features[self._num_query:], dist,
scores[:self._num_query])
cmc, all_AP, all_INP = evaluate_rank(dist,
query_features,
gallery_features,
query_pids,
gallery_pids,
query_camids,
gallery_camids,
use_distmat=True)
else:
cmc, all_AP, all_INP = evaluate_rank(dist,
query_features,
gallery_features,
query_pids,
gallery_pids,
query_camids,
gallery_camids,
use_distmat=False)
mAP = np.mean(all_AP)
mINP = np.mean(all_INP)
for r in [1, 5, 10]:
self._results['Rank-{}'.format(r)] = cmc[r - 1]
self._results['mAP'] = mAP
self._results['mINP'] = mINP
if self.cfg.TEST.ROC_ENABLED:
scores, labels = evaluate_roc(dist, query_features,
gallery_features, query_pids,
gallery_pids, query_camids,
gallery_camids)
fprs, tprs, thres = metrics.roc_curve(labels, scores)
for fpr in [1e-4, 1e-3, 1e-2]:
ind = np.argmin(np.abs(fprs - fpr))
self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind]
return copy.deepcopy(self._results)