def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() pred_logits = comm.gather(self.pred_logits) pred_logits = sum(pred_logits, []) labels = comm.gather(self.labels) labels = sum(labels, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: pred_logits = self.pred_logits labels = self.labels pred_logits = torch.cat(pred_logits, dim=0) labels = torch.stack(labels) # measure accuracy and record loss acc1, = accuracy(pred_logits, labels, topk=(1, )) self._results = OrderedDict() self._results["Acc@1"] = acc1 self._results["metric"] = acc1 return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions pred_logits = [] labels = [] for prediction in predictions: pred_logits.append(prediction['logits']) labels.append(prediction['labels']) pred_logits = torch.cat(pred_logits, dim=0) labels = torch.cat(labels, dim=0) # measure accuracy and record loss acc1, = accuracy(pred_logits, labels, topk=(1, )) self._results = OrderedDict() self._results["Acc@1"] = acc1 self._results["metric"] = acc1 return copy.deepcopy(self._results)
def _write_metrics(self, metrics_dict: dict): """ Args: metrics_dict (dict): dict of scalar metrics """ metrics_dict = { k: v.detach().cpu().item() if isinstance(v, torch.Tensor) else float(v) for k, v in metrics_dict.items() } # gather metrics among all workers for logging # This assumes we do DDP-style training, which is currently the only # supported method in fastreid. all_metrics_dict = comm.gather(metrics_dict) if comm.is_main_process(): if "data_time" in all_metrics_dict[0]: # data_time among workers can have high variance. The actual latency # caused by data_time is the maximum among workers. data_time = np.max( [x.pop("data_time") for x in all_metrics_dict]) self.storage.put_scalar("data_time", data_time) # average the rest metrics metrics_dict = { k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys() } total_losses_reduced = sum(loss for loss in metrics_dict.values()) self.storage.put_scalar("total_loss", total_losses_reduced) if len(metrics_dict) > 1: self.storage.put_scalars(**metrics_dict)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions features = [] pids = [] # camids = [] for prediction in predictions: features.append(prediction['feats']) pids.append(prediction['pids']) # camids.append(prediction['camids']) features = torch.cat(features, dim=0) pids = torch.cat(pids, dim=0).numpy() rerank_dist = compute_jaccard_distance( features, k1=self.cfg.CLUSTER.JACCARD.K1, k2=self.cfg.CLUSTER.JACCARD.K2, ) pseudo_labels = self.cluster.fit_predict(rerank_dist) contingency_matrix = metrics.cluster.contingency_matrix( pids, pseudo_labels) purity = np.sum(np.amax(contingency_matrix, axis=0)) / np.sum(contingency_matrix) return purity
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: features = self.features features = torch.cat(features, dim=0) features = F.normalize(features, p=2, dim=1).numpy() self._results = OrderedDict() tpr, fpr, accuracy, best_thresholds = evaluate(features, self.labels) self._results["Accuracy"] = accuracy.mean() * 100 self._results["Threshold"] = best_thresholds.mean() self._results["metric"] = accuracy.mean() * 100 buf = gen_plot(fpr, tpr) roc_curve = Image.open(buf) PathManager.mkdirs(self._output_dir) roc_curve.save( os.path.join(self._output_dir, self.dataset_name + "_roc.png")) return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions features = [] pids = [] # camids = [] for prediction in predictions: features.append(prediction['feats']) pids.append(prediction['pids']) # camids.append(prediction['camids']) features = torch.cat(features, dim=0) pids = torch.cat(pids, dim=0).numpy() rerank_dist = compute_jaccard_distance( features, k1=self.cfg.CLUSTER.JACCARD.K1, k2=self.cfg.CLUSTER.JACCARD.K2, ) pseudo_labels = self.cluster.fit_predict(rerank_dist) ARI_score = metrics.adjusted_rand_score(pids, pseudo_labels) return ARI_score
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) labels = comm.gather(self.labels) labels = sum(labels, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: features = self.features labels = self.labels features = torch.cat(features, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_labels = np.asarray(labels[:self._num_query]) # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(labels[self._num_query:]) self._results = OrderedDict() if self._num_query == len(features): cmc = recall_at_ks(query_features, query_labels, self.recalls, cosine=True) else: cmc = recall_at_ks(query_features, query_labels, self.recalls, gallery_features, gallery_pids, cosine=True) for r in self.recalls: self._results['Recall@{}'.format(r)] = cmc[r] self._results["metric"] = cmc[self.recalls[0]] return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() pred_logits = comm.gather(self.pred_logits) pred_logits = sum(pred_logits, []) gt_labels = comm.gather(self.gt_labels) gt_labels = sum(gt_labels, []) if not comm.is_main_process(): return {} else: pred_logits = self.pred_logits gt_labels = self.gt_labels pred_logits = torch.stack(pred_logits, dim=0).numpy() gt_labels = torch.stack(gt_labels, dim=0).numpy() # Pedestrian attribute metrics thres = self.cfg.TEST.THRES self._results = self.get_attr_metrics(gt_labels, pred_logits, thres) return copy.deepcopy(self._results)
def _write_metrics(self, loss_dict: Dict[str, torch.Tensor], data_time: float): """ Args: loss_dict (dict): dict of scalar losses data_time (float): time taken by the dataloader iteration """ device = next(iter(loss_dict.values())).device # Use a new stream so these ops don't wait for DDP or backward with torch.cuda.stream(torch.cuda.Stream() if device.type == "cuda" else None): metrics_dict = { k: v.detach().cpu().item() for k, v in loss_dict.items() } metrics_dict["data_time"] = data_time # Gather metrics among all workers for logging # This assumes we do DDP-style training, which is currently the only # supported method in detectron2. all_metrics_dict = comm.gather(metrics_dict) if comm.is_main_process(): storage = get_event_storage() # data_time among workers can have high variance. The actual latency # caused by data_time is the maximum among workers. data_time = np.max([x.pop("data_time") for x in all_metrics_dict]) storage.put_scalar("data_time", data_time) # average the rest metrics metrics_dict = { k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys() } total_losses_reduced = sum(metrics_dict.values()) if not np.isfinite(total_losses_reduced): raise FloatingPointError( f"Loss became infinite or NaN at iteration={self.iter}!\n" f"loss_dict = {metrics_dict}") storage.put_scalar("total_loss", total_losses_reduced) if len(metrics_dict) > 1: storage.put_scalars(**metrics_dict)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions total_correct_num = 0 total_samples = 0 for prediction in predictions: total_correct_num += prediction["num_correct"] total_samples += prediction["num_samples"] acc1 = total_correct_num / total_samples * 100 self._results = OrderedDict() self._results["Acc@1"] = acc1 self._results["metric"] = acc1 return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) pids = comm.gather(self.pids) pids = sum(pids, []) camids = comm.gather(self.camids) camids = sum(camids, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: features = self.features pids = self.pids camids = self.camids features = torch.cat(features, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = np.asarray(pids[:self._num_query]) query_camids = np.asarray(camids[:self._num_query]) # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(pids[self._num_query:]) gallery_camids = np.asarray(camids[self._num_query:]) self._results = OrderedDict() if self.cfg.TEST.AQE.ENABLED: logger.info("Test with AQE setting") qe_time = self.cfg.TEST.AQE.QE_TIME qe_k = self.cfg.TEST.AQE.QE_K alpha = self.cfg.TEST.AQE.ALPHA query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha) dist = build_dist(query_features, gallery_features, self.cfg.TEST.METRIC) if self.cfg.TEST.RERANK.ENABLED: logger.info("Test with rerank setting") k1 = self.cfg.TEST.RERANK.K1 k2 = self.cfg.TEST.RERANK.K2 lambda_value = self.cfg.TEST.RERANK.LAMBDA if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) rerank_dist = build_dist(query_features, gallery_features, metric="jaccard", k1=k1, k2=k2) dist = rerank_dist * (1 - lambda_value) + dist * lambda_value cmc, all_AP, all_INP = evaluate_rank(dist, query_pids, gallery_pids, query_camids, gallery_camids) mAP = np.mean(all_AP) mINP = np.mean(all_INP) for r in [1, 5, 10]: self._results['Rank-{}'.format(r)] = cmc[r - 1] self._results['mAP'] = mAP self._results['mINP'] = mINP self._results["metric"] = (mAP + cmc[0]) / 2 if self.cfg.TEST.ROC_ENABLED: scores, labels = evaluate_roc(dist, query_pids, gallery_pids, query_camids, gallery_camids) fprs, tprs, thres = metrics.roc_curve(labels, scores) for fpr in [1e-4, 1e-3, 1e-2]: ind = np.argmin(np.abs(fprs - fpr)) self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind] return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) pids = comm.gather(self.pids) pids = sum(pids, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: features = self.features pids = self.pids features = torch.cat(features, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = np.asarray(pids[:self._num_query]) # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(pids[self._num_query:]) if self.cfg.TEST.AQE.ENABLED: logger.info("Test with AQE setting") qe_time = self.cfg.TEST.AQE.QE_TIME qe_k = self.cfg.TEST.AQE.QE_K alpha = self.cfg.TEST.AQE.ALPHA query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha) if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) dist = build_dist(query_features, gallery_features, self.cfg.TEST.METRIC) if self.cfg.TEST.RERANK.ENABLED: logger.info("Test with rerank setting") k1 = self.cfg.TEST.RERANK.K1 k2 = self.cfg.TEST.RERANK.K2 lambda_value = self.cfg.TEST.RERANK.LAMBDA if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) rerank_dist = build_dist(query_features, gallery_features, metric="jaccard", k1=k1, k2=k2) dist = rerank_dist * (1 - lambda_value) + dist * lambda_value if self.cfg.TEST.SAVE_DISTMAT: np.save(os.path.join(self.cfg.OUTPUT_DIR, "distmat.npy"), dist) results = defaultdict(list) topk_indices = partition_arg_topK(dist, K=200, axis=1) for i in range(topk_indices.shape[0]): results[query_pids[i]].extend(gallery_pids[topk_indices[i]]) with open(os.path.join(self.cfg.OUTPUT_DIR, "submit.json"), 'w') as f: json.dump(results, f) return {}
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) pids = comm.gather(self.pids) pids = sum(pids, []) camids = comm.gather(self.camids) camids = sum(camids, []) names = comm.gather(self.names) names = sum(names, []) if not comm.is_main_process(): return {} else: features = self.features pids = self.pids camids = self.camids names = self.names features = torch.cat(features, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = np.asarray(pids[:self._num_query]) query_camids = np.asarray(camids[:self._num_query]) query_names = names[:self._num_query] # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(pids[self._num_query:]) gallery_camids = np.asarray(camids[self._num_query:]) gallery_names = names[self._num_query:] # save_dicts = {'q_feat': query_features.numpy(), # 'q_id': query_pids, # 'q_cam': query_camids, # 'q_name': query_names, # 'g_feat': gallery_features.numpy(), # 'g_id': gallery_pids, # 'g_cam': gallery_camids, # 'g_name': gallery_names} # import pickle # print('Saving features ...') # with open('feats/pcl_filter_flip_aug.pkl', 'wb') as f: # pickle.dump(save_dicts, f) self._results = OrderedDict() if self.cfg.TEST.AQE.ENABLED: logger.info("Test with AQE setting") qe_time = self.cfg.TEST.AQE.QE_TIME qe_k = self.cfg.TEST.AQE.QE_K alpha = self.cfg.TEST.AQE.ALPHA query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha) if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) dist = self.cal_dist(self.cfg.TEST.METRIC, query_features, gallery_features) if self.cfg.TEST.RERANK.ENABLED: logger.info("Test with rerank setting") k1 = self.cfg.TEST.RERANK.K1 k2 = self.cfg.TEST.RERANK.K2 lambda_value = self.cfg.TEST.RERANK.LAMBDA q_q_dist = self.cal_dist(self.cfg.TEST.METRIC, query_features, query_features) g_g_dist = self.cal_dist(self.cfg.TEST.METRIC, gallery_features, gallery_features) re_dist = re_ranking(dist, q_q_dist, g_g_dist, k1, k2, lambda_value) query_features = query_features.numpy() gallery_features = gallery_features.numpy() cmc, all_AP, all_INP = evaluate_rank(re_dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids, use_distmat=True) else: query_features = query_features.numpy() gallery_features = gallery_features.numpy() cmc, all_AP, all_INP = evaluate_rank(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids, use_distmat=False) mAP = np.mean(all_AP) mINP = np.mean(all_INP) for r in [1, 5, 10]: self._results['Rank-{}'.format(r)] = cmc[r - 1] self._results['mAP'] = mAP self._results['mINP'] = mINP if self.cfg.TEST.ROC_ENABLED: scores, labels = evaluate_roc(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids) fprs, tprs, thres = metrics.roc_curve(labels, scores) for fpr in [1e-4, 1e-3, 1e-2]: ind = np.argmin(np.abs(fprs - fpr)) self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind] return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) spatial_features = comm.gather(self.spatial_features) spatial_features = sum(spatial_features, []) scores = comm.gather(self.scores) scores = sum(scores, []) pids = comm.gather(self.pids) pids = sum(pids, []) camids = comm.gather(self.camids) camids = sum(camids, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: features = self.features spatial_features = self.spatial_features scores = self.scores pids = self.pids camids = self.camids features = torch.cat(features, dim=0) spatial_features = torch.cat(spatial_features, dim=0).numpy() scores = torch.cat(scores, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = np.asarray(pids[:self._num_query]) query_camids = np.asarray(camids[:self._num_query]) # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(pids[self._num_query:]) gallery_camids = np.asarray(camids[self._num_query:]) if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) dist = 1 - torch.mm(query_features, gallery_features.t()).numpy() self._results = OrderedDict() query_features = query_features.numpy() gallery_features = gallery_features.numpy() if self.cfg.TEST.DSR.ENABLED: logger.info("Testing with DSR setting") dsr_dist = compute_dsr_dist(spatial_features[:self._num_query], spatial_features[self._num_query:], dist, scores[:self._num_query]) max_value = 0 k = 0 for i in range(0, 101): lamb = 0.01 * i dist1 = (1 - lamb) * dist + lamb * dsr_dist cmc, all_AP, all_INP = evaluate_rank(dist1, query_pids, gallery_pids, query_camids, gallery_camids) if (cmc[0] > max_value): k = lamb max_value = cmc[0] dist1 = (1 - k) * dist + k * dsr_dist cmc, all_AP, all_INP = evaluate_rank(dist1, query_pids, gallery_pids, query_camids, gallery_camids) else: cmc, all_AP, all_INP = evaluate_rank(dist, query_pids, gallery_pids, query_camids, gallery_camids) mAP = np.mean(all_AP) mINP = np.mean(all_INP) for r in [1, 5, 10]: self._results['Rank-{}'.format(r)] = cmc[r - 1] * 100 self._results['mAP'] = mAP * 100 self._results['mINP'] = mINP * 100 return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions features = [] pids = [] camids = [] for prediction in predictions: features.append(prediction['feats']) pids.append(prediction['pids']) camids.append(prediction['camids']) features = torch.cat(features, dim=0) pids = torch.cat(pids, dim=0).numpy() camids = torch.cat(camids, dim=0).numpy() # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = pids[:self._num_query] query_camids = camids[:self._num_query] # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = pids[self._num_query:] gallery_camids = camids[self._num_query:] query_features = query_features.numpy() gallery_features = gallery_features.numpy() self._results = OrderedDict() if self.cfg.TEST.AQE.ENABLED: logger.info("Test with AQE setting") qe_time = self.cfg.TEST.AQE.QE_TIME qe_k = self.cfg.TEST.AQE.QE_K alpha = self.cfg.TEST.AQE.ALPHA query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha) query_features = query_features.numpy() gallery_features = gallery_features.numpy() dist = build_dist(query_features, gallery_features, self.cfg.TEST.METRIC) if self.cfg.TEST.RERANK.ENABLED: logger.info("Test with rerank setting") k1 = self.cfg.TEST.RERANK.K1 k2 = self.cfg.TEST.RERANK.K2 lambda_value = self.cfg.TEST.RERANK.LAMBDA if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) rerank_dist = build_dist(query_features, gallery_features, metric="jaccard", k1=k1, k2=k2) dist = rerank_dist * (1 - lambda_value) + dist * lambda_value cmc, all_AP, all_INP = evaluate_rank(dist, query_pids, gallery_pids, query_camids, gallery_camids) mAP = np.mean(all_AP) mINP = np.mean(all_INP) for r in [1, 5, 10]: self._results['Rank-{}'.format(r)] = cmc[r - 1] * 100 self._results['mAP'] = mAP * 100 self._results['mINP'] = mINP * 100 self._results["metric"] = (mAP + cmc[0]) / 2 * 100 if self.cfg.TEST.ROC.ENABLED: scores, labels = evaluate_roc(dist, query_pids, gallery_pids, query_camids, gallery_camids) fprs, tprs, thres = metrics.roc_curve(labels, scores) for fpr in [1e-4, 1e-3, 1e-2]: ind = np.argmin(np.abs(fprs - fpr)) self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind] return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() predictions = comm.gather(self._predictions, dst=0) predictions = list(itertools.chain(*predictions)) if not comm.is_main_process(): return {} else: predictions = self._predictions features = [] pids = [] # camids = [] for prediction in predictions: features.append(prediction['feats']) pids.append(prediction['pids']) # camids.append(prediction['camids']) features = torch.cat(features, dim=0) pids = torch.cat(pids, dim=0).numpy() rerank_dist = compute_jaccard_distance( features, k1=self.cfg.CLUSTER.JACCARD.K1, k2=self.cfg.CLUSTER.JACCARD.K2, ) pseudo_labels = self.cluster.fit_predict(rerank_dist) ARI_score = metrics.adjusted_rand_score(pids, pseudo_labels) contingency_matrix = metrics.cluster.contingency_matrix( pids, pseudo_labels) purity = np.sum(np.amax(contingency_matrix, axis=0)) / np.sum(contingency_matrix) return ARI_score, purity # def extract_cnn_feature(model, inputs, return_branches=False): # inputs = to_torch(inputs).cuda() # if return_branches: # outputs, branches_lists = model(inputs, return_branches=True) # outputs = outputs.data.cpu() # for i in range(len(branches_lists)): # branches_lists[i] = branches_lists[i].data.cpu() # return outputs, branches_lists # else: # outputs = model(inputs) # outputs = outputs.data.cpu() # return outputs # def extract_features(model, data_loader, print_freq=50, with_mem_idx=False, return_branches=False): # model.eval() # batch_time = AverageMeter() # data_time = AverageMeter() # features = OrderedDict() # labels = OrderedDict() # end = time.time() # with torch.no_grad(): # for i, data in enumerate(data_loader): # if with_mem_idx: # imgs, fnames, pids, _, _ = data # else: # imgs, fnames, pids, _ = data # data_time.update(time.time() - end) # if return_branches: # outputs, branches_lists = extract_cnn_feature(model, imgs, return_branches=True) # for i_batch, (fname, output, pid) in enumerate(zip(fnames, outputs, pids)): # features[fname] = { # 'feat': output, # 'branches_list': [branches_lists[j_branch][i_batch] for j_branch in range(len(branches_lists))] # } # labels[fname] = pid # else: # outputs = extract_cnn_feature(model, imgs) # for fname, output, pid in zip(fnames, outputs, pids): # features[fname] = output # labels[fname] = pid # batch_time.update(time.time() - end) # end = time.time() # if (i + 1) % print_freq == 0: # print('Extract Features: [{}/{}]\t' # 'Time {:.3f} ({:.3f})\t' # 'Data {:.3f} ({:.3f})\t' # .format(i + 1, len(data_loader), # batch_time.val, batch_time.avg, # data_time.val, data_time.avg)) # return features, labels
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) pids = comm.gather(self.pids) pids = sum(pids, []) camids = comm.gather(self.camids) camids = sum(camids, []) img_paths = comm.gather(self.img_paths) img_paths = sum(img_paths, []) if not comm.is_main_process(): return {} else: features = self.features pids = self.pids camids = self.camids img_paths = self.img_paths features = torch.cat(features, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = np.asarray(pids[:self._num_query]) query_camids = np.asarray(camids[:self._num_query]) # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(pids[self._num_query:]) gallery_camids = np.asarray(camids[self._num_query:]) self._results = OrderedDict() if self.cfg.TEST.AQE.ENABLED: logger.info("Test with AQE setting") qe_time = self.cfg.TEST.AQE.QE_TIME qe_k = self.cfg.TEST.AQE.QE_K alpha = self.cfg.TEST.AQE.ALPHA query_features, gallery_features = aqe(query_features, gallery_features, qe_time, qe_k, alpha) if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) dist = self.cal_dist(self.cfg.TEST.METRIC, query_features, gallery_features) if self.cfg.TEST.RERANK.ENABLED: logger.info("Test with rerank setting") k1 = self.cfg.TEST.RERANK.K1 k2 = self.cfg.TEST.RERANK.K2 lambda_value = self.cfg.TEST.RERANK.LAMBDA # q_q_dist = self.cal_dist(self.cfg.TEST.METRIC, query_features, query_features) # g_g_dist = self.cal_dist(self.cfg.TEST.METRIC, gallery_features, gallery_features) # re_dist = re_ranking(dist, q_q_dist, g_g_dist, k1, k2, lambda_value) # Luo rerank re_dist = re_ranking(query_features, gallery_features, k1, k2, lambda_value) print('re_dist',re_dist.shape) query_features = query_features.numpy() gallery_features = gallery_features.numpy() # gen json eval_json(img_paths,self._num_query,re_dist,query_features,gallery_features,self._output_dir,use_distmat=True) return # cmc, all_AP, all_INP = evaluate_rank(re_dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids, use_distmat=True) else: query_features = query_features.numpy() gallery_features = gallery_features.numpy() # gen json eval_json(img_paths,self._num_query,dist,query_features,gallery_features,self._output_dir,use_distmat=False) return # cmc, all_AP, all_INP = evaluate_rank(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids, use_distmat=False) mAP = np.mean(all_AP) mINP = np.mean(all_INP) for r in [1, 5, 10]: self._results['Rank-{}'.format(r)] = cmc[r - 1] self._results['mAP'] = mAP self._results['mINP'] = mINP if self.cfg.TEST.ROC_ENABLED: scores, labels = evaluate_roc(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids) fprs, tprs, thres = metrics.roc_curve(labels, scores) for fpr in [1e-4, 1e-3, 1e-2]: ind = np.argmin(np.abs(fprs - fpr)) self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind] return copy.deepcopy(self._results)
def evaluate(self): if comm.get_world_size() > 1: comm.synchronize() features = comm.gather(self.features) features = sum(features, []) spatial_features = comm.gather(self.spatial_features) spatial_features = sum(spatial_features, []) scores = comm.gather(self.scores) scores = sum(scores, []) pids = comm.gather(self.pids) pids = sum(pids, []) camids = comm.gather(self.camids) camids = sum(camids, []) # fmt: off if not comm.is_main_process(): return {} # fmt: on else: features = self.features spatial_features = self.spatial_features scores = self.scores pids = self.pids camids = self.camids features = torch.cat(features, dim=0) spatial_features = torch.cat(spatial_features, dim=0).numpy() scores = torch.cat(scores, dim=0) # query feature, person ids and camera ids query_features = features[:self._num_query] query_pids = np.asarray(pids[:self._num_query]) query_camids = np.asarray(camids[:self._num_query]) # gallery features, person ids and camera ids gallery_features = features[self._num_query:] gallery_pids = np.asarray(pids[self._num_query:]) gallery_camids = np.asarray(camids[self._num_query:]) if self.cfg.TEST.METRIC == "cosine": query_features = F.normalize(query_features, dim=1) gallery_features = F.normalize(gallery_features, dim=1) dist = 1 - torch.mm(query_features, gallery_features.t()).numpy() self._results = OrderedDict() query_features = query_features.numpy() gallery_features = gallery_features.numpy() if self.cfg.TEST.DSR.ENABLED: logger.info("Testing with DSR setting") dist = compute_dsr_dist(spatial_features[:self._num_query], spatial_features[self._num_query:], dist, scores[:self._num_query]) cmc, all_AP, all_INP = evaluate_rank(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids, use_distmat=True) else: cmc, all_AP, all_INP = evaluate_rank(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids, use_distmat=False) mAP = np.mean(all_AP) mINP = np.mean(all_INP) for r in [1, 5, 10]: self._results['Rank-{}'.format(r)] = cmc[r - 1] self._results['mAP'] = mAP self._results['mINP'] = mINP if self.cfg.TEST.ROC_ENABLED: scores, labels = evaluate_roc(dist, query_features, gallery_features, query_pids, gallery_pids, query_camids, gallery_camids) fprs, tprs, thres = metrics.roc_curve(labels, scores) for fpr in [1e-4, 1e-3, 1e-2]: ind = np.argmin(np.abs(fprs - fpr)) self._results["TPR@FPR={:.0e}".format(fpr)] = tprs[ind] return copy.deepcopy(self._results)