def test_delta(self): """ This tests runs the evaluation for an arbitrary random set of predictions. This score is then captured in this very test such that if we change the eval code, this test will trigger if the results changed. """ random.seed(42) np.random.seed(42) assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.' nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False) with open(self.res_mockup, 'w') as f: json.dump(self._mock_submission(nusc, 'mini_val'), f, indent=2) cfg = config_factory('detection_cvpr_2019') nusc_eval = DetectionEval(nusc, cfg, self.res_mockup, eval_set='mini_val', output_dir=self.res_eval_folder, verbose=False) metrics, md_list = nusc_eval.evaluate() # 1. Score = 0.22082865720221012. Measured on the branch "release_v0.2" on March 7 2019. # 2. Score = 0.2199307290627096. Changed to measure center distance from the ego-vehicle. # 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script. # 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings. # 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall. # 6. Score = 0.20636954644294506. After bike-rack filtering. # 7. Score = 0.20237925145690996. After TP reversion bug. # 8. Score = 0.24047129251302665. After bike racks bug. # 9. Score = 0.24104572227466886. After bug fix in calc_tp. Include the max recall and exclude the min recall. # 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split. self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
def quick_test(dataroot='/data/nuscenes', gpuid=0, nworkers=10): """Evaluate detections with PKL. """ nusc = NuScenes(version='v1.0-mini', dataroot=dataroot, verbose=True) nusc_maps = get_nusc_maps(dataroot) cfg = config_factory('detection_cvpr_2019') device = torch.device(f'cuda:{gpuid}') if gpuid >= 0\ else torch.device('cpu') print(f'using device: {device}') get_example_submission() nusc_eval = DetectionEval(nusc, config=cfg, result_path='./example_submission.json', eval_set='mini_train', output_dir='./res', verbose=True) info = calculate_pkl(nusc_eval.gt_boxes, nusc_eval.pred_boxes, nusc_eval.sample_tokens, nusc_eval.nusc, nusc_maps, device, nworkers, bsz=128, plot_kextremes=5, verbose=True) print({k: v for k, v in info.items() if k != 'full'})
def og_detection_eval(version, eval_set, result_path, dataroot='/data/nuscenes'): """Evaluate according to NDS. """ nusc = NuScenes(version='v1.0-{}'.format(version), dataroot=dataroot, verbose=True) cfg = config_factory('detection_cvpr_2019') nusc_eval = DetectionEval(nusc, config=cfg, result_path=result_path, eval_set=eval_set, output_dir='./res', verbose=True) nusc_eval.main(plot_examples=0, render_curves=False)
def do_nuScenes_detection_evaluation(dataset, predictions, output_folder, logger): mock_meta = { 'use_camera': True, 'use_lidar': False, 'use_radar': False, 'use_map': False, 'use_external': False, } mock_results = {} for image_token, prediction in predictions.items(): sample_res = [] for p in prediction: p = p.numpy() p = p.round(4) type = ID_TYPE_CONVERSION[int(p[0])] sample_res.append({ 'sample_token': image_token, 'translation': p[9:12].tolist(), 'size': p[6:9].tolist(), 'rotation': euler_to_quaternion(p[12], 0, 0), 'velocity': [0, 0], 'detection_name': type, 'attribute_name': random_attr(type) }) mock_results[image_token] = sample_res mock_submission = {'meta': mock_meta, 'results': mock_results} logger.info("Evaluate on nuScenes dataset") output_file = output_folder + ".json" with open(output_file, 'w') as f: json.dump(mock_submission, f, indent=2) cfg = config_factory('detection_cvpr_2019') nusc_eval = DetectionEval(dataset.nusc, cfg, output_file, eval_set='val', output_dir=output_folder, verbose=False) metrics, md_list = nusc_eval.evaluate() print(md_list)
def evaluation(self, det_annos, class_names, **kwargs): eval_det_annos = copy.deepcopy(det_annos) # Create NuScenes JSON output file nusc_annos = {} for sample in eval_det_annos: try: sample_idx = sample['sample_idx'][0] except: continue sample_results = [] calib = self.get_calib(sample_idx) sample['boxes_lidar'] = np.array(sample['boxes_lidar']) positions = sample['boxes_lidar'][:, :3] dimensions = sample['boxes_lidar'][:, 3:6] rotations = sample['boxes_lidar'][:, 6] for center, dimension, yaw, label, score in zip( positions, dimensions, rotations, sample['name'], sample['score']): quaternion = Quaternion(axis=[0, 0, 1], radians=yaw) box = Box(center, dimension, quaternion) # Move box to ego vehicle coord system box.rotate(Quaternion(calib.lidar_calibrated['rotation'])) box.translate(np.array(calib.lidar_calibrated['translation'])) # Move box to global coord system box.rotate(Quaternion(calib.ego_pose['rotation'])) box.translate(np.array(calib.ego_pose['translation'])) if (float(score) < 0): score = 0 if (float(score) > 1): score = 1 if (label == 'Cyclist'): label = 'bicycle' sample_results.append({ "sample_token": sample_idx, "translation": box.center.tolist(), "size": box.wlh.tolist(), "rotation": box.orientation.elements.tolist(), "lidar_yaw": float(yaw), "velocity": (0, 0), "detection_name": label.lower(), "detection_score": float(score), "attribute_name": self.DefaultAttribute[label.lower()], }) nusc_annos[sample_idx] = sample_results for sample_id in self.sample_id_list: if sample_id not in nusc_annos: nusc_annos[sample_id] = [] nusc_submission = { "meta": { "use_camera": False, "use_lidar": True, "use_radar": False, "use_map": False, "use_external": False, }, "results": nusc_annos, } eval_file = os.path.join(kwargs['output_dir'], 'nusc_results.json') with open(eval_file, "w") as f: json.dump(nusc_submission, f, indent=2) # Call NuScenes evaluation cfg = config_factory('detection_cvpr_2019') nusc_eval = DetectionEval(self.nusc, config=cfg, result_path=eval_file, eval_set=self.split, output_dir=kwargs['output_dir'], verbose=True) metric_summary = nusc_eval.main(plot_examples=10, render_curves=True) # Reformat the metrics summary a bit for the tensorboard logger err_name_mapping = { 'trans_err': 'mATE', 'scale_err': 'mASE', 'orient_err': 'mAOE', 'vel_err': 'mAVE', 'attr_err': 'mAAE' } result = {} result['mean_ap'] = metric_summary['mean_ap'] for tp_name, tp_val in metric_summary['tp_errors'].items(): result[tp_name] = tp_val class_aps = metric_summary['mean_dist_aps'] class_tps = metric_summary['label_tp_errors'] for class_name in class_aps.keys(): result['mAP_' + class_name] = class_aps[class_name] for key, val in err_name_mapping.items(): result[val + '_' + class_name] = class_tps[class_name][key] return str(result), result