def evaluation(self, det_annos, class_names, **kwargs): import json from nuscenes.nuscenes import NuScenes from . import nuscenes_utils nusc = NuScenes(version=self.dataset_cfg.VERSION, dataroot=str(self.root_path), verbose=True) nusc_annos = nuscenes_utils.transform_det_annos_to_nusc_annos( det_annos, nusc) nusc_annos['meta'] = { 'use_camera': False, 'use_lidar': True, 'use_radar': False, 'use_map': False, 'use_external': False, } output_path = Path(kwargs['output_path']) output_path.mkdir(exist_ok=True, parents=True) res_path = str(output_path / 'results_nusc.json') with open(res_path, 'w') as f: json.dump(nusc_annos, f) self.logger.info( f'The predictions of NuScenes have been saved to {res_path}') if self.dataset_cfg.VERSION == 'v1.0-test': return 'No ground-truth annotations for evaluation', {} from nuscenes.eval.detection.config import config_factory from nuscenes.eval.detection.evaluate import NuScenesEval eval_set_map = { 'v1.0-mini': 'mini_val', 'v1.0-trainval': 'val', 'v1.0-test': 'test' } try: eval_version = 'detection_cvpr_2019' eval_config = config_factory(eval_version) except: eval_version = 'cvpr_2019' eval_config = config_factory(eval_version) nusc_eval = NuScenesEval( nusc, config=eval_config, result_path=res_path, eval_set=eval_set_map[self.dataset_cfg.VERSION], output_dir=str(output_path), verbose=True, ) metrics_summary = nusc_eval.main(plot_examples=0, render_curves=False) with open(output_path / 'metrics_summary.json', 'r') as f: metrics = json.load(f) result_str, result_dict = nuscenes_utils.format_nuscene_results( metrics, self.class_names, version=eval_version) return result_str, result_dict
def cluster_trailer_box(info_path, class_name="bus"): with open(info_path, 'rb') as f: nusc_infos = pickle.load(f)["infos"] from nuscenes.eval.detection.config import config_factory from nuscenes.eval.detection.config import DetectionConfig cfg = config_factory(self.eval_version) cls_range_map = cfg.class_range gt_boxes_list = [] for info in nusc_infos: gt_boxes = info["gt_boxes"] gt_names = info["gt_names"] mask = np.array([s == class_name for s in info["gt_names"]], dtype=np.bool_) gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] det_range = np.array([cls_range_map[n] for n in gt_names]) det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]]) mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1) mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1) gt_boxes_list.append(gt_boxes[mask].reshape(-1, 7)) gt_boxes_list = np.concatenate(gt_boxes_list, axis=0) trailer_dims = gt_boxes_list[:, 3:6] from sklearn.cluster import DBSCAN from sklearn.preprocessing import StandardScaler X = StandardScaler().fit_transform(trailer_dims) db = DBSCAN(eps=0.3, min_samples=10).fit(X) core_samples_mask = np.zeros_like(db.labels_, dtype=bool) core_samples_mask[db.core_sample_indices_] = True labels = db.labels_ n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) n_noise_ = list(labels).count(-1) print(n_clusters_, n_noise_) print(trailer_dims) import matplotlib.pyplot as plt unique_labels = set(labels) colors = [ plt.cm.Spectral(each) for each in np.linspace(0, 1, len(unique_labels)) ] for k, col in zip(unique_labels, colors): if k == -1: # Black used for noise. col = [0, 0, 0, 1] class_member_mask = (labels == k) xy = trailer_dims[class_member_mask & core_samples_mask] plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col), markeredgecolor='k', markersize=14) xy = trailer_dims[class_member_mask & ~core_samples_mask] plt.plot(xy[:, 0], xy[:, 1], 'o', markerfacecolor=tuple(col), markeredgecolor='k', markersize=6) plt.show()
def __init__(self, ann_file, pipeline=None, data_root=None, classes=None, load_interval=1, with_velocity=True, modality=None, box_type_3d='LiDAR', filter_empty_gt=True, test_mode=False, eval_version='mmda_merge', use_valid_flag=False): self.load_interval = load_interval self.use_valid_flag = use_valid_flag assert classes is not None super().__init__(data_root=data_root, ann_file=ann_file, pipeline=pipeline, classes=classes, modality=modality, box_type_3d=box_type_3d, filter_empty_gt=filter_empty_gt, test_mode=test_mode) self.with_velocity = with_velocity self.eval_version = eval_version from nuscenes.eval.detection.config import config_factory self.eval_detection_configs = config_factory(self.eval_version)
def _lidar_nusc_box_to_global(info, boxes, classes, eval_version="detection_cvpr_2019"): import pyquaternion box_list = [] for box in boxes: # Move box to ego vehicle coord system box.rotate(pyquaternion.Quaternion(info['lidar2ego_rotation'])) box.translate(np.array(info['lidar2ego_translation'])) from nuscenes.eval.detection.config import config_factory from nuscenes.eval.detection.config import DetectionConfig cfg = config_factory(eval_version) cls_range_map = cfg.class_range radius = np.linalg.norm(box.center[:2], 2) det_range = cls_range_map[classes[box.label]] if radius > det_range: continue if np.sum(box.wlh) == 0: print("ZERO BOX") continue # Move box to global coord system box.rotate(pyquaternion.Quaternion(info['ego2global_rotation'])) box.translate(np.array(info['ego2global_translation'])) box_list.append(box) return box_list
def test_delta(self): """ This tests runs the evaluation for an arbitrary random set of predictions. This score is then captured in this very test such that if we change the eval code, this test will trigger if the results changed. """ random.seed(42) np.random.seed(42) assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.' nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False) with open(self.res_mockup, 'w') as f: json.dump(self._mock_submission(nusc, 'mini_val'), f, indent=2) cfg = config_factory('cvpr_2019') nusc_eval = NuScenesEval(nusc, cfg, self.res_mockup, eval_set='mini_val', output_dir=self.res_eval_folder, verbose=False) metrics, md_list = nusc_eval.evaluate() # 1. Score = 0.22082865720221012. Measured on the branch "release_v0.2" on March 7 2019. # 2. Score = 0.2199307290627096. Changed to measure center distance from the ego-vehicle. # 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script. # 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings. # 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall. # 6. Score = 0.20636954644294506. After bike-rack filtering. # 7. Score = 0.20237925145690996. After TP reversion bug. # 8. Score = 0.24047129251302665. After bike racks bug. # 9. Score = 0.24104572227466886. After bug fix in calc_tp. Include the max recall and exclude the min recall. # 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split. self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
def ground_truth_annotations(self): if "gt_boxes" not in self._nusc_infos[0]: return None cls_range_map = config_factory(self.eval_version).serialize()['class_range'] gt_annos = [] for info in self._nusc_infos: gt_names = np.array(info["gt_names"]) gt_boxes = info["gt_boxes"] mask = np.array([n != "ignore" for n in gt_names], dtype=np.bool_) gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] # det_range = np.array([cls_range_map[n] for n in gt_names_mapped]) det_range = np.array([cls_range_map[n] for n in gt_names]) det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]]) mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1) mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1) N = int(np.sum(mask)) gt_annos.append( { "bbox": np.tile(np.array([[0, 0, 50, 50]]), [N, 1]), "alpha": np.full(N, -10), "occluded": np.zeros(N), "truncated": np.zeros(N), "name": gt_names[mask], "location": gt_boxes[mask][:, :3], "dimensions": gt_boxes[mask][:, 3:6], "rotation_y": gt_boxes[mask][:, 6], "token": info["token"], } ) return gt_annos
def main(result_path, output_dir, eval_set, dataroot, version, verbose, config_name, plot_examples): # Init. cfg = config_factory(config_name) nusc_ = NuScenes(version=version, verbose=verbose, dataroot=dataroot) nusc_eval = NuScenesEval(nusc_, config=cfg, result_path=result_path, eval_set=eval_set, output_dir=output_dir, verbose=verbose) # Visualize samples. random.seed(43) if plot_examples: sample_tokens_ = list(nusc_eval.sample_tokens) random.shuffle(sample_tokens_) for sample_token_ in sample_tokens_: visualize_sample( nusc_, sample_token_, nusc_eval.gt_boxes, nusc_eval.pred_boxes, eval_range=max(nusc_eval.cfg.class_range.values()), savepath=os.path.join(output_dir, '{}.png'.format(sample_token_))) # Run evaluation. metrics, md_list = nusc_eval.run() nusc_eval.render(md_list, metrics)
def __init__(self, data_root, load_interval=1, with_velocity=True, modality=None, box_type_3d='Camera', eval_version='detection_cvpr_2019', use_valid_flag=False, version='v1.0-trainval', **kwargs): super().__init__(**kwargs) self.data_root = data_root self.load_interval = load_interval self.with_velocity = with_velocity self.modality = modality self.box_type_3d, self.box_mode_3d = get_box_type(box_type_3d) self.eval_version = eval_version self.use_valid_flag = use_valid_flag self.bbox_code_size = 9 self.version = version if self.eval_version is not None: from nuscenes.eval.detection.config import config_factory self.eval_detection_configs = config_factory(self.eval_version) if self.modality is None: self.modality = dict(use_camera=True, use_lidar=False, use_radar=False, use_map=False, use_external=False)
def __init__(self, root_path, info_path, class_names=None, prep_func=None, num_point_features=None): self._root_path = Path(root_path) with open(info_path, 'rb') as f: data = pickle.load(f) self._nusc_infos = list( sorted(data["infos"], key=lambda e: e["timestamp"])) self._metadata = data["metadata"] self._class_names = class_names self._prep_func = prep_func # kitti map: nusc det name -> kitti eval name self._kitti_name_mapping = { "car": "car", "pedestrian": "pedestrian", } # we only eval these classes in kitti self.version = self._metadata["version"] self.eval_version = 'cvpr_2019' self._eval_cfg = config_factory(self.eval_version) self._with_velocity = False print(self._metadata)
def ground_truth_annotations(self): if "gt_boxes" not in self._nusc_infos[0]: return None # from nuscenes.eval.detection.config import eval_detection_configs # cls_range_map = eval_detection_configs[self. # eval_version]["class_range"] from nuscenes.eval.detection.config import config_factory cfg = config_factory(self.eval_version) cls_range_map = cfg.class_range gt_annos = [] for info in self._nusc_infos: gt_names = info["gt_names"] gt_boxes = info["gt_boxes"] num_lidar_pts = info["num_lidar_pts"] mask = num_lidar_pts > 0 gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] num_lidar_pts = num_lidar_pts[mask] mask = np.array([n in self._kitti_name_mapping for n in gt_names], dtype=np.bool_) gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] num_lidar_pts = num_lidar_pts[mask] gt_names_mapped = [self._kitti_name_mapping[n] for n in gt_names] det_range = np.array([cls_range_map[n] for n in gt_names_mapped]) det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]]) mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1) mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1) gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] num_lidar_pts = num_lidar_pts[mask] # use occluded to control easy/moderate/hard in kitti easy_mask = num_lidar_pts > 15 moderate_mask = num_lidar_pts > 7 occluded = np.zeros([num_lidar_pts.shape[0]]) occluded[:] = 2 occluded[moderate_mask] = 1 occluded[easy_mask] = 0 N = len(gt_boxes) gt_annos.append({ "bbox": np.tile(np.array([[0, 0, 50, 50]]), [N, 1]), "alpha": np.full(N, -10), "occluded": occluded, "truncated": np.zeros(N), "name": gt_names, "location": gt_boxes[:, :3], "dimensions": gt_boxes[:, 3:6], "rotation_y": gt_boxes[:, 6], }) return gt_annos
def eval_main_old(root_path, version, eval_version, res_path, eval_set, output_dir): #import pdb; pdb.set_trace() nusc = NuScenes(version=version, dataroot=str(root_path), verbose=False) cfg = config_factory(eval_version) nusc_eval = NuScenesEval(nusc, config=cfg, result_path=res_path, eval_set=eval_set, output_dir=output_dir, verbose=False) nusc_eval.main(render_curves=False)
def execute_official_evaluator(nusc_path, result_path, output_path, task="detection", nusc_version="v1.0-trainval", eval_version="detection_cvpr_2019", verbose=True): ''' Call official evaluator implemented in Nuscenes devkit on evaluation set :param result_path: For detection, it's the path to the submission json file; For lidarseg, it's the path to the submission zip ''' from nuscenes import NuScenes nusc = NuScenes(version=nusc_version, dataroot=nusc_path, verbose=verbose) if task == "detection": from nuscenes.eval.detection.config import config_factory from nuscenes.eval.detection.evaluate import NuScenesEval nusc_eval = NuScenesEval(nusc, config=config_factory(eval_version), result_path=result_path, eval_set='val', output_dir=output_path, verbose=verbose) nusc_eval.main(render_curves=False) elif task == "lidarseg": from nuscenes.eval.lidarseg.evaluate import LidarSegEval tempfolder = tempfile.mkdtemp() if verbose: print("Extracting submission to", tempfolder) with zipfile.ZipFile(result_path, "r") as archive: archive.extractall(tempfolder) try: nusc_eval = LidarSegEval(nusc, results_folder=tempfolder, eval_set='val', verbose=verbose) results = nusc_eval.evaluate() if verbose: print("Results:", results) output_path = Path(output_path) output_path.mkdir(exist_ok=True, parents=True) with open(output_path / "lidarseg_results.json", "w") as fout: json.dump(results, fout, indent=" ") finally: shutil.rmtree(tempfolder) else: raise ValueError("Unsupported evaluation task!")
def eval_main(root_path, version, eval_version, res_path, eval_set, output_dir): nusc = NuScenes(version=version, dataroot=str(root_path), verbose=False) cfg = config_factory(eval_version) nusc_eval = NuScenesEval(nusc, config=cfg, result_path=res_path, eval_set=eval_set, output_dir=output_dir, verbose=False) nusc_eval.main(render_curves=True, plot_examples=10)
def eval_main(root_path, version, eval_version, res_path, eval_set, output_dir): nusc = NuScenes(version=version, dataroot=str(root_path), verbose=False) cfg = config_factory(eval_version) nusc_eval = NuScenesEval(nusc, config=cfg, result_path=res_path, eval_set=eval_set, output_dir=output_dir, verbose=False) _, _ = nusc_eval.run()
def eval_main(nusc, eval_version, res_path, eval_set, output_dir): # nusc = NuScenes(version=version, dataroot=str(root_path), verbose=True) cfg = config_factory(eval_version) nusc_eval = NuScenesEval( nusc, config=cfg, result_path=res_path, eval_set=eval_set, output_dir=output_dir, verbose=True, ) metrics_summary = nusc_eval.main(plot_examples=10, )
def eval(res_path, eval_set, output_dir=None, root_path=None, nusc=None): from nuscenes.eval.detection.evaluate import NuScenesEval from nuscenes.eval.detection.config import config_factory cfg = config_factory("detection_cvpr_2019") nusc_eval = NuScenesEval( nusc, config=cfg, result_path=res_path, eval_set=eval_set, output_dir=output_dir, verbose=True, ) metrics_summary = nusc_eval.main(render_curves=False)
def __init__(self, ann_file, pipeline=None, data_root=None, classes=None, load_interval=1, with_velocity=True, modality=None, box_type_3d='LiDAR', filter_empty_gt=True, test_mode=False, eval_version='detection_cvpr_2019', use_valid_flag=False, with_point_label=False): self.load_interval = load_interval self.use_valid_flag = use_valid_flag super().__init__(data_root=data_root, ann_file=ann_file, pipeline=pipeline, classes=classes, modality=modality, box_type_3d=box_type_3d, filter_empty_gt=filter_empty_gt, test_mode=test_mode) self.with_velocity = with_velocity self.eval_version = eval_version from nuscenes.eval.detection.config import config_factory self.eval_detection_configs = config_factory(self.eval_version) if self.modality is None: self.modality = dict( use_camera=False, use_lidar=True, use_radar=False, use_map=False, use_external=False, ) self.with_point_label = with_point_label if self.with_point_label: point_label_mapping = [] for name in NuScenesDataset.POINT_CLASS_GENERAL: point_label_mapping.append( NuScenesDataset.POINT_CLASS_SEG.index( NuScenesDataset.PointClassMapping[name])) self.point_label_mapping = np.array(point_label_mapping, dtype=np.uint8)
def get_box_mean(info_path, class_name="vehicle.car", eval_version="cvpr_2019"): with open(info_path, 'rb') as f: nusc_infos = pickle.load(f)["infos"] ## from nuscenes.eval.detection.config import config_factory # from nuscenes.eval.detection.config import eval_detection_configs from nuscenes.eval.detection.config import DetectionConfig # cls_range_map = eval_detection_configs[eval_version]["class_range"] cfg = config_factory(eval_version) cls_range_map = cfg.class_range gt_boxes_list = [] gt_vels_list = [] for info in nusc_infos: gt_boxes = info["gt_boxes"] gt_vels = info["gt_velocity"] gt_names = info["gt_names"] mask = np.array([s == class_name for s in info["gt_names"]], dtype=np.bool_) gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] gt_vels = gt_vels[mask] det_range = np.array([cls_range_map[n] for n in gt_names]) det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]]) mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1) mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1) gt_boxes_list.append(gt_boxes[mask].reshape(-1, 7)) gt_vels_list.append(gt_vels[mask].reshape(-1, 2)) gt_boxes_list = np.concatenate(gt_boxes_list, axis=0) gt_vels_list = np.concatenate(gt_vels_list, axis=0) nan_mask = np.isnan(gt_vels_list[:, 0]) gt_vels_list = gt_vels_list[~nan_mask] # return gt_vels_list.mean(0).tolist() return { "box3d": gt_boxes_list.mean(0).tolist(), "detail": gt_boxes_list # "velocity": gt_vels_list.mean(0).tolist(), }
def _lidar_nusc_box_to_global(info, boxes, classes, eval_version="cvpr_2019"): eval_cfg = config_factory(eval_version) box_list = [] for box in boxes: # Move box to ego vehicle coord system box.rotate(Quaternion(info['lidar2ego_rotation'])) box.translate(np.array(info['lidar2ego_translation'])) # filter det in ego. radius = np.linalg.norm(box.center[:2], 2) det_range = eval_cfg.class_range[classes[box.label]] if radius > det_range: continue # Move box to global coord system box.rotate(Quaternion(info['ego2global_rotation'])) box.translate(np.array(info['ego2global_translation'])) box_list.append(box) return box_list
def __init__( self, ann_file, pipeline=None, data_root=None, classes=None, load_interval=1, with_velocity=True, modality=None, box_type_3d="LiDAR", filter_empty_gt=True, test_mode=False, eval_version="detection_cvpr_2019", use_valid_flag=False, ): self.load_interval = load_interval self.use_valid_flag = use_valid_flag super().__init__( data_root=data_root, ann_file=ann_file, pipeline=pipeline, classes=classes, modality=modality, box_type_3d=box_type_3d, filter_empty_gt=filter_empty_gt, test_mode=test_mode, ) self.with_velocity = with_velocity self.eval_version = eval_version from nuscenes.eval.detection.config import config_factory self.eval_detection_configs = config_factory(self.eval_version) if self.modality is None: self.modality = dict( use_camera=False, use_lidar=True, use_radar=False, use_map=False, use_external=False, )
def _lidar_nusc_box_to_global(info, boxes, classes, eval_version="cvpr_2019"): import pyquaternion box_list = [] for box in boxes: # Move box to ego vehicle coord system box.rotate(pyquaternion.Quaternion(info['lidar2ego_rotation'])) box.translate(np.array(info['lidar2ego_translation'])) # from nuscenes.eval.detection.config import eval_detection_configs # filter det in ego. # cls_range_map = eval_detection_configs[eval_version]["class_range"] from nuscenes.eval.detection.config import config_factory cfg = config_factory(eval_version) cls_range_map = cfg.class_range radius = np.linalg.norm(box.center[:2], 2) det_range = cls_range_map[classes[box.label]] if radius > det_range: continue # Move box to global coord system box.rotate(pyquaternion.Quaternion(info['ego2global_rotation'])) box.translate(np.array(info['ego2global_translation'])) box_list.append(box) return box_list
def get_box_mean(info_path, class_name="vehicle.car", eval_version="cvpr_2019"): with open(info_path, 'rb') as f: nusc_infos = pickle.load(f)["infos"] eval_cfg = config_factory(eval_version) gt_boxes_list = [] gt_vels_list = [] for info in nusc_infos: gt_boxes = info["gt_boxes"] gt_vels = info["gt_velocity"] gt_names = info["gt_names"] mask = np.isin(gt_names, class_name) gt_names = gt_names[mask] gt_boxes = gt_boxes[mask] gt_vels = gt_vels[mask] det_range = np.array([eval_cfg.class_range[n] for n in gt_names]) det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]]) mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1) mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1) gt_boxes_list.append(gt_boxes[mask].reshape(-1, 7)) gt_vels_list.append(gt_vels[mask].reshape(-1, 2)) gt_boxes_list = np.concatenate(gt_boxes_list, axis=0) gt_vels_list = np.concatenate(gt_vels_list, axis=0) nan_mask = np.isnan(gt_vels_list[:, 0]) # gt_vels_list = gt_vels_list[~nan_mask] # return gt_vels_list.mean(0).tolist() return { "box3d": gt_boxes_list.mean(0).tolist(), "detail": gt_boxes_list # "velocity": gt_vels_list.mean(0).tolist(), }
def eval_main(nusc, eval_version, res_path, eval_set, output_dir): # nusc = NuScenes(version=version, dataroot=str(root_path), verbose=True) cfg = config_factory(eval_version) if nusc.version == "v1.0-trainval": train_scenes = splits.train # random.shuffle(train_scenes) # train_scenes = train_scenes[:int(len(train_scenes)*0.2)] val_scenes = splits.val elif nusc.version == "v1.0-test": train_scenes = splits.test val_scenes = [] elif nusc.version == "v1.0-mini": train_scenes = splits.mini_train val_scenes = splits.mini_val else: raise ValueError("unknown") available_scenes = _get_available_scenes(nusc) available_scene_names = [s["name"] for s in available_scenes] val_scenes = list(filter(lambda x: x in available_scene_names, val_scenes)) val_scenes = set([ available_scenes[available_scene_names.index(s)]["token"] for s in val_scenes ]) nusc_eval = NuScenesEval( nusc, config=cfg, result_path=res_path, eval_set=eval_set, output_dir=output_dir, verbose=True, val_scenes=val_scenes, ) metrics_summary = nusc_eval.main(plot_examples=10, )
help='How many example visualizations to write to disk.') parser.add_argument('--render_curves', type=int, default=1, help='Whether to render PR and TP curves to disk.') parser.add_argument('--verbose', type=int, default=1, help='Whether to print to stdout.') args = parser.parse_args() result_path_ = os.path.expanduser(args.result_path) output_dir_ = os.path.expanduser(args.output_dir) eval_set_ = args.eval_set dataroot_ = args.dataroot version_ = args.version config_name_ = args.config_name plot_examples_ = args.plot_examples render_curves_ = bool(args.render_curves) verbose_ = bool(args.verbose) cfg_ = config_factory(config_name_) nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_) nusc_eval = NuScenesEval(nusc_, config=cfg_, result_path=result_path_, eval_set=eval_set_, output_dir=output_dir_, verbose=verbose_) nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
def test_all_configs(self): """ Confirm that all configurations in eval_detection_configs work in the factory """ for name in eval_detection_configs.keys(): detect_config = config_factory(name) self.assertTrue(isinstance(detect_config, DetectionConfig))
def main(): args = parse_args() pred_paths = glob.glob(os.path.join(args.ensemble_dir, '*.pkl')) print(pred_paths) preds = [] for path in pred_paths: preds.append(get_pred(path)) merged_predictions = {} for token in preds[0].keys(): annos = [pred[token] for pred in preds] merged_predictions[token] = concatenate_list(annos) predictions = merged_predictions print("Finish Merging") nusc_annos = { "results": {}, "meta": None, } for sample_token, prediction in tqdm(predictions.items()): annos = [] # reorganize pred by class pred_dicts = reorganize_pred_by_class(prediction) for name, pred in pred_dicts.items(): # in global coordinate top_boxes, top_scores = get_sample_data(pred) with torch.no_grad(): top_boxes_tensor = torch.from_numpy(top_boxes) boxes_for_nms = top_boxes_tensor[:, [0, 1, 2, 4, 3, 5, -1]] boxes_for_nms[:, -1] = boxes_for_nms[:, -1] + np.pi / 2 top_scores_tensor = torch.from_numpy(top_scores) selected = box_torch_ops.rotate_nms( boxes_for_nms, top_scores_tensor, pre_max_size=None, post_max_size=50, iou_threshold=0.2, ).numpy() pred = [pred[s] for s in selected] annos.extend(pred) nusc_annos["results"].update({sample_token: annos}) nusc_annos["meta"] = { "use_camera": False, "use_lidar": True, "use_radar": True, "use_map": False, "use_external": False, } res_dir = os.path.join(args.work_dir) if not os.path.exists(res_dir): os.makedirs(res_dir) with open(os.path.join(args.work_dir, 'result.json'), "w") as f: json.dump(nusc_annos, f) from nuscenes.eval.detection.config import config_factory from nuscenes.eval.detection.evaluate import NuScenesEval nusc = NuScenes(version="v1.0-trainval", dataroot=args.data_root, verbose=True) cfg = config_factory("cvpr_2019") nusc_eval = NuScenesEval( nusc, config=cfg, result_path=os.path.join(args.work_dir, 'result.json'), eval_set='val', output_dir=args.work_dir, verbose=True, ) metrics_summary = nusc_eval.main(plot_examples=0, )
parser.add_argument('--verbose', type=int, default=1, help='Whether to print to stdout.') args = parser.parse_args() result_path_ = os.path.expanduser(args.result_path) output_dir_ = os.path.expanduser(args.output_dir) eval_set_ = args.eval_set dataroot_ = args.dataroot version_ = args.version config_path = args.config_path plot_examples_ = args.plot_examples render_curves_ = bool(args.render_curves) verbose_ = bool(args.verbose) if config_path == '': cfg_ = config_factory('cvpr_2019') else: with open(config_path, 'r') as f: cfg_ = DetectionConfig.deserialize(json.load(f)) nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_) nusc_eval = NuScenesEval(nusc_, config=cfg_, result_path=result_path_, eval_set=eval_set_, output_dir=output_dir_, verbose=verbose_) nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
def __init__(self, ann_file, pipeline=None, data_root=None, classes=None, load_interval=1, with_velocity=True, modality=None, box_type_3d='LiDAR', filter_empty_gt=True, test_mode=False, eval_version='detection_cvpr_2019', use_valid_flag=False): # Feng Xiang code # code begin self.AttrMapping = { 'cycle.with_rider': 0, 'cycle.without_rider': 1, 'pedestrian.moving': 2, 'pedestrian.standing': 3, 'pedestrian.sitting_lying_down': 4, 'vehicle.moving': 5, 'vehicle.parked': 6, 'vehicle.stopped': 7, } self.ATTR_CLASSES = ('cycle.with_rider', 'cycle.without_rider', 'pedestrian.moving', 'pedestrian.standing', 'pedestrian.sitting_lying_down', 'vehicle.moving', 'vehicle.parked', 'vehicle.stopped',) self.Attr_To_Class = { 'cycle.with_rider': 'bicycle', 'cycle.without_rider': 'bicycle', 'pedestrian.moving': 'pedestrian', 'pedestrian.standing': 'pedestrian', 'pedestrian.sitting_lying_down': 'pedestrian', 'vehicle.moving': 'car', 'vehicle.parked': 'car', 'vehicle.stopped': 'car', } # code end self.load_interval = load_interval self.use_valid_flag = use_valid_flag super().__init__( data_root=data_root, ann_file=ann_file, pipeline=pipeline, classes=classes, modality=modality, box_type_3d=box_type_3d, filter_empty_gt=filter_empty_gt, test_mode=test_mode) self.with_velocity = with_velocity self.eval_version = eval_version from nuscenes.eval.detection.config import config_factory # Feng Xiang # detection_cvpr_2019 json file is located in # /home/ubuntu/anaconda3/env/open-mmlab/lib/python3.7/site-packages/nuscenes/eval/detection self.eval_detection_configs = config_factory(self.eval_version) if self.modality is None: self.modality = dict( use_camera=False, use_lidar=True, use_radar=False, use_map=False, use_external=False, )
def evaluation(self, det_annos, class_names, **kwargs): eval_det_annos = copy.deepcopy(det_annos) # Create NuScenes JSON output file nusc_annos = {} for sample in eval_det_annos: try: sample_idx = sample['sample_idx'][0] except: continue sample_results = [] calib = self.get_calib(sample_idx) sample['boxes_lidar'] = np.array(sample['boxes_lidar']) positions = sample['boxes_lidar'][:, :3] dimensions = sample['boxes_lidar'][:, 3:6] rotations = sample['boxes_lidar'][:, 6] for center, dimension, yaw, label, score in zip( positions, dimensions, rotations, sample['name'], sample['score']): quaternion = Quaternion(axis=[0, 0, 1], radians=yaw) box = Box(center, dimension, quaternion) # Move box to ego vehicle coord system box.rotate(Quaternion(calib.lidar_calibrated['rotation'])) box.translate(np.array(calib.lidar_calibrated['translation'])) # Move box to global coord system box.rotate(Quaternion(calib.ego_pose['rotation'])) box.translate(np.array(calib.ego_pose['translation'])) if (float(score) < 0): score = 0 if (float(score) > 1): score = 1 if (label == 'Cyclist'): label = 'bicycle' sample_results.append({ "sample_token": sample_idx, "translation": box.center.tolist(), "size": box.wlh.tolist(), "rotation": box.orientation.elements.tolist(), "lidar_yaw": float(yaw), "velocity": (0, 0), "detection_name": label.lower(), "detection_score": float(score), "attribute_name": self.DefaultAttribute[label.lower()], }) nusc_annos[sample_idx] = sample_results for sample_id in self.sample_id_list: if sample_id not in nusc_annos: nusc_annos[sample_id] = [] nusc_submission = { "meta": { "use_camera": False, "use_lidar": True, "use_radar": False, "use_map": False, "use_external": False, }, "results": nusc_annos, } eval_file = os.path.join(kwargs['output_dir'], 'nusc_results.json') with open(eval_file, "w") as f: json.dump(nusc_submission, f, indent=2) # Call NuScenes evaluation cfg = config_factory('detection_cvpr_2019') nusc_eval = DetectionEval(self.nusc, config=cfg, result_path=eval_file, eval_set=self.split, output_dir=kwargs['output_dir'], verbose=True) metric_summary = nusc_eval.main(plot_examples=10, render_curves=True) # Reformat the metrics summary a bit for the tensorboard logger err_name_mapping = { 'trans_err': 'mATE', 'scale_err': 'mASE', 'orient_err': 'mAOE', 'vel_err': 'mAVE', 'attr_err': 'mAAE' } result = {} result['mean_ap'] = metric_summary['mean_ap'] for tp_name, tp_val in metric_summary['tp_errors'].items(): result[tp_name] = tp_val class_aps = metric_summary['mean_dist_aps'] class_tps = metric_summary['label_tp_errors'] for class_name in class_aps.keys(): result['mAP_' + class_name] = class_aps[class_name] for key, val in err_name_mapping.items(): result[val + '_' + class_name] = class_tps[class_name][key] return str(result), result
class TestAlgo(unittest.TestCase): this_dir = os.path.dirname(os.path.abspath(__file__)) cfg = config_factory('cvpr_2019') @staticmethod def _mock_results(nsamples, ngt, npred, detection_name): def random_attr(): """ This is the most straight-forward way to generate a random attribute. Not currently used b/c we want the test fixture to be back-wards compatible. """ # Get relevant attributes. rel_attributes = detection_name_to_rel_attributes(detection_name) if len(rel_attributes) == 0: # Empty string for classes without attributes. return '' else: # Pick a random attribute otherwise. return rel_attributes[np.random.randint(0, len(rel_attributes))] pred = EvalBoxes() gt = EvalBoxes() for sample_itt in range(nsamples): this_gt = [] for box_itt in range(ngt): this_gt.append(EvalBox( sample_token=str(sample_itt), translation=tuple(list(np.random.rand(2)*15) + [0.0]), size=tuple(np.random.rand(3)*4), rotation=tuple(np.random.rand(4)), velocity=tuple(np.random.rand(3)[:2]*4), detection_name=detection_name, detection_score=random.random(), attribute_name=random_attr(), ego_dist=random.random()*10, )) gt.add_boxes(str(sample_itt), this_gt) for sample_itt in range(nsamples): this_pred = [] for box_itt in range(npred): this_pred.append(EvalBox( sample_token=str(sample_itt), translation=tuple(list(np.random.rand(2) * 10) + [0.0]), size=tuple(np.random.rand(3) * 4), rotation=tuple(np.random.rand(4)), velocity=tuple(np.random.rand(3)[:2] * 4), detection_name=detection_name, detection_score=random.random(), attribute_name=random_attr(), ego_dist=random.random() * 10, )) pred.add_boxes(str(sample_itt), this_pred) return gt, pred def test_weighted_sum(self): """ This tests runs the full evaluation for an arbitrary random set of predictions. """ random.seed(42) np.random.seed(42) mdl = MetricDataList() for class_name in self.cfg.class_names: gt, pred = self._mock_results(30, 3, 25, class_name) for dist_th in self.cfg.dist_ths: mdl.set(class_name, dist_th, accumulate(gt, pred, class_name, 'center_distance', 2)) metrics = DetectionMetrics(self.cfg) for class_name in self.cfg.class_names: for dist_th in self.cfg.dist_ths: ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall, self.cfg.min_precision) metrics.add_label_ap(class_name, dist_th, ap) for metric_name in TP_METRICS: metric_data = mdl[(class_name, self.cfg.dist_th_tp)] if class_name in ['traffic_cone'] and metric_name in ['attr_err', 'vel_err', 'orient_err']: tp = np.nan elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']: tp = np.nan else: tp = calc_tp(metric_data, self.cfg.min_recall, metric_name) metrics.add_label_tp(class_name, metric_name, tp) self.assertEqual(0.08606662159639042, metrics.weighted_sum) def test_calc_tp(self): """Test for calc_tp().""" random.seed(42) np.random.seed(42) md = MetricData.random_md() # min_recall greater than 1. self.assertEqual(1.0, calc_tp(md, min_recall=1, metric_name='trans_err')) def test_calc_ap(self): """Test for calc_ap().""" random.seed(42) np.random.seed(42) md = MetricData.random_md() # Negative min_recall and min_precision self.assertRaises(AssertionError, calc_ap, md, -0.5, 0.4) self.assertRaises(AssertionError, calc_ap, md, 0.5, -0.8) # More than 1 min_precision/min_recall self.assertRaises(AssertionError, calc_ap, md, 0.7, 1) self.assertRaises(AssertionError, calc_ap, md, 1.2, 0)