def __init__(self,
nusc: NuScenes,
config: DetectionConfig,
result_path: str,
eval_set: str,
output_dir: str = None,
verbose: bool = True):
"""
Initialize a DetectionEval object.
:param nusc: A NuScenes object.
:param config: A DetectionConfig object.
:param result_path: Path of the nuScenes JSON result file.
:param eval_set: The dataset split to evaluate on, e.g. train, val or test.
:param output_dir: Folder to save plots and results to.
:param verbose: Whether to print to stdout.
"""
self.nusc = nusc
self.result_path = result_path
self.eval_set = eval_set
self.output_dir = output_dir
self.verbose = verbose
self.cfg = config
# Check result file exists.
assert os.path.exists(result_path), 'Error: The result file does not exist!'
# Make dirs.
self.plot_dir = os.path.join(self.output_dir, 'plots')
if not os.path.isdir(self.output_dir):
os.makedirs(self.output_dir)
if not os.path.isdir(self.plot_dir):
os.makedirs(self.plot_dir)
# Load data.
if verbose:
print('Initializing nuScenes detection evaluation')
self.pred_boxes, self.meta = load_prediction(self.result_path, self.cfg.max_boxes_per_sample, DetectionBox,
verbose=verbose)
self.gt_boxes = load_gt(self.nusc, self.eval_set, DetectionBox, verbose=verbose)
assert set(self.pred_boxes.sample_tokens) == set(self.gt_boxes.sample_tokens), \
"Samples in split doesn't match samples in predictions."
# Add center distances.
self.pred_boxes = add_center_dist(nusc, self.pred_boxes)
self.gt_boxes = add_center_dist(nusc, self.gt_boxes)
# Filter boxes (distance, points per box, etc.).
if verbose:
print('Filtering predictions')
self.pred_boxes = filter_eval_boxes(nusc, self.pred_boxes, self.cfg.class_range, verbose=verbose)
if verbose:
print('Filtering ground truth annotations')
self.gt_boxes = filter_eval_boxes(nusc, self.gt_boxes, self.cfg.class_range, verbose=verbose)
self.sample_tokens = self.gt_boxes.sample_tokens
def generate_perfect(version, eval_set, dataroot='/data/nuscenes'):
"""Generate perfect detections.
"""
nusc = NuScenes(version='v1.0-{}'.format(version),
dataroot=dataroot,
verbose=True)
cfg = config_factory('detection_cvpr_2019')
gt_boxes = load_gt(nusc, eval_set, DetectionBox, verbose=True)
gt_boxes = add_center_dist(nusc, gt_boxes)
gt_boxes = filter_eval_boxes(nusc, gt_boxes, cfg.class_range, verbose=True)
submission = synthetic_noise_trunk(gt_boxes)
outname = f'perfect_{version}_{eval_set}.json'
print('saving', outname)
with open(outname, 'w') as writer:
json.dump(submission, writer)
def generate_drop_noise(version, eval_set, drop_p, dataroot='/data/nuscenes'):
"""Generate pseudo submissions where every box is dropped with
probability p.
"""
nusc = NuScenes(version='v1.0-{}'.format(version),
dataroot=dataroot,
verbose=True)
cfg = config_factory('detection_cvpr_2019')
gt_boxes = load_gt(nusc, eval_set, DetectionBox, verbose=True)
gt_boxes = add_center_dist(nusc, gt_boxes)
gt_boxes = filter_eval_boxes(nusc, gt_boxes, cfg.class_range, verbose=True)
for drop_p in [drop_p]:
submission = synthetic_noise_trunk(gt_boxes, drop_p=drop_p)
outname = f'perfect_{version}_{eval_set}_{drop_p}.json'
print('saving', outname)
with open(outname, 'w') as writer:
json.dump(submission, writer)
detection_file = '/juno/u/hkchiu/dataset/nuscenes_new/megvii_train.json'
data_root = '/juno/u/hkchiu/dataset/nuscenes/trainval'
version = 'v1.0-trainval'
elif 'val' in eval_set_:
detection_file = '/juno/u/hkchiu/dataset/nuscenes_new/megvii_val.json'
data_root = '/juno/u/hkchiu/dataset/nuscenes/trainval'
version = 'v1.0-trainval'
elif 'test' in eval_set_:
detection_file = '/juno/u/hkchiu/dataset/nuscenes_new/megvii_test.json'
data_root = '/juno/u/hkchiu/dataset/nuscenes/test'
version = 'v1.0-test'
nusc = NuScenes(version=version, dataroot=data_root, verbose=True)
pred_boxes, _ = load_prediction(detection_file, 10000, DetectionBox)
gt_boxes = load_gt(nusc, eval_set_, TrackingBox)
assert set(pred_boxes.sample_tokens) == set(gt_boxes.sample_tokens), \
"Samples in split don't match samples in predicted tracks."
# Add center distances.
pred_boxes = add_center_dist(nusc, pred_boxes)
gt_boxes = add_center_dist(nusc, gt_boxes)
print('len(pred_boxes.sample_tokens): ', len(pred_boxes.sample_tokens))
print('len(gt_boxes.sample_tokens): ', len(gt_boxes.sample_tokens))
tracks_gt = create_tracks(gt_boxes, nusc, eval_set_, gt=True)
mean, std, var = get_mean(tracks_gt)
print('GT: Global coordinate system')
def __init__(self,
config: TrackingConfig,
result_path: str,
eval_set: str,
output_dir: str,
nusc_version: str,
nusc_dataroot: str,
verbose: bool = True,
render_classes: List[str] = None):
"""
Initialize a TrackingEval object.
:param config: A TrackingConfig object.
:param result_path: Path of the nuScenes JSON result file.
:param eval_set: The dataset split to evaluate on, e.g. train, val or test.
:param output_dir: Folder to save plots and results to.
:param nusc_version: The version of the NuScenes dataset.
:param nusc_dataroot: Path of the nuScenes dataset on disk.
:param verbose: Whether to print to stdout.
:param render_classes: Classes to render to disk or None.
"""
self.cfg = config
self.result_path = result_path
self.eval_set = eval_set
self.output_dir = output_dir
self.verbose = verbose
self.render_classes = render_classes
# Check result file exists.
assert os.path.exists(
result_path), 'Error: The result file does not exist!'
# Make dirs.
self.plot_dir = os.path.join(self.output_dir, 'plots')
if not os.path.isdir(self.output_dir):
os.makedirs(self.output_dir)
if not os.path.isdir(self.plot_dir):
os.makedirs(self.plot_dir)
# Initialize NuScenes object.
# We do not store it in self to let garbage collection take care of it and save memory.
nusc = NuScenes(version=nusc_version,
verbose=verbose,
dataroot=nusc_dataroot)
self.nusc = nusc
# Load data.
if verbose:
print('Initializing nuScenes tracking evaluation')
pred_boxes, self.meta = load_prediction(self.result_path,
self.cfg.max_boxes_per_sample,
TrackingBox,
verbose=verbose)
gt_boxes = load_gt(nusc, self.eval_set, TrackingBox, verbose=verbose)
assert set(pred_boxes.sample_tokens) == set(gt_boxes.sample_tokens), \
"Samples in split don't match samples in predicted tracks."
# Add center distances.
pred_boxes = add_center_dist(nusc, pred_boxes)
gt_boxes = add_center_dist(nusc, gt_boxes)
# Filter boxes (distance, points per box, etc.).
if verbose:
print('Filtering tracks')
pred_boxes = filter_eval_boxes(nusc,
pred_boxes,
self.cfg.class_range,
verbose=verbose)
if verbose:
print('Filtering ground truth tracks')
gt_boxes = filter_eval_boxes(nusc,
gt_boxes,
self.cfg.class_range,
verbose=verbose)
self.sample_tokens = gt_boxes.sample_tokens #len():6019
# Convert boxes to tracks format.
self.tracks_gt = create_tracks(gt_boxes, nusc, self.eval_set, gt=True)
self.tracks_pred = create_tracks(pred_boxes,
nusc,
self.eval_set,
gt=False)