def evaluation(self, det_annos, class_names, **kwargs):
import json
from nuscenes.nuscenes import NuScenes
from . import nuscenes_utils
nusc = NuScenes(version=self.dataset_cfg.VERSION,
dataroot=str(self.root_path),
verbose=True)
nusc_annos = nuscenes_utils.transform_det_annos_to_nusc_annos(
det_annos, nusc)
nusc_annos['meta'] = {
'use_camera': False,
'use_lidar': True,
'use_radar': False,
'use_map': False,
'use_external': False,
}
output_path = Path(kwargs['output_path'])
output_path.mkdir(exist_ok=True, parents=True)
res_path = str(output_path / 'results_nusc.json')
with open(res_path, 'w') as f:
json.dump(nusc_annos, f)
self.logger.info(
f'The predictions of NuScenes have been saved to {res_path}')
if self.dataset_cfg.VERSION == 'v1.0-test':
return 'No ground-truth annotations for evaluation', {}
from nuscenes.eval.detection.config import config_factory
from nuscenes.eval.detection.evaluate import NuScenesEval
eval_set_map = {
'v1.0-mini': 'mini_val',
'v1.0-trainval': 'val',
'v1.0-test': 'test'
}
try:
eval_version = 'detection_cvpr_2019'
eval_config = config_factory(eval_version)
except:
eval_version = 'cvpr_2019'
eval_config = config_factory(eval_version)
nusc_eval = NuScenesEval(
nusc,
config=eval_config,
result_path=res_path,
eval_set=eval_set_map[self.dataset_cfg.VERSION],
output_dir=str(output_path),
verbose=True,
)
metrics_summary = nusc_eval.main(plot_examples=0, render_curves=False)
with open(output_path / 'metrics_summary.json', 'r') as f:
metrics = json.load(f)
result_str, result_dict = nuscenes_utils.format_nuscene_results(
metrics, self.class_names, version=eval_version)
return result_str, result_dict
def cluster_trailer_box(info_path, class_name="bus"):
with open(info_path, 'rb') as f:
nusc_infos = pickle.load(f)["infos"]
from nuscenes.eval.detection.config import config_factory
from nuscenes.eval.detection.config import DetectionConfig
cfg = config_factory(self.eval_version)
cls_range_map = cfg.class_range
gt_boxes_list = []
for info in nusc_infos:
gt_boxes = info["gt_boxes"]
gt_names = info["gt_names"]
mask = np.array([s == class_name for s in info["gt_names"]],
dtype=np.bool_)
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
det_range = np.array([cls_range_map[n] for n in gt_names])
det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]])
mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1)
mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1)
gt_boxes_list.append(gt_boxes[mask].reshape(-1, 7))
gt_boxes_list = np.concatenate(gt_boxes_list, axis=0)
trailer_dims = gt_boxes_list[:, 3:6]
from sklearn.cluster import DBSCAN
from sklearn.preprocessing import StandardScaler
X = StandardScaler().fit_transform(trailer_dims)
db = DBSCAN(eps=0.3, min_samples=10).fit(X)
core_samples_mask = np.zeros_like(db.labels_, dtype=bool)
core_samples_mask[db.core_sample_indices_] = True
labels = db.labels_
n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)
n_noise_ = list(labels).count(-1)
print(n_clusters_, n_noise_)
print(trailer_dims)
import matplotlib.pyplot as plt
unique_labels = set(labels)
colors = [
plt.cm.Spectral(each) for each in np.linspace(0, 1, len(unique_labels))
]
for k, col in zip(unique_labels, colors):
if k == -1:
# Black used for noise.
col = [0, 0, 0, 1]
class_member_mask = (labels == k)
xy = trailer_dims[class_member_mask & core_samples_mask]
plt.plot(xy[:, 0],
xy[:, 1],
'o',
markerfacecolor=tuple(col),
markeredgecolor='k',
markersize=14)
xy = trailer_dims[class_member_mask & ~core_samples_mask]
plt.plot(xy[:, 0],
xy[:, 1],
'o',
markerfacecolor=tuple(col),
markeredgecolor='k',
markersize=6)
plt.show()
def __init__(self,
ann_file,
pipeline=None,
data_root=None,
classes=None,
load_interval=1,
with_velocity=True,
modality=None,
box_type_3d='LiDAR',
filter_empty_gt=True,
test_mode=False,
eval_version='mmda_merge',
use_valid_flag=False):
self.load_interval = load_interval
self.use_valid_flag = use_valid_flag
assert classes is not None
super().__init__(data_root=data_root,
ann_file=ann_file,
pipeline=pipeline,
classes=classes,
modality=modality,
box_type_3d=box_type_3d,
filter_empty_gt=filter_empty_gt,
test_mode=test_mode)
self.with_velocity = with_velocity
self.eval_version = eval_version
from nuscenes.eval.detection.config import config_factory
self.eval_detection_configs = config_factory(self.eval_version)
def _lidar_nusc_box_to_global(info,
boxes,
classes,
eval_version="detection_cvpr_2019"):
import pyquaternion
box_list = []
for box in boxes:
# Move box to ego vehicle coord system
box.rotate(pyquaternion.Quaternion(info['lidar2ego_rotation']))
box.translate(np.array(info['lidar2ego_translation']))
from nuscenes.eval.detection.config import config_factory
from nuscenes.eval.detection.config import DetectionConfig
cfg = config_factory(eval_version)
cls_range_map = cfg.class_range
radius = np.linalg.norm(box.center[:2], 2)
det_range = cls_range_map[classes[box.label]]
if radius > det_range:
continue
if np.sum(box.wlh) == 0:
print("ZERO BOX")
continue
# Move box to global coord system
box.rotate(pyquaternion.Quaternion(info['ego2global_rotation']))
box.translate(np.array(info['ego2global_translation']))
box_list.append(box)
return box_list
def test_delta(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
random.seed(42)
np.random.seed(42)
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini', dataroot=os.environ['NUSCENES'], verbose=False)
with open(self.res_mockup, 'w') as f:
json.dump(self._mock_submission(nusc, 'mini_val'), f, indent=2)
cfg = config_factory('cvpr_2019')
nusc_eval = NuScenesEval(nusc, cfg, self.res_mockup, eval_set='mini_val', output_dir=self.res_eval_folder,
verbose=False)
metrics, md_list = nusc_eval.evaluate()
# 1. Score = 0.22082865720221012. Measured on the branch "release_v0.2" on March 7 2019.
# 2. Score = 0.2199307290627096. Changed to measure center distance from the ego-vehicle.
# 3. Score = 0.24954451673961747. Changed to 1.0-mini and cleaned up build script.
# 4. Score = 0.20478832626986893. Updated treatment of cones, barriers, and other algo tunings.
# 5. Score = 0.2043569666105005. AP calculation area is changed from >=min_recall to >min_recall.
# 6. Score = 0.20636954644294506. After bike-rack filtering.
# 7. Score = 0.20237925145690996. After TP reversion bug.
# 8. Score = 0.24047129251302665. After bike racks bug.
# 9. Score = 0.24104572227466886. After bug fix in calc_tp. Include the max recall and exclude the min recall.
# 10. Score = 0.19449091580477748. Changed to use v1.0 mini_val split.
self.assertAlmostEqual(metrics.nd_score, 0.19449091580477748)
def ground_truth_annotations(self):
if "gt_boxes" not in self._nusc_infos[0]:
return None
cls_range_map = config_factory(self.eval_version).serialize()['class_range']
gt_annos = []
for info in self._nusc_infos:
gt_names = np.array(info["gt_names"])
gt_boxes = info["gt_boxes"]
mask = np.array([n != "ignore" for n in gt_names], dtype=np.bool_)
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
# det_range = np.array([cls_range_map[n] for n in gt_names_mapped])
det_range = np.array([cls_range_map[n] for n in gt_names])
det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]])
mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1)
mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1)
N = int(np.sum(mask))
gt_annos.append(
{
"bbox": np.tile(np.array([[0, 0, 50, 50]]), [N, 1]),
"alpha": np.full(N, -10),
"occluded": np.zeros(N),
"truncated": np.zeros(N),
"name": gt_names[mask],
"location": gt_boxes[mask][:, :3],
"dimensions": gt_boxes[mask][:, 3:6],
"rotation_y": gt_boxes[mask][:, 6],
"token": info["token"],
}
)
return gt_annos
def main(result_path, output_dir, eval_set, dataroot, version, verbose,
config_name, plot_examples):
# Init.
cfg = config_factory(config_name)
nusc_ = NuScenes(version=version, verbose=verbose, dataroot=dataroot)
nusc_eval = NuScenesEval(nusc_,
config=cfg,
result_path=result_path,
eval_set=eval_set,
output_dir=output_dir,
verbose=verbose)
# Visualize samples.
random.seed(43)
if plot_examples:
sample_tokens_ = list(nusc_eval.sample_tokens)
random.shuffle(sample_tokens_)
for sample_token_ in sample_tokens_:
visualize_sample(
nusc_,
sample_token_,
nusc_eval.gt_boxes,
nusc_eval.pred_boxes,
eval_range=max(nusc_eval.cfg.class_range.values()),
savepath=os.path.join(output_dir,
'{}.png'.format(sample_token_)))
# Run evaluation.
metrics, md_list = nusc_eval.run()
nusc_eval.render(md_list, metrics)
def __init__(self,
data_root,
load_interval=1,
with_velocity=True,
modality=None,
box_type_3d='Camera',
eval_version='detection_cvpr_2019',
use_valid_flag=False,
version='v1.0-trainval',
**kwargs):
super().__init__(**kwargs)
self.data_root = data_root
self.load_interval = load_interval
self.with_velocity = with_velocity
self.modality = modality
self.box_type_3d, self.box_mode_3d = get_box_type(box_type_3d)
self.eval_version = eval_version
self.use_valid_flag = use_valid_flag
self.bbox_code_size = 9
self.version = version
if self.eval_version is not None:
from nuscenes.eval.detection.config import config_factory
self.eval_detection_configs = config_factory(self.eval_version)
if self.modality is None:
self.modality = dict(use_camera=True,
use_lidar=False,
use_radar=False,
use_map=False,
use_external=False)
def __init__(self,
root_path,
info_path,
class_names=None,
prep_func=None,
num_point_features=None):
self._root_path = Path(root_path)
with open(info_path, 'rb') as f:
data = pickle.load(f)
self._nusc_infos = list(
sorted(data["infos"], key=lambda e: e["timestamp"]))
self._metadata = data["metadata"]
self._class_names = class_names
self._prep_func = prep_func
# kitti map: nusc det name -> kitti eval name
self._kitti_name_mapping = {
"car": "car",
"pedestrian": "pedestrian",
} # we only eval these classes in kitti
self.version = self._metadata["version"]
self.eval_version = 'cvpr_2019'
self._eval_cfg = config_factory(self.eval_version)
self._with_velocity = False
print(self._metadata)
def ground_truth_annotations(self):
if "gt_boxes" not in self._nusc_infos[0]:
return None
# from nuscenes.eval.detection.config import eval_detection_configs
# cls_range_map = eval_detection_configs[self.
# eval_version]["class_range"]
from nuscenes.eval.detection.config import config_factory
cfg = config_factory(self.eval_version)
cls_range_map = cfg.class_range
gt_annos = []
for info in self._nusc_infos:
gt_names = info["gt_names"]
gt_boxes = info["gt_boxes"]
num_lidar_pts = info["num_lidar_pts"]
mask = num_lidar_pts > 0
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
num_lidar_pts = num_lidar_pts[mask]
mask = np.array([n in self._kitti_name_mapping for n in gt_names],
dtype=np.bool_)
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
num_lidar_pts = num_lidar_pts[mask]
gt_names_mapped = [self._kitti_name_mapping[n] for n in gt_names]
det_range = np.array([cls_range_map[n] for n in gt_names_mapped])
det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]])
mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1)
mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1)
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
num_lidar_pts = num_lidar_pts[mask]
# use occluded to control easy/moderate/hard in kitti
easy_mask = num_lidar_pts > 15
moderate_mask = num_lidar_pts > 7
occluded = np.zeros([num_lidar_pts.shape[0]])
occluded[:] = 2
occluded[moderate_mask] = 1
occluded[easy_mask] = 0
N = len(gt_boxes)
gt_annos.append({
"bbox":
np.tile(np.array([[0, 0, 50, 50]]), [N, 1]),
"alpha":
np.full(N, -10),
"occluded":
occluded,
"truncated":
np.zeros(N),
"name":
gt_names,
"location":
gt_boxes[:, :3],
"dimensions":
gt_boxes[:, 3:6],
"rotation_y":
gt_boxes[:, 6],
})
return gt_annos
def eval_main_old(root_path, version, eval_version, res_path, eval_set, output_dir):
#import pdb; pdb.set_trace()
nusc = NuScenes(version=version, dataroot=str(root_path), verbose=False)
cfg = config_factory(eval_version)
nusc_eval = NuScenesEval(nusc, config=cfg, result_path=res_path, eval_set=eval_set,
output_dir=output_dir,
verbose=False)
nusc_eval.main(render_curves=False)
def execute_official_evaluator(nusc_path,
result_path,
output_path,
task="detection",
nusc_version="v1.0-trainval",
eval_version="detection_cvpr_2019",
verbose=True):
'''
Call official evaluator implemented in Nuscenes devkit on evaluation set
:param result_path: For detection, it's the path to the submission json file; For lidarseg,
it's the path to the submission zip
'''
from nuscenes import NuScenes
nusc = NuScenes(version=nusc_version, dataroot=nusc_path, verbose=verbose)
if task == "detection":
from nuscenes.eval.detection.config import config_factory
from nuscenes.eval.detection.evaluate import NuScenesEval
nusc_eval = NuScenesEval(nusc,
config=config_factory(eval_version),
result_path=result_path,
eval_set='val',
output_dir=output_path,
verbose=verbose)
nusc_eval.main(render_curves=False)
elif task == "lidarseg":
from nuscenes.eval.lidarseg.evaluate import LidarSegEval
tempfolder = tempfile.mkdtemp()
if verbose:
print("Extracting submission to", tempfolder)
with zipfile.ZipFile(result_path, "r") as archive:
archive.extractall(tempfolder)
try:
nusc_eval = LidarSegEval(nusc,
results_folder=tempfolder,
eval_set='val',
verbose=verbose)
results = nusc_eval.evaluate()
if verbose:
print("Results:", results)
output_path = Path(output_path)
output_path.mkdir(exist_ok=True, parents=True)
with open(output_path / "lidarseg_results.json", "w") as fout:
json.dump(results, fout, indent=" ")
finally:
shutil.rmtree(tempfolder)
else:
raise ValueError("Unsupported evaluation task!")
def eval_main(root_path, version, eval_version, res_path, eval_set,
output_dir):
nusc = NuScenes(version=version, dataroot=str(root_path), verbose=False)
cfg = config_factory(eval_version)
nusc_eval = NuScenesEval(nusc,
config=cfg,
result_path=res_path,
eval_set=eval_set,
output_dir=output_dir,
verbose=False)
nusc_eval.main(render_curves=True, plot_examples=10)
def eval_main(root_path, version, eval_version, res_path, eval_set,
output_dir):
nusc = NuScenes(version=version, dataroot=str(root_path), verbose=False)
cfg = config_factory(eval_version)
nusc_eval = NuScenesEval(nusc,
config=cfg,
result_path=res_path,
eval_set=eval_set,
output_dir=output_dir,
verbose=False)
_, _ = nusc_eval.run()
def eval_main(nusc, eval_version, res_path, eval_set, output_dir):
# nusc = NuScenes(version=version, dataroot=str(root_path), verbose=True)
cfg = config_factory(eval_version)
nusc_eval = NuScenesEval(
nusc,
config=cfg,
result_path=res_path,
eval_set=eval_set,
output_dir=output_dir,
verbose=True,
)
metrics_summary = nusc_eval.main(plot_examples=10, )
def eval(res_path, eval_set, output_dir=None, root_path=None, nusc=None):
from nuscenes.eval.detection.evaluate import NuScenesEval
from nuscenes.eval.detection.config import config_factory
cfg = config_factory("detection_cvpr_2019")
nusc_eval = NuScenesEval(
nusc,
config=cfg,
result_path=res_path,
eval_set=eval_set,
output_dir=output_dir,
verbose=True,
)
metrics_summary = nusc_eval.main(render_curves=False)
def __init__(self,
ann_file,
pipeline=None,
data_root=None,
classes=None,
load_interval=1,
with_velocity=True,
modality=None,
box_type_3d='LiDAR',
filter_empty_gt=True,
test_mode=False,
eval_version='detection_cvpr_2019',
use_valid_flag=False,
with_point_label=False):
self.load_interval = load_interval
self.use_valid_flag = use_valid_flag
super().__init__(data_root=data_root,
ann_file=ann_file,
pipeline=pipeline,
classes=classes,
modality=modality,
box_type_3d=box_type_3d,
filter_empty_gt=filter_empty_gt,
test_mode=test_mode)
self.with_velocity = with_velocity
self.eval_version = eval_version
from nuscenes.eval.detection.config import config_factory
self.eval_detection_configs = config_factory(self.eval_version)
if self.modality is None:
self.modality = dict(
use_camera=False,
use_lidar=True,
use_radar=False,
use_map=False,
use_external=False,
)
self.with_point_label = with_point_label
if self.with_point_label:
point_label_mapping = []
for name in NuScenesDataset.POINT_CLASS_GENERAL:
point_label_mapping.append(
NuScenesDataset.POINT_CLASS_SEG.index(
NuScenesDataset.PointClassMapping[name]))
self.point_label_mapping = np.array(point_label_mapping,
dtype=np.uint8)
def get_box_mean(info_path,
class_name="vehicle.car",
eval_version="cvpr_2019"):
with open(info_path, 'rb') as f:
nusc_infos = pickle.load(f)["infos"]
##
from nuscenes.eval.detection.config import config_factory
# from nuscenes.eval.detection.config import eval_detection_configs
from nuscenes.eval.detection.config import DetectionConfig
# cls_range_map = eval_detection_configs[eval_version]["class_range"]
cfg = config_factory(eval_version)
cls_range_map = cfg.class_range
gt_boxes_list = []
gt_vels_list = []
for info in nusc_infos:
gt_boxes = info["gt_boxes"]
gt_vels = info["gt_velocity"]
gt_names = info["gt_names"]
mask = np.array([s == class_name for s in info["gt_names"]],
dtype=np.bool_)
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
gt_vels = gt_vels[mask]
det_range = np.array([cls_range_map[n] for n in gt_names])
det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]])
mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1)
mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1)
gt_boxes_list.append(gt_boxes[mask].reshape(-1, 7))
gt_vels_list.append(gt_vels[mask].reshape(-1, 2))
gt_boxes_list = np.concatenate(gt_boxes_list, axis=0)
gt_vels_list = np.concatenate(gt_vels_list, axis=0)
nan_mask = np.isnan(gt_vels_list[:, 0])
gt_vels_list = gt_vels_list[~nan_mask]
# return gt_vels_list.mean(0).tolist()
return {
"box3d": gt_boxes_list.mean(0).tolist(),
"detail": gt_boxes_list
# "velocity": gt_vels_list.mean(0).tolist(),
}
def _lidar_nusc_box_to_global(info, boxes, classes, eval_version="cvpr_2019"):
eval_cfg = config_factory(eval_version)
box_list = []
for box in boxes:
# Move box to ego vehicle coord system
box.rotate(Quaternion(info['lidar2ego_rotation']))
box.translate(np.array(info['lidar2ego_translation']))
# filter det in ego.
radius = np.linalg.norm(box.center[:2], 2)
det_range = eval_cfg.class_range[classes[box.label]]
if radius > det_range:
continue
# Move box to global coord system
box.rotate(Quaternion(info['ego2global_rotation']))
box.translate(np.array(info['ego2global_translation']))
box_list.append(box)
return box_list
def __init__(
self,
ann_file,
pipeline=None,
data_root=None,
classes=None,
load_interval=1,
with_velocity=True,
modality=None,
box_type_3d="LiDAR",
filter_empty_gt=True,
test_mode=False,
eval_version="detection_cvpr_2019",
use_valid_flag=False,
):
self.load_interval = load_interval
self.use_valid_flag = use_valid_flag
super().__init__(
data_root=data_root,
ann_file=ann_file,
pipeline=pipeline,
classes=classes,
modality=modality,
box_type_3d=box_type_3d,
filter_empty_gt=filter_empty_gt,
test_mode=test_mode,
)
self.with_velocity = with_velocity
self.eval_version = eval_version
from nuscenes.eval.detection.config import config_factory
self.eval_detection_configs = config_factory(self.eval_version)
if self.modality is None:
self.modality = dict(
use_camera=False,
use_lidar=True,
use_radar=False,
use_map=False,
use_external=False,
)
def _lidar_nusc_box_to_global(info, boxes, classes, eval_version="cvpr_2019"):
import pyquaternion
box_list = []
for box in boxes:
# Move box to ego vehicle coord system
box.rotate(pyquaternion.Quaternion(info['lidar2ego_rotation']))
box.translate(np.array(info['lidar2ego_translation']))
# from nuscenes.eval.detection.config import eval_detection_configs
# filter det in ego.
# cls_range_map = eval_detection_configs[eval_version]["class_range"]
from nuscenes.eval.detection.config import config_factory
cfg = config_factory(eval_version)
cls_range_map = cfg.class_range
radius = np.linalg.norm(box.center[:2], 2)
det_range = cls_range_map[classes[box.label]]
if radius > det_range:
continue
# Move box to global coord system
box.rotate(pyquaternion.Quaternion(info['ego2global_rotation']))
box.translate(np.array(info['ego2global_translation']))
box_list.append(box)
return box_list
def get_box_mean(info_path,
class_name="vehicle.car",
eval_version="cvpr_2019"):
with open(info_path, 'rb') as f:
nusc_infos = pickle.load(f)["infos"]
eval_cfg = config_factory(eval_version)
gt_boxes_list = []
gt_vels_list = []
for info in nusc_infos:
gt_boxes = info["gt_boxes"]
gt_vels = info["gt_velocity"]
gt_names = info["gt_names"]
mask = np.isin(gt_names, class_name)
gt_names = gt_names[mask]
gt_boxes = gt_boxes[mask]
gt_vels = gt_vels[mask]
det_range = np.array([eval_cfg.class_range[n] for n in gt_names])
det_range = det_range[..., np.newaxis] @ np.array([[-1, -1, 1, 1]])
mask = (gt_boxes[:, :2] >= det_range[:, :2]).all(1)
mask &= (gt_boxes[:, :2] <= det_range[:, 2:]).all(1)
gt_boxes_list.append(gt_boxes[mask].reshape(-1, 7))
gt_vels_list.append(gt_vels[mask].reshape(-1, 2))
gt_boxes_list = np.concatenate(gt_boxes_list, axis=0)
gt_vels_list = np.concatenate(gt_vels_list, axis=0)
nan_mask = np.isnan(gt_vels_list[:, 0])
# gt_vels_list = gt_vels_list[~nan_mask]
# return gt_vels_list.mean(0).tolist()
return {
"box3d": gt_boxes_list.mean(0).tolist(),
"detail": gt_boxes_list
# "velocity": gt_vels_list.mean(0).tolist(),
}
def eval_main(nusc, eval_version, res_path, eval_set, output_dir):
# nusc = NuScenes(version=version, dataroot=str(root_path), verbose=True)
cfg = config_factory(eval_version)
if nusc.version == "v1.0-trainval":
train_scenes = splits.train
# random.shuffle(train_scenes)
# train_scenes = train_scenes[:int(len(train_scenes)*0.2)]
val_scenes = splits.val
elif nusc.version == "v1.0-test":
train_scenes = splits.test
val_scenes = []
elif nusc.version == "v1.0-mini":
train_scenes = splits.mini_train
val_scenes = splits.mini_val
else:
raise ValueError("unknown")
available_scenes = _get_available_scenes(nusc)
available_scene_names = [s["name"] for s in available_scenes]
val_scenes = list(filter(lambda x: x in available_scene_names, val_scenes))
val_scenes = set([
available_scenes[available_scene_names.index(s)]["token"]
for s in val_scenes
])
nusc_eval = NuScenesEval(
nusc,
config=cfg,
result_path=res_path,
eval_set=eval_set,
output_dir=output_dir,
verbose=True,
val_scenes=val_scenes,
)
metrics_summary = nusc_eval.main(plot_examples=10, )
help='How many example visualizations to write to disk.')
parser.add_argument('--render_curves',
type=int,
default=1,
help='Whether to render PR and TP curves to disk.')
parser.add_argument('--verbose',
type=int,
default=1,
help='Whether to print to stdout.')
args = parser.parse_args()
result_path_ = os.path.expanduser(args.result_path)
output_dir_ = os.path.expanduser(args.output_dir)
eval_set_ = args.eval_set
dataroot_ = args.dataroot
version_ = args.version
config_name_ = args.config_name
plot_examples_ = args.plot_examples
render_curves_ = bool(args.render_curves)
verbose_ = bool(args.verbose)
cfg_ = config_factory(config_name_)
nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_)
nusc_eval = NuScenesEval(nusc_,
config=cfg_,
result_path=result_path_,
eval_set=eval_set_,
output_dir=output_dir_,
verbose=verbose_)
nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
def test_all_configs(self):
""" Confirm that all configurations in eval_detection_configs work in the factory """
for name in eval_detection_configs.keys():
detect_config = config_factory(name)
self.assertTrue(isinstance(detect_config, DetectionConfig))
def main():
args = parse_args()
pred_paths = glob.glob(os.path.join(args.ensemble_dir, '*.pkl'))
print(pred_paths)
preds = []
for path in pred_paths:
preds.append(get_pred(path))
merged_predictions = {}
for token in preds[0].keys():
annos = [pred[token] for pred in preds]
merged_predictions[token] = concatenate_list(annos)
predictions = merged_predictions
print("Finish Merging")
nusc_annos = {
"results": {},
"meta": None,
}
for sample_token, prediction in tqdm(predictions.items()):
annos = []
# reorganize pred by class
pred_dicts = reorganize_pred_by_class(prediction)
for name, pred in pred_dicts.items():
# in global coordinate
top_boxes, top_scores = get_sample_data(pred)
with torch.no_grad():
top_boxes_tensor = torch.from_numpy(top_boxes)
boxes_for_nms = top_boxes_tensor[:, [0, 1, 2, 4, 3, 5, -1]]
boxes_for_nms[:, -1] = boxes_for_nms[:, -1] + np.pi / 2
top_scores_tensor = torch.from_numpy(top_scores)
selected = box_torch_ops.rotate_nms(
boxes_for_nms,
top_scores_tensor,
pre_max_size=None,
post_max_size=50,
iou_threshold=0.2,
).numpy()
pred = [pred[s] for s in selected]
annos.extend(pred)
nusc_annos["results"].update({sample_token: annos})
nusc_annos["meta"] = {
"use_camera": False,
"use_lidar": True,
"use_radar": True,
"use_map": False,
"use_external": False,
}
res_dir = os.path.join(args.work_dir)
if not os.path.exists(res_dir):
os.makedirs(res_dir)
with open(os.path.join(args.work_dir, 'result.json'), "w") as f:
json.dump(nusc_annos, f)
from nuscenes.eval.detection.config import config_factory
from nuscenes.eval.detection.evaluate import NuScenesEval
nusc = NuScenes(version="v1.0-trainval",
dataroot=args.data_root,
verbose=True)
cfg = config_factory("cvpr_2019")
nusc_eval = NuScenesEval(
nusc,
config=cfg,
result_path=os.path.join(args.work_dir, 'result.json'),
eval_set='val',
output_dir=args.work_dir,
verbose=True,
)
metrics_summary = nusc_eval.main(plot_examples=0, )
parser.add_argument('--verbose',
type=int,
default=1,
help='Whether to print to stdout.')
args = parser.parse_args()
result_path_ = os.path.expanduser(args.result_path)
output_dir_ = os.path.expanduser(args.output_dir)
eval_set_ = args.eval_set
dataroot_ = args.dataroot
version_ = args.version
config_path = args.config_path
plot_examples_ = args.plot_examples
render_curves_ = bool(args.render_curves)
verbose_ = bool(args.verbose)
if config_path == '':
cfg_ = config_factory('cvpr_2019')
else:
with open(config_path, 'r') as f:
cfg_ = DetectionConfig.deserialize(json.load(f))
nusc_ = NuScenes(version=version_, verbose=verbose_, dataroot=dataroot_)
nusc_eval = NuScenesEval(nusc_,
config=cfg_,
result_path=result_path_,
eval_set=eval_set_,
output_dir=output_dir_,
verbose=verbose_)
nusc_eval.main(plot_examples=plot_examples_, render_curves=render_curves_)
def __init__(self,
ann_file,
pipeline=None,
data_root=None,
classes=None,
load_interval=1,
with_velocity=True,
modality=None,
box_type_3d='LiDAR',
filter_empty_gt=True,
test_mode=False,
eval_version='detection_cvpr_2019',
use_valid_flag=False):
# Feng Xiang code
# code begin
self.AttrMapping = {
'cycle.with_rider': 0,
'cycle.without_rider': 1,
'pedestrian.moving': 2,
'pedestrian.standing': 3,
'pedestrian.sitting_lying_down': 4,
'vehicle.moving': 5,
'vehicle.parked': 6,
'vehicle.stopped': 7,
}
self.ATTR_CLASSES = ('cycle.with_rider',
'cycle.without_rider',
'pedestrian.moving',
'pedestrian.standing',
'pedestrian.sitting_lying_down',
'vehicle.moving',
'vehicle.parked',
'vehicle.stopped',)
self.Attr_To_Class = {
'cycle.with_rider': 'bicycle',
'cycle.without_rider': 'bicycle',
'pedestrian.moving': 'pedestrian',
'pedestrian.standing': 'pedestrian',
'pedestrian.sitting_lying_down': 'pedestrian',
'vehicle.moving': 'car',
'vehicle.parked': 'car',
'vehicle.stopped': 'car',
}
# code end
self.load_interval = load_interval
self.use_valid_flag = use_valid_flag
super().__init__(
data_root=data_root,
ann_file=ann_file,
pipeline=pipeline,
classes=classes,
modality=modality,
box_type_3d=box_type_3d,
filter_empty_gt=filter_empty_gt,
test_mode=test_mode)
self.with_velocity = with_velocity
self.eval_version = eval_version
from nuscenes.eval.detection.config import config_factory
# Feng Xiang
# detection_cvpr_2019 json file is located in
# /home/ubuntu/anaconda3/env/open-mmlab/lib/python3.7/site-packages/nuscenes/eval/detection
self.eval_detection_configs = config_factory(self.eval_version)
if self.modality is None:
self.modality = dict(
use_camera=False,
use_lidar=True,
use_radar=False,
use_map=False,
use_external=False,
)
def evaluation(self, det_annos, class_names, **kwargs):
eval_det_annos = copy.deepcopy(det_annos)
# Create NuScenes JSON output file
nusc_annos = {}
for sample in eval_det_annos:
try:
sample_idx = sample['sample_idx'][0]
except:
continue
sample_results = []
calib = self.get_calib(sample_idx)
sample['boxes_lidar'] = np.array(sample['boxes_lidar'])
positions = sample['boxes_lidar'][:, :3]
dimensions = sample['boxes_lidar'][:, 3:6]
rotations = sample['boxes_lidar'][:, 6]
for center, dimension, yaw, label, score in zip(
positions, dimensions, rotations, sample['name'],
sample['score']):
quaternion = Quaternion(axis=[0, 0, 1], radians=yaw)
box = Box(center, dimension, quaternion)
# Move box to ego vehicle coord system
box.rotate(Quaternion(calib.lidar_calibrated['rotation']))
box.translate(np.array(calib.lidar_calibrated['translation']))
# Move box to global coord system
box.rotate(Quaternion(calib.ego_pose['rotation']))
box.translate(np.array(calib.ego_pose['translation']))
if (float(score) < 0):
score = 0
if (float(score) > 1):
score = 1
if (label == 'Cyclist'):
label = 'bicycle'
sample_results.append({
"sample_token":
sample_idx,
"translation":
box.center.tolist(),
"size":
box.wlh.tolist(),
"rotation":
box.orientation.elements.tolist(),
"lidar_yaw":
float(yaw),
"velocity": (0, 0),
"detection_name":
label.lower(),
"detection_score":
float(score),
"attribute_name":
self.DefaultAttribute[label.lower()],
})
nusc_annos[sample_idx] = sample_results
for sample_id in self.sample_id_list:
if sample_id not in nusc_annos:
nusc_annos[sample_id] = []
nusc_submission = {
"meta": {
"use_camera": False,
"use_lidar": True,
"use_radar": False,
"use_map": False,
"use_external": False,
},
"results": nusc_annos,
}
eval_file = os.path.join(kwargs['output_dir'], 'nusc_results.json')
with open(eval_file, "w") as f:
json.dump(nusc_submission, f, indent=2)
# Call NuScenes evaluation
cfg = config_factory('detection_cvpr_2019')
nusc_eval = DetectionEval(self.nusc,
config=cfg,
result_path=eval_file,
eval_set=self.split,
output_dir=kwargs['output_dir'],
verbose=True)
metric_summary = nusc_eval.main(plot_examples=10, render_curves=True)
# Reformat the metrics summary a bit for the tensorboard logger
err_name_mapping = {
'trans_err': 'mATE',
'scale_err': 'mASE',
'orient_err': 'mAOE',
'vel_err': 'mAVE',
'attr_err': 'mAAE'
}
result = {}
result['mean_ap'] = metric_summary['mean_ap']
for tp_name, tp_val in metric_summary['tp_errors'].items():
result[tp_name] = tp_val
class_aps = metric_summary['mean_dist_aps']
class_tps = metric_summary['label_tp_errors']
for class_name in class_aps.keys():
result['mAP_' + class_name] = class_aps[class_name]
for key, val in err_name_mapping.items():
result[val + '_' + class_name] = class_tps[class_name][key]
return str(result), result
class TestAlgo(unittest.TestCase):
this_dir = os.path.dirname(os.path.abspath(__file__))
cfg = config_factory('cvpr_2019')
@staticmethod
def _mock_results(nsamples, ngt, npred, detection_name):
def random_attr():
"""
This is the most straight-forward way to generate a random attribute.
Not currently used b/c we want the test fixture to be back-wards compatible.
"""
# Get relevant attributes.
rel_attributes = detection_name_to_rel_attributes(detection_name)
if len(rel_attributes) == 0:
# Empty string for classes without attributes.
return ''
else:
# Pick a random attribute otherwise.
return rel_attributes[np.random.randint(0, len(rel_attributes))]
pred = EvalBoxes()
gt = EvalBoxes()
for sample_itt in range(nsamples):
this_gt = []
for box_itt in range(ngt):
this_gt.append(EvalBox(
sample_token=str(sample_itt),
translation=tuple(list(np.random.rand(2)*15) + [0.0]),
size=tuple(np.random.rand(3)*4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2]*4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_dist=random.random()*10,
))
gt.add_boxes(str(sample_itt), this_gt)
for sample_itt in range(nsamples):
this_pred = []
for box_itt in range(npred):
this_pred.append(EvalBox(
sample_token=str(sample_itt),
translation=tuple(list(np.random.rand(2) * 10) + [0.0]),
size=tuple(np.random.rand(3) * 4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2] * 4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_dist=random.random() * 10,
))
pred.add_boxes(str(sample_itt), this_pred)
return gt, pred
def test_weighted_sum(self):
"""
This tests runs the full evaluation for an arbitrary random set of predictions.
"""
random.seed(42)
np.random.seed(42)
mdl = MetricDataList()
for class_name in self.cfg.class_names:
gt, pred = self._mock_results(30, 3, 25, class_name)
for dist_th in self.cfg.dist_ths:
mdl.set(class_name, dist_th, accumulate(gt, pred, class_name, 'center_distance', 2))
metrics = DetectionMetrics(self.cfg)
for class_name in self.cfg.class_names:
for dist_th in self.cfg.dist_ths:
ap = calc_ap(mdl[(class_name, dist_th)], self.cfg.min_recall, self.cfg.min_precision)
metrics.add_label_ap(class_name, dist_th, ap)
for metric_name in TP_METRICS:
metric_data = mdl[(class_name, self.cfg.dist_th_tp)]
if class_name in ['traffic_cone'] and metric_name in ['attr_err', 'vel_err', 'orient_err']:
tp = np.nan
elif class_name in ['barrier'] and metric_name in ['attr_err', 'vel_err']:
tp = np.nan
else:
tp = calc_tp(metric_data, self.cfg.min_recall, metric_name)
metrics.add_label_tp(class_name, metric_name, tp)
self.assertEqual(0.08606662159639042, metrics.weighted_sum)
def test_calc_tp(self):
"""Test for calc_tp()."""
random.seed(42)
np.random.seed(42)
md = MetricData.random_md()
# min_recall greater than 1.
self.assertEqual(1.0, calc_tp(md, min_recall=1, metric_name='trans_err'))
def test_calc_ap(self):
"""Test for calc_ap()."""
random.seed(42)
np.random.seed(42)
md = MetricData.random_md()
# Negative min_recall and min_precision
self.assertRaises(AssertionError, calc_ap, md, -0.5, 0.4)
self.assertRaises(AssertionError, calc_ap, md, 0.5, -0.8)
# More than 1 min_precision/min_recall
self.assertRaises(AssertionError, calc_ap, md, 0.7, 1)
self.assertRaises(AssertionError, calc_ap, md, 1.2, 0)
