def _mock_results(nsamples, ngt, npred, detection_name):
def random_attr():
"""
This is the most straight-forward way to generate a random attribute.
Not currently used b/c we want the test fixture to be back-wards compatible.
"""
# Get relevant attributes.
rel_attributes = detection_name_to_rel_attributes(detection_name)
if len(rel_attributes) == 0:
# Empty string for classes without attributes.
return ''
else:
# Pick a random attribute otherwise.
return rel_attributes[np.random.randint(0, len(rel_attributes))]
pred = EvalBoxes()
gt = EvalBoxes()
for sample_itt in range(nsamples):
this_gt = []
for box_itt in range(ngt):
translation_xy = tuple(np.random.rand(2) * 15)
this_gt.append(DetectionBox(
sample_token=str(sample_itt),
translation=(translation_xy[0], translation_xy[1], 0.0),
size=tuple(np.random.rand(3)*4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2]*4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_dist=random.random()*10,
))
gt.add_boxes(str(sample_itt), this_gt)
for sample_itt in range(nsamples):
this_pred = []
for box_itt in range(npred):
translation_xy = tuple(np.random.rand(2) * 10)
this_pred.append(DetectionBox(
sample_token=str(sample_itt),
translation=(translation_xy[0], translation_xy[1], 0.0),
size=tuple(np.random.rand(3) * 4),
rotation=tuple(np.random.rand(4)),
velocity=tuple(np.random.rand(3)[:2] * 4),
detection_name=detection_name,
detection_score=random.random(),
attribute_name=random_attr(),
ego_dist=random.random() * 10,
))
pred.add_boxes(str(sample_itt), this_pred)
return gt, pred
def load_prediction(result_path: str, max_boxes_per_sample: int, box_cls, verbose: bool = False) \
-> Tuple[EvalBoxes, Dict]:
"""
Loads object predictions from file.
:param result_path: Path to the .json result file provided by the user.
:param max_boxes_per_sample: Maximim number of boxes allowed per sample.
:param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox.
:param verbose: Whether to print messages to stdout.
:return: The deserialized results and meta data.
"""
# Load from file and check that the format is correct.
with open(result_path) as f:
data = json.load(f)
assert 'results' in data, 'Error: No field `results` in result file. Please note that the result format changed.' \
'See https://www.nuscenes.org/object-detection for more information.'
# Deserialize results and get meta data.
all_results = EvalBoxes.deserialize(data['results'], box_cls)
meta = data['meta']
print('!' * 500)
print(all_results.sample_tokens)
if verbose:
print("Loaded results from {}. Found detections for {} samples.".
format(result_path, len(all_results.sample_tokens)) + '%' * 300)
# Check that each sample has no more than x predicted boxes.
for sample_token in all_results.sample_tokens:
assert len(all_results.boxes[sample_token]) <= max_boxes_per_sample, \
"Error: Only <= %d boxes per sample allowed!" % max_boxes_per_sample
return all_results, meta
def get_metric_data(gts: Dict[str, List[Dict]],
preds: Dict[str, List[Dict]],
detection_name: str,
dist_th: float) -> DetectionMetricData:
"""
Calculate and check the AP value.
:param gts: Ground truth data.
:param preds: Predictions.
:param detection_name: Name of the class we are interested in.
:param dist_th: Distance threshold for matching.
"""
# Some or all of the defaults will be replaced by if given.
defaults = {'trans': (0, 0, 0), 'size': (1, 1, 1), 'rot': (0, 0, 0, 0),
'vel': (0, 0), 'attr': 'vehicle.parked', 'score': -1.0, 'name': 'car'}
# Create GT EvalBoxes instance.
gt_eval_boxes = EvalBoxes()
for sample_token, data in gts.items():
gt_boxes = []
for gt in data:
gt = {**defaults, **gt} # The defaults will be replaced by gt if given.
eb = DetectionBox(sample_token=sample_token, translation=gt['trans'], size=gt['size'],
rotation=gt['rot'], detection_name=gt['name'], attribute_name=gt['attr'],
velocity=gt['vel'])
gt_boxes.append(eb)
gt_eval_boxes.add_boxes(sample_token, gt_boxes)
# Create Predictions EvalBoxes instance.
pred_eval_boxes = EvalBoxes()
for sample_token, data in preds.items():
pred_boxes = []
for pred in data:
pred = {**defaults, **pred}
eb = DetectionBox(sample_token=sample_token, translation=pred['trans'], size=pred['size'],
rotation=pred['rot'], detection_name=pred['name'], detection_score=pred['score'],
velocity=pred['vel'], attribute_name=pred['attr'])
pred_boxes.append(eb)
pred_eval_boxes.add_boxes(sample_token, pred_boxes)
metric_data = accumulate(gt_eval_boxes, pred_eval_boxes, class_name=detection_name,
dist_fcn=center_distance, dist_th=dist_th)
return metric_data
def test_serialization(self):
""" Test that instance serialization protocol works with json encoding. """
boxes = EvalBoxes()
for i in range(10):
boxes.add_boxes(str(i),
[DetectionBox(),
DetectionBox(),
DetectionBox()])
recovered = EvalBoxes.deserialize(
json.loads(json.dumps(boxes.serialize())), DetectionBox)
self.assertEqual(boxes, recovered)
def load_groundtruth(groundtruth_path: str, box_cls = DetectionBox, verbose: bool = False) \
-> EvalBoxes:
"""
Loads object GT boxes from file.
:param groundtruth_path: Path to the .json GT file provided by the user.
:param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox.
:param verbose: Whether to print messages to stdout.
:return: The deserialized results and meta data.
"""
# Load from file and check that the format is correct.
with open(groundtruth_path) as f:
data = json.load(f)
assert 'GT_boxes' in data, 'Error: No field `GT_boxes` in ground truth file.'
# Deserialize results and get meta data.
all_results = EvalBoxes.deserialize(data['GT_boxes'], box_cls)
if verbose:
print("Loaded GT from {}. Found detections for {} samples.".format(
groundtruth_path, len(all_results.sample_tokens)))
return all_results
def test_get_box_class_field(self):
eval_boxes = EvalBoxes()
box1 = DetectionBox(sample_token='box1',
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(25.0, 0.0, 0.0))
box2 = DetectionBox(sample_token='box2',
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle',
ego_translation=(45.0, 0.0, 0.0))
eval_boxes.add_boxes('sample1', [])
eval_boxes.add_boxes('sample2', [box1, box2])
class_field = _get_box_class_field(eval_boxes)
self.assertEqual(class_field, 'detection_name')
def main(self,
plot_examples: int = 0,
render_curves: bool = True) -> Dict[str, Any]:
"""
Main function that loads the evaluation code, visualizes samples, runs the evaluation and renders stat plots.
:param plot_examples: How many example visualizations to write to disk.
:param render_curves: Whether to render PR and TP curves to disk.
:return: A dict that stores the high-level metrics and meta data.
"""
if plot_examples > 0:
# Select a random but fixed subset to plot.
random.seed(42)
sample_tokens = list(self.sample_tokens)
random.shuffle(sample_tokens)
sample_tokens = sample_tokens[:plot_examples]
# Visualize samples.
example_dir = os.path.join(self.output_dir, 'examples')
if not os.path.isdir(example_dir):
os.mkdir(example_dir)
for sample_token in sample_tokens:
visualize_sample(self.nusc,
sample_token,
self.gt_boxes if self.eval_set != 'test' else EvalBoxes(),
# Don't render test GT.
self.pred_boxes,
eval_range=max(self.cfg.class_range.values()),
savepath=os.path.join(example_dir, '{}.png'.format(sample_token)))
# Run evaluation.
metrics, metric_data_list = self.evaluate()
# Render PR and TP curves.
if render_curves:
self.render(metrics, metric_data_list)
# Dump the metric data, meta and metrics to disk.
if self.verbose:
print('Saving metrics to: %s' % self.output_dir)
metrics_summary = metrics.serialize()
metrics_summary['meta'] = self.meta.copy()
with open(os.path.join(self.output_dir, 'metrics_summary.json'), 'w') as f:
json.dump(metrics_summary, f, indent=2)
with open(os.path.join(self.output_dir, 'metrics_details.json'), 'w') as f:
json.dump(metric_data_list.serialize(), f, indent=2)
# Print high-level metrics.
print('mAP: %.4f' % (metrics_summary['mean_ap']))
err_name_mapping = {
'trans_err': 'mATE',
'scale_err': 'mASE',
'orient_err': 'mAOE',
'vel_err': 'mAVE',
'attr_err': 'mAAE'
}
for tp_name, tp_val in metrics_summary['tp_errors'].items():
print('%s: %.4f' % (err_name_mapping[tp_name], tp_val))
print('NDS: %.4f' % (metrics_summary['nd_score']))
print('Eval time: %.1fs' % metrics_summary['eval_time'])
# Print per-class metrics.
print()
print('Per-class results:')
print('Object Class\tAP\tATE\tASE\tAOE\tAVE\tAAE')
class_aps = metrics_summary['mean_dist_aps']
class_tps = metrics_summary['label_tp_errors']
for class_name in class_aps.keys():
print('%s\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f\t%.3f'
% (class_name, class_aps[class_name],
class_tps[class_name]['trans_err'],
class_tps[class_name]['scale_err'],
class_tps[class_name]['orient_err'],
class_tps[class_name]['vel_err'],
class_tps[class_name]['attr_err']))
return metrics_summary
def load_gt(nusc: NuScenes,
eval_split: str,
box_cls,
verbose: bool = False) -> EvalBoxes:
"""
Loads ground truth boxes from DB.
:param nusc: A NuScenes instance.
:param eval_split: The evaluation split for which we load GT boxes.
:param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox.
:param verbose: Whether to print messages to stdout.
:return: The GT boxes.
"""
# Init.
if box_cls == DetectionBox:
attribute_map = {a['token']: a['name'] for a in nusc.attribute}
if verbose:
print('Loading annotations for {} split from nuScenes version: {}'.
format(eval_split, nusc.version))
# Read out all sample_tokens in DB.
sample_tokens_all = [s['token'] for s in nusc.sample]
assert len(sample_tokens_all) > 0, "Error: Database has no samples!"
# Only keep samples from this split.
splits = create_splits_scenes()
# Check compatibility of split with nusc_version.
version = nusc.version
if eval_split in {'train', 'val', 'train_detect', 'train_track'}:
assert version.endswith('trainval'), \
'Error: Requested split {} which is not compatible with NuScenes version {}'.format(eval_split, version)
elif eval_split in {'mini_train', 'mini_val'}:
assert version.endswith('mini'), \
'Error: Requested split {} which is not compatible with NuScenes version {}'.format(eval_split, version)
elif eval_split == 'test':
assert version.endswith('test'), \
'Error: Requested split {} which is not compatible with NuScenes version {}'.format(eval_split, version)
else:
raise ValueError(
'Error: Requested split {} which this function cannot map to the correct NuScenes version.'
.format(eval_split))
if eval_split == 'test':
# Check that you aren't trying to cheat :).
assert len(nusc.sample_annotation) > 0, \
'Error: You are trying to evaluate on the test set but you do not have the annotations!'
sample_tokens = []
for sample_token in sample_tokens_all:
scene_token = nusc.get('sample', sample_token)['scene_token']
scene_record = nusc.get('scene', scene_token)
if scene_record['name'] in splits[eval_split]:
sample_tokens.append(sample_token)
all_annotations = EvalBoxes()
# Load annotations and filter predictions and annotations.
tracking_id_set = set()
for sample_token in tqdm.tqdm(sample_tokens, leave=verbose):
sample = nusc.get('sample', sample_token)
sample_annotation_tokens = sample['anns']
sample_boxes = []
for sample_annotation_token in sample_annotation_tokens:
sample_annotation = nusc.get('sample_annotation',
sample_annotation_token)
if box_cls == DetectionBox:
# Get label name in detection task and filter unused labels.
detection_name = category_to_detection_name(
sample_annotation['category_name'])
if detection_name is None:
continue
# Get attribute_name.
attr_tokens = sample_annotation['attribute_tokens']
attr_count = len(attr_tokens)
if attr_count == 0:
attribute_name = ''
elif attr_count == 1:
attribute_name = attribute_map[attr_tokens[0]]
else:
raise Exception(
'Error: GT annotations must not have more than one attribute!'
)
sample_boxes.append(
box_cls(
sample_token=sample_token,
translation=sample_annotation['translation'],
size=sample_annotation['size'],
rotation=sample_annotation['rotation'],
velocity=nusc.box_velocity(
sample_annotation['token'])[:2],
num_pts=sample_annotation['num_lidar_pts'] +
sample_annotation['num_radar_pts'],
detection_name=detection_name,
detection_score=-1.0, # GT samples do not have a score.
attribute_name=attribute_name))
elif box_cls == TrackingBox:
# Use nuScenes token as tracking id.
tracking_id = sample_annotation['instance_token']
tracking_id_set.add(tracking_id)
# Get label name in detection task and filter unused labels.
tracking_name = category_to_tracking_name(
sample_annotation['category_name'])
if tracking_name is None:
continue
sample_boxes.append(
box_cls(
sample_token=sample_token,
translation=sample_annotation['translation'],
size=sample_annotation['size'],
rotation=sample_annotation['rotation'],
velocity=nusc.box_velocity(
sample_annotation['token'])[:2],
num_pts=sample_annotation['num_lidar_pts'] +
sample_annotation['num_radar_pts'],
tracking_id=tracking_id,
tracking_name=tracking_name,
tracking_score=-1.0 # GT samples do not have a score.
))
else:
raise NotImplementedError('Error: Invalid box_cls %s!' %
box_cls)
all_annotations.add_boxes(sample_token, sample_boxes)
if verbose:
print("Loaded ground truth annotations for {} samples.".format(
len(all_annotations.sample_tokens)))
return all_annotations
def filter_eval_boxes(nusc: NuScenes,
eval_boxes: EvalBoxes,
max_dist: Dict[str, float],
verbose: bool = False) -> EvalBoxes:
"""
Applies filtering to boxes. Distance, bike-racks and points per box.
:param nusc: An instance of the NuScenes class.
:param eval_boxes: An instance of the EvalBoxes class.
:param max_dist: Maps the detection name to the eval distance threshold for that class.
:param verbose: Whether to print to stdout.
"""
# Retrieve box type for detectipn/tracking boxes.
class_field = _get_box_class_field(eval_boxes)
# Accumulators for number of filtered boxes.
total, dist_filter, point_filter, bike_rack_filter = 0, 0, 0, 0
for ind, sample_token in enumerate(eval_boxes.sample_tokens):
print('a')
print(sample_token)
# Filter on distance first.
total += len(eval_boxes[sample_token])
eval_boxes.boxes[sample_token] = [
box for box in eval_boxes[sample_token]
if box.ego_dist < max_dist[box.__getattribute__(class_field)]
]
dist_filter += len(eval_boxes[sample_token])
# Then remove boxes with zero points in them. Eval boxes have -1 points by default.
eval_boxes.boxes[sample_token] = [
box for box in eval_boxes[sample_token] if not box.num_pts == 0
]
point_filter += len(eval_boxes[sample_token])
# Perform bike-rack filtering.
sample_anns = nusc.get('sample', sample_token)['anns']
bikerack_recs = [
nusc.get('sample_annotation', ann) for ann in sample_anns
if nusc.get('sample_annotation', ann)['category_name'] ==
'static_object.bicycle_rack'
]
bikerack_boxes = [
Box(rec['translation'], rec['size'], Quaternion(rec['rotation']))
for rec in bikerack_recs
]
filtered_boxes = []
for box in eval_boxes[sample_token]:
if box.__getattribute__(class_field) in ['bicycle', 'motorcycle']:
in_a_bikerack = False
for bikerack_box in bikerack_boxes:
if np.sum(
points_in_box(
bikerack_box,
np.expand_dims(np.array(box.translation),
axis=1))) > 0:
in_a_bikerack = True
if not in_a_bikerack:
filtered_boxes.append(box)
else:
filtered_boxes.append(box)
eval_boxes.boxes[sample_token] = filtered_boxes
bike_rack_filter += len(eval_boxes.boxes[sample_token])
if verbose:
print("=> Original number of boxes: %d" % total)
print("=> After distance based filtering: %d" % dist_filter)
print("=> After LIDAR points based filtering: %d" % point_filter)
print("=> After bike rack filtering: %d" % bike_rack_filter)
return eval_boxes
def main():
# test pmbm tracking in val split of NuScenes
detection_file = '/home/mqdao/Downloads/nuScene/detection-megvii/megvii_val.json'
data_root = '/home/mqdao/Downloads/nuScene/v1.0-trainval'
version = 'v1.0-trainval'
nusc = NuScenes(version=version, dataroot=data_root, verbose=True)
# load detection
with open(detection_file) as f:
data = json.load(f)
all_results = EvalBoxes.deserialize(data['results'], DetectionBox)
meta = data['meta']
print('meta: ', meta)
print("Loaded results from {}. Found detections for {} samples.".format(
detection_file, len(all_results.sample_tokens)))
# to filter detection
all_score_theshold = [0.35, 0.3, 0.25, 0.2]
# init tracking results
tracking_results = {}
processed_scene_tokens = set()
for sample_token_idx in tqdm(range(len(all_results.sample_tokens))):
sample_token = all_results.sample_tokens[sample_token_idx]
scene_token = nusc.get('sample', sample_token)['scene_token']
if scene_token in processed_scene_tokens:
continue
# initialize filter
config = FilterConfig(state_dim=6, measurement_dim=3)
density = get_gaussian_density_NuScenes_CV()
pmbm_filter = PoissonMultiBernoulliMixture(config, density)
current_sample_token = nusc.get('scene',
scene_token)['first_sample_token']
current_time_step = 0
while current_sample_token != '':
# filter detections with low detection score
sample_record = nusc.get('sample', current_sample_token)
gt_num_objects = len(sample_record['anns'])
filtered_detections = []
i_threshold = 0
while len(filtered_detections
) < gt_num_objects and i_threshold < len(
all_score_theshold):
filtered_detections = [
detection
for detection in all_results.boxes[current_sample_token]
if detection.detection_score >=
all_score_theshold[i_threshold]
]
i_threshold += 1
# create measurement for pmbm filter
measurements = []
for detection in filtered_detections:
# get obj_type
if detection.detection_name not in NUSCENES_TRACKING_NAMES:
continue
obj_type = detection.detection_name
# get object pose
x, y, z = detection.translation
quaternion = Quaternion(detection.rotation)
yaw = quaternion.angle if quaternion.axis[
2] > 0 else -quaternion.angle
# get object size
size = list(detection.size)
# get detection score
score = detection.detection_score
# create object detection
obj_det = ObjectDetection(z=np.array([x, y,
yaw]).reshape(3, 1),
size=size,
obj_type=obj_type,
height=z,
score=score,
empty_constructor=False)
measurements.append(obj_det)
# print('Time {} - Number of measurements: {}'.format(current_time_step, len(measurements)))
# initialize tracking results for this sample
tracking_results[current_sample_token] = []
# invoke filter and extract estimation
if len(measurements) > 0:
estimation = pmbm_filter.run(measurements)
# log estimation
for target_id, target_est in estimation.items():
sample_result = format_result(
current_sample_token,
target_est['translation'] + [target_est['height']],
target_est['size'], target_est['orientation'],
target_est['velocity'], target_id, target_est['class'],
target_est['score'])
tracking_results[current_sample_token].append(
sample_result)
# move on
current_sample_token = sample_record['next']
current_time_step += 1
processed_scene_tokens.add(scene_token)
# save tracking result
output_data = {'meta': meta, 'results': tracking_results}
with open(
'./estimation-result/all-results-validataion-set-{}.json'.format(
datetime.now().strftime("%Y%m%d-%H%M%S")), 'w') as outfile:
json.dump(output_data, outfile)
def load_merge_from_pkl(nusc: NuScenes,
pkl_path: str,
box_cls,
verbose: bool = False) -> EvalBoxes:
# Init.
if box_cls == DetectionBox:
attribute_map = {a['token']: a['name'] for a in nusc.attribute}
if verbose:
print('Loading annotations for {} split from nuScenes version: {}'.
format(pkl_path, nusc.version))
import mmcv
infos = mmcv.load(pkl_path)['infos']
samples = []
for info in infos:
samples.append(nusc.get('sample', info['token']))
all_annotations = EvalBoxes()
# Load annotations and filter predictions and annotations.
merge_map = dict(car='vehicle',
truck='vehicle',
bus='vehicle',
trailer='vehicle',
construction_vehicle='vehicle',
pedestrian='pedestrian',
motorcycle='bike',
bicycle='bike',
traffic_cone='traffic_boundary',
barrier='traffic_boundary')
for sample in tqdm.tqdm(samples, leave=verbose):
sample_token = sample['token']
cam_token = sample['data']['CAM_FRONT']
_, boxes_cam, _ = nusc.get_sample_data(cam_token)
sample_annotation_tokens = [box.token for box in boxes_cam]
# sample = nusc.get('sample', sample_token)
# sample_annotation_tokens = sample['anns']
sample_boxes = []
for sample_annotation_token in sample_annotation_tokens:
sample_annotation = nusc.get('sample_annotation',
sample_annotation_token)
if box_cls == DetectionBox:
# Get label name in detection task and filter unused labels.
detection_name = category_to_detection_name(
sample_annotation['category_name'])
if detection_name is None:
continue
detection_name = merge_map[detection_name]
# Get attribute_name.
attr_tokens = sample_annotation['attribute_tokens']
attr_count = len(attr_tokens)
if attr_count == 0:
attribute_name = ''
elif attr_count == 1:
attribute_name = attribute_map[attr_tokens[0]]
else:
raise Exception(
'Error: GT annotations must not have more than one attribute!'
)
sample_boxes.append(
box_cls(
sample_token=sample_token,
translation=sample_annotation['translation'],
size=sample_annotation['size'],
rotation=sample_annotation['rotation'],
velocity=nusc.box_velocity(
sample_annotation['token'])[:2],
num_pts=sample_annotation['num_lidar_pts'] +
sample_annotation['num_radar_pts'],
detection_name=detection_name,
detection_score=-1.0, # GT samples do not have a score.
attribute_name=attribute_name))
else:
raise NotImplementedError('Error: Invalid box_cls %s!' %
box_cls)
all_annotations.add_boxes(sample_token, sample_boxes)
if verbose:
print("Loaded ground truth annotations for {} samples.".format(
len(all_annotations.sample_tokens)))
return all_annotations
def track_nuscenes(data_root: str,
detection_file: str,
save_dir: str,
eval_set: str = 'val',
covariance_id: int = 0,
match_distance: str = 'iou',
match_threshold: float = 0.1,
match_algorithm: str = 'h',
use_angular_velocity: bool = False):
'''
submission {
"meta": {
"use_camera": <bool> -- Whether this submission uses camera data as an input.
"use_lidar": <bool> -- Whether this submission uses lidar data as an input.
"use_radar": <bool> -- Whether this submission uses radar data as an input.
"use_map": <bool> -- Whether this submission uses map data as an input.
"use_external": <bool> -- Whether this submission uses external data as an input.
},
"results": {
sample_token <str>: List[sample_result] -- Maps each sample_token to a list of sample_results.
}
}
'''
if 'train' in eval_set:
version = 'v1.0-trainval'
elif 'val' in eval_set:
version = 'v1.0-trainval'
elif 'mini' in eval_set:
version = 'v1.0-mini'
elif 'test' in eval_set:
version = 'v1.0-test'
else:
version = eval_set
print("WARNING: Unknown subset version: '{}'".format(version))
nusc = NuScenes(version=version, dataroot=data_root, verbose=True)
mkdir_if_missing(save_dir)
output_path = os.path.join(save_dir, 'probabilistic_tracking_results.json')
results = {}
total_time = 0.0
total_frames = 0
with open(detection_file) as f:
data = json.load(f)
assert 'results' in data, 'Error: No field `results` in result file. Please note that the result format changed.' \
'See https://www.nuscenes.org/object-detection for more information.'
all_results = EvalBoxes.deserialize(data['results'], DetectionBox)
meta = data['meta']
print('meta: ', meta)
print("Loaded results from {}. Found detections for {} samples.".format(
detection_file, len(all_results.sample_tokens)))
processed_scene_tokens = set()
for sample_token_idx in tqdm(range(len(all_results.sample_tokens))):
sample_token = all_results.sample_tokens[sample_token_idx]
scene_token = nusc.get('sample', sample_token)['scene_token']
if scene_token in processed_scene_tokens:
continue
first_sample_token = nusc.get('scene',
scene_token)['first_sample_token']
current_sample_token = first_sample_token
mot_trackers = {
tracking_name: AB3DMOT(covariance_id,
tracking_name=tracking_name,
use_angular_velocity=use_angular_velocity,
tracking_nuscenes=True)
for tracking_name in NUSCENES_TRACKING_NAMES
}
while current_sample_token != '':
results[current_sample_token] = []
dets = {
tracking_name: []
for tracking_name in NUSCENES_TRACKING_NAMES
}
info = {
tracking_name: []
for tracking_name in NUSCENES_TRACKING_NAMES
}
for box in all_results.boxes[current_sample_token]:
if box.detection_name not in NUSCENES_TRACKING_NAMES:
continue
q = Quaternion(box.rotation)
angle = q.angle if q.axis[2] > 0 else -q.angle
#print('box.rotation, angle, axis: ', box.rotation, q.angle, q.axis)
#print('box.rotation, angle, axis: ', q.angle, q.axis)
#[h, w, l, x, y, z, rot_y]
detection = np.array([
box.size[2], box.size[0], box.size[1], box.translation[0],
box.translation[1], box.translation[2], angle
])
#print('detection: ', detection)
information = np.array([box.detection_score])
dets[box.detection_name].append(detection)
info[box.detection_name].append(information)
dets_all = {
tracking_name: {
'dets': np.array(dets[tracking_name]),
'info': np.array(info[tracking_name])
}
for tracking_name in NUSCENES_TRACKING_NAMES
}
total_frames += 1
start_time = time.time()
for tracking_name in NUSCENES_TRACKING_NAMES:
if dets_all[tracking_name]['dets'].shape[0] > 0:
trackers = mot_trackers[tracking_name].update(
dets_all[tracking_name], match_distance,
match_threshold, match_algorithm, scene_token)
# (N, 9)
# (h, w, l, x, y, z, rot_y), tracking_id, tracking_score
# print('trackers: ', trackers)
for i in range(trackers.shape[0]):
sample_result = format_sample_result(
current_sample_token, tracking_name, trackers[i])
results[current_sample_token].append(sample_result)
cycle_time = time.time() - start_time
total_time += cycle_time
# get next frame and continue the while loop
current_sample_token = nusc.get('sample',
current_sample_token)['next']
# left while loop and mark this scene as processed
processed_scene_tokens.add(scene_token)
# finished tracking all scenes, write output data
output_data = {'meta': meta, 'results': results}
with open(output_path, 'w') as outfile:
json.dump(output_data, outfile)
print("Total Tracking took: %.3f for %d frames or %.1f FPS" %
(total_time, total_frames, total_frames / total_time))
def load_gt(root_path: str,
info_path: str,
eval_split: str,
box_cls,
verbose: bool = False) -> EvalBoxes:
"""
Loads ground truth boxes from DB.
:param nusc: A NuScenes instance.
:param eval_split: The evaluation split for which we load GT boxes.
:param box_cls: Type of box to load, e.g. DetectionBox or TrackingBox.
:param verbose: Whether to print messages to stdout.
:return: The GT boxes.
"""
with open(info_path, 'rb') as f:
data = pickle.load(f)
udi_infos = data["infos"]
metadata = data["metadata"]
sample_tokens_all = [info['token'] for info in udi_infos]
assert len(sample_tokens_all) > 0, "Error: Database has no samples!"
sample_tokens = sample_tokens_all
## UDI Dataset samples 11428 samples
all_annotations = EvalBoxes()
label_root_path = root_path + "/label"
filenames = os.listdir(label_root_path)
for filename in tqdm.tqdm(filenames, leave=verbose):
sample_boxes = []
index = filename.split("_")[0]
label_path = label_root_path + "/" + filename
assert Path(label_path).exists()
with open(label_path, encoding='utf-8') as f:
res = f.read()
result = json.loads(res)
boxes = result["elem"]
for box in boxes:
class_name = box["class"]
box_loc = box["position"]
box_size = box["size"]
yaw = box["yaw"]
quat = pyquaternion.Quaternion(axis=[0, 0, 1], radians=yaw)
sample_boxes.append(
box_cls(
sample_token=int(index),
translation=tuple(
[box_loc["x"], box_loc["y"], box_loc["z"]]),
size=tuple([
box_size["width"], box_size["depth"],
box_size["height"]
]),
rotation=tuple(quat),
velocity=tuple([0, 0]),
num_pts=-1,
detection_name=class_name,
detection_score=-1.0, # GT samples do not have a score.
attribute_name=''))
all_annotations.add_boxes(int(index), sample_boxes)
if verbose:
print("Loaded ground truth annotations for {} samples.".format(
len(all_annotations.sample_tokens)))
return all_annotations
def test_filter_eval_boxes(self):
"""
This tests runs the evaluation for an arbitrary random set of predictions.
This score is then captured in this very test such that if we change the eval code,
this test will trigger if the results changed.
"""
# Get the maximum distance from the config
cfg = config_factory('detection_cvpr_2019')
max_dist = cfg.class_range
assert 'NUSCENES' in os.environ, 'Set NUSCENES env. variable to enable tests.'
nusc = NuScenes(version='v1.0-mini',
dataroot=os.environ['NUSCENES'],
verbose=False)
sample_token = '0af0feb5b1394b928dd13d648de898f5'
# This sample has a bike rack instance 'bfe685042aa34ab7b2b2f24ee0f1645f' with these parameters
# 'translation': [683.681, 1592.002, 0.809],
# 'size': [1.641, 14.465, 1.4],
# 'rotation': [0.3473693995546558, 0.0, 0.0, 0.9377283723195315]
# Test bicycle filtering by creating a box at the same position as the bike rack.
box1 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # box1 should be filtered.
# Test motorcycle filtering by creating a box at the same position as the bike rack.
box2 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
0) # both box1 and box2 should be filtered.
# Now create a car at the same position as the bike rack.
box3 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
1) # box1 and box2 to be filtered. box3 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
# Now add a bike outside the bike rack.
box4 = DetectionBox(sample_token=sample_token,
translation=(68.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle')
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2 to be filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center,
# but set the ego_dist (derived from ego_translation) higher than what's defined in max_dist
box5 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(100.0, 0.0, 0.0))
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token, [box1, box2, box3, box4, box5])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5 filtered. box3, box4 to stay.
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Add another bike on the bike rack center but set the num_pts to be zero so that it gets filtered.
box6 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
2) # box1, box2, box5, box6 filtered. box3, box4 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].detection_name,
'car')
self.assertEqual(filtered_boxes.boxes[sample_token][1].detection_name,
'bicycle')
self.assertEqual(filtered_boxes.boxes[sample_token][1].translation[0],
68.681)
# Check for a sample where there are no bike racks. Everything should be filtered correctly.
sample_token = 'ca9a282c9e77460f8360f564131a8af5' # This sample has no bike-racks.
box1 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
ego_translation=(25.0, 0.0, 0.0))
box2 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='motorcycle',
ego_translation=(45.0, 0.0, 0.0))
box3 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_translation=(45.0, 0.0, 0.0))
box4 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='car',
ego_translation=(55.0, 0.0, 0.0))
box5 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=1)
box6 = DetectionBox(sample_token=sample_token,
translation=(683.681, 1592.002, 0.809),
size=(1, 1, 1),
detection_name='bicycle',
num_pts=0)
eval_boxes = EvalBoxes()
eval_boxes.add_boxes(sample_token,
[box1, box2, box3, box4, box5, box6])
filtered_boxes = filter_eval_boxes(nusc, eval_boxes, max_dist)
self.assertEqual(len(filtered_boxes.boxes[sample_token]),
3) # box2, box4, box6 filtered. box1, box3, box5 stay
self.assertEqual(filtered_boxes.boxes[sample_token][0].ego_dist, 25.0)
self.assertEqual(filtered_boxes.boxes[sample_token][1].ego_dist, 45.0)
self.assertEqual(filtered_boxes.boxes[sample_token][2].num_pts, 1)