def testPerfectBox(self):
metadata = waymo_metadata.WaymoMetadata()
params = waymo_ap_metric.WaymoAPMetrics.Params(metadata)
m = params.Instantiate()
# Make one update with a perfect box.
update_dict = py_utils.NestedMap(
groundtruth_labels=np.array([1]),
groundtruth_bboxes=np.ones(shape=(1, 7)),
groundtruth_difficulties=np.zeros(shape=(1)),
groundtruth_num_points=None,
detection_scores=np.ones(shape=(5, 1)),
detection_boxes=np.ones(shape=(5, 1, 7)),
detection_heights_in_pixels=np.ones(shape=(5, 1)))
m.Update('1234', update_dict)
waymo_ap = m.value
self.assertAllClose(waymo_ap, 1. / 3.)
# Write a summary.
summary = m.Summary('foo')
# Check that both AP and APH are in the tags.
tags = [v.tag for v in summary.value]
self.assertIn('foo/Pedestrian/AP_default', tags)
self.assertIn('foo/Pedestrian/APH_default', tags)
def _FilterKeepLabels(params, label_names):
"""Keep only label names in 'label_names' from input."""
metadata = waymo_metadata.WaymoMetadata()
filtered_labels = [
metadata.ClassNames().index(label_name) for label_name in label_names
]
params.extractors.labels.filter_labels = filtered_labels
def Params(cls):
p = super(WaymoOpenDatasetDecoder, cls).Params()
p.Define(
'draw_visualizations', True, 'Boolean for whether to draw '
'visualizations. This is independent of laser_sampling_rate.')
p.ap_metric = waymo_ap_metric.WaymoAPMetrics.Params(
waymo_metadata.WaymoMetadata())
return p
def testWaymoBreakdowns(self):
metadata = waymo_metadata.WaymoMetadata()
params = waymo_ap_metric.WaymoAPMetrics.Params(metadata)
params.waymo_breakdown_metrics = ['RANGE', 'VELOCITY']
m = params.Instantiate()
# Make one update with a perfect box.
update_dict = py_utils.NestedMap(
groundtruth_labels=np.array([1]),
groundtruth_bboxes=np.ones(shape=(1, 7)),
groundtruth_difficulties=np.zeros(shape=(1)),
groundtruth_num_points=None,
groundtruth_speed=np.zeros(shape=(1, 2)),
detection_scores=np.ones(shape=(5, 1)),
detection_boxes=np.ones(shape=(5, 1, 7)),
detection_heights_in_pixels=np.ones(shape=(5, 1)))
m.Update('1234', update_dict)
# Write a summary.
summary = m.Summary('foo')
# Check that the summary value for default ap and
# a waymo breakdown version by range is the same.
for v in summary.value:
if v.tag == 'foo/Vehicle/AP_LEVEL_1':
default_val = v.simple_value
elif v.tag == 'foo/Vehicle/APH_LEVEL_1':
aph_default_val = v.simple_value
elif v.tag == 'foo_extra/AP_RANGE_TYPE_VEHICLE_[0, 30)_LEVEL_1':
ap_bd_val_l1 = v.simple_value
elif v.tag == 'foo_extra/AP_RANGE_TYPE_VEHICLE_[0, 30)_LEVEL_2':
ap_bd_val_l2 = v.simple_value
elif v.tag == 'foo_extra/APH_RANGE_TYPE_VEHICLE_[0, 30)_LEVEL_1':
aph_bd_val_l1 = v.simple_value
elif v.tag == 'foo_extra/APH_RANGE_TYPE_VEHICLE_[0, 30)_LEVEL_2':
aph_bd_val_l2 = v.simple_value
elif v.tag == 'foo_extra/AP_VELOCITY_TYPE_VEHICLE_STATIONARY_LEVEL_1':
vbd_val_l1 = v.simple_value
elif v.tag == 'foo_extra/AP_VELOCITY_TYPE_VEHICLE_STATIONARY_LEVEL_2':
vbd_val_l2 = v.simple_value
self.assertEqual(ap_bd_val_l1, default_val)
self.assertEqual(ap_bd_val_l2, default_val)
self.assertEqual(aph_bd_val_l1, aph_default_val)
self.assertEqual(aph_bd_val_l2, aph_default_val)
self.assertEqual(vbd_val_l1, default_val)
self.assertEqual(vbd_val_l2, default_val)
# Check that eval classes not evaluated are not present.
tags = [v.tag for v in summary.value]
self.assertNotIn('foo_extra/APH_RANGE_TYPE_SIGN_[0, 30)_LEVEL_1', tags)
self.assertNotIn('foo_extra/APH_RANGE_TYPE_SIGN_[0, 30)_LEVEL_2', tags)
def Params(cls):
p = super(WaymoOpenDatasetDecoder, cls).Params()
p.Define(
'draw_visualizations', False, 'Boolean for whether to draw '
'visualizations. This is independent of laser_sampling_rate.')
p.ap_metric = waymo_ap_metric.WaymoAPMetrics.Params(
waymo_metadata.WaymoMetadata())
p.Define(
'extra_ap_metrics', {},
'Dictionary of extra AP metrics to run in the decoder. The key'
'is the name of the metric and the value is a sub-class of '
'APMetric')
return p
def build_waymo_metric(pred_bbox,
pred_class_id,
pred_class_score,
pred_frame_id,
gt_bbox,
gt_class_id,
gt_frame_id,
gt_speed,
box_type='3d',
breakdowns=None):
"""Build waymo evaluation metric."""
metadata = waymo_metadata.WaymoMetadata()
if breakdowns is None:
breakdowns = ['RANGE']
waymo_metric_config = _build_waymo_metric_config(metadata, box_type,
breakdowns)
ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics(
prediction_bbox=pred_bbox,
prediction_type=pred_class_id,
prediction_score=pred_class_score,
prediction_frame_id=tf.cast(pred_frame_id, tf.int64),
prediction_overlap_nlz=tf.zeros_like(pred_frame_id, dtype=tf.bool),
ground_truth_bbox=gt_bbox,
ground_truth_type=gt_class_id,
ground_truth_frame_id=tf.cast(gt_frame_id, tf.int64),
ground_truth_difficulty=tf.zeros_like(gt_frame_id, dtype=tf.uint8),
ground_truth_speed=gt_speed,
config=waymo_metric_config.SerializeToString())
# All tensors returned by Waymo's metric op have a leading dimension
# B=number of breakdowns. At this moment we always use B=1 to make
# it compatible to the python code.
scalar_metrics = {
'%s_ap' % box_type: ap[0],
'%s_ap_ha_weighted' % box_type: ap_ha[0]
}
curve_metrics = {
'%s_pr' % box_type: pr[0],
'%s_pr_ha_weighted' % box_type: pr_ha[0]
}
breakdown_names = config_util.get_breakdown_names_from_config(
waymo_metric_config)
for i, metric in enumerate(breakdown_names):
# There is a scalar / curve for every breakdown.
scalar_metrics['%s_ap_%s' % (box_type, metric)] = ap[i]
scalar_metrics['%s_ap_ha_weighted_%s' % (box_type, metric)] = ap_ha[i]
curve_metrics['%s_pr_%s' % (box_type, metric)] = pr[i]
curve_metrics['%s_pr_ha_weighted_%s' % (box_type, metric)] = pr_ha[i]
return scalar_metrics, curve_metrics
def testWaymoAPConfig(self):
metadata = waymo_metadata.WaymoMetadata()
# Use 2D metric.
config = waymo_ap_metric._BuildWaymoMetricConfig(metadata, '2d', [])
vehicle_idx = label_pb2.Label.Type.Value('TYPE_VEHICLE')
ped_idx = label_pb2.Label.Type.Value('TYPE_PEDESTRIAN')
cyc_idx = label_pb2.Label.Type.Value('TYPE_CYCLIST')
thresholds_meta = metadata.IoUThresholds()
self.assertNear(config.iou_thresholds[vehicle_idx],
thresholds_meta['Vehicle'], 1e-6)
self.assertNear(config.iou_thresholds[ped_idx],
thresholds_meta['Pedestrian'], 1e-6)
self.assertNear(config.iou_thresholds[cyc_idx],
thresholds_meta['Cyclist'], 1e-6)
def Task(self):
p = super(StarNetVehicle, self).Task()
p.train.l2_regularizer_weight = 3e-5
class_names = waymo_metadata.WaymoMetadata().ClassNames()
num_classes = len(class_names)
p.per_class_loss_weight = [0.] * num_classes
p.per_class_loss_weight[class_names.index('Vehicle')] = 1.0
# See WaymoMetadata.EvalClassIndices for correspondences for metric_weights.
p.output_decoder.ap_metric.metric_weights = {
'default': np.array([1.0, 0.0, 0.0]),
}
p.nms_iou_threshold[class_names.index('Vehicle')] = 0.03
p.nms_score_threshold[class_names.index('Vehicle')] = 0.31
return p
def Params(cls):
p = super().Params()
p.Define(
'draw_visualizations', False, 'Boolean for whether to draw '
'visualizations. This is independent of laser_sampling_rate.')
p.ap_metric = waymo_ap_metric.WaymoAPMetrics.Params(
waymo_metadata.WaymoMetadata())
p.Define(
'extra_ap_metrics', {},
'Dictionary of extra AP metrics to run in the decoder. The key'
'is the name of the metric and the value is a sub-class of '
'APMetric')
p.Define(
'save_residuals', False,
'If True, this expects the residuals and ground-truth to be available '
'in the decoder output dictionary, and it will save it to the decoder '
'output file. See decode_include_residuals in PointDetectorBase '
'for details.')
return p
def testPerfectBox(self):
metadata = waymo_metadata.WaymoMetadata()
params = waymo_ap_metric.WaymoAPMetrics.Params(metadata)
m = params.Instantiate()
# Make one update with a perfect box.
update_dict = py_utils.NestedMap(
groundtruth_labels=np.array([1]),
groundtruth_bboxes=np.ones(shape=(1, 7)),
groundtruth_difficulties=np.zeros(shape=(1)),
groundtruth_num_points=None,
detection_scores=np.ones(shape=(5, 1)),
detection_boxes=np.ones(shape=(5, 1, 7)),
detection_heights_in_pixels=np.ones(shape=(5, 1)))
m.Update('1234', update_dict)
waymo_ap = m._AveragePrecisionByDifficulty()
self.assertAllClose(waymo_ap['default'][0], 1.)
self.assertTrue(np.isnan(waymo_ap['default'][1]))
def Task(self):
metadata = waymo_metadata.WaymoMetadata()
num_classes = len(metadata.ClassNames())
p = starnet.ModelV2.Params(
num_classes,
num_anchor_bboxes_offsets=self.NUM_ANCHOR_BBOX_OFFSETS,
num_anchor_bboxes_rotations=self.NUM_ANCHOR_BBOX_ROTATIONS,
num_anchor_bboxes_dimensions=self.NUM_ANCHOR_BBOX_DIMENSIONS,
num_laser_features=3)
# Update the Point Cloud Featurizer architecture
starnet_builder = starnet.Builder()
starnet_builder.linear_params_init = (
py_utils.WeightInit.KaimingUniformFanInRelu())
gin_layers = [[
self.GIN_HIDDEN_DIMS * 2, self.GIN_HIDDEN_DIMS * 4,
self.GIN_HIDDEN_DIMS
]] * self.NUM_GIN_LAYERS # pyformat: disable
p.cell_featurizer = starnet_builder.GINFeaturizerV2(
'feat',
num_laser_features=3,
fc_dims=self.GIN_HIDDEN_DIMS,
mlp_dims=gin_layers,
fc_use_bn=False)
p.cell_feature_dims = self.GIN_HIDDEN_DIMS * (self.NUM_GIN_LAYERS + 1)
p.output_decoder = waymo_decoder.WaymoOpenDatasetDecoder.Params()
p.max_nms_boxes = 512
p.use_oriented_per_class_nms = True
# Note: Sub-classes need to set nms_iou_threshold and nms_score_threshold
# appropriately.
p.nms_iou_threshold = [0.0] * num_classes
# TODO(jngiam): 1.1 for untrained classes is needed to avoid an issue
# with boxutils error.
p.nms_score_threshold = [1.1] * num_classes
p.name = 'starnet'
tp = p.train
tp.optimizer = optimizer.Adam.Params()
tp.clip_gradient_norm_to_value = 5
ep = p.eval
# Train set uses a smaller decoding set, so we can
# safely eval over the entire input.
ep.samples_per_summary = 0
# To be tuned.
p.train.l2_regularizer_weight = 1e-8
cluster = cluster_factory.Current()
train_cluster_p = cluster.params.Copy()
train_cluster_p.job = 'trainer_client'
train_cluster_p.mode = 'sync'
# When running a decoding only job, there are no trainer workers, so we set
# worker replicas to 1 as a dummy value.
if train_cluster_p.worker.replicas <= 0:
train_cluster_p.worker.replicas = 1
# Set learning rate and schedule.
with cluster_factory.Cluster(train_cluster_p):
train_input_p = self.Train()
# Adapted from V1 tuning.
tp.ema_decay = 0.99
# TODO(b/148537111): consider setting this to True.
tp.ema_decay_moving_vars = False
tp.learning_rate = 0.001
lr_util.SetExponentialLR(train_p=tp,
train_input_p=train_input_p,
exp_start_epoch=5,
total_epoch=75)
p.dimension_loss_weight = .3
p.location_loss_weight = 3.
p.loss_weight_classification = 1.
p.loss_weight_localization = 3.
p.rotation_loss_weight = 0.3
return p