def _BuildConfig(self, additional_config_str=''):
"""Builds a metrics config."""
config = metrics_pb2.Config()
# OBJECT_TYPE adds 4 breakdowns
# RANGE adds 12
config_text = """
num_desired_score_cutoffs: 2
breakdown_generator_ids: OBJECT_TYPE
difficulties {
levels: LEVEL_1
}
breakdown_generator_ids: RANGE
difficulties {
levels: LEVEL_1
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
score_cutoffs: 0.5
score_cutoffs: 0.9
""" + additional_config_str
text_format.Merge(config_text, config)
return config
def get_2d_detection_default_config():
"""Returns the config proto for WOD 2D detection Evaluation."""
config = metrics_pb2.Config()
config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.OBJECT_TYPE)
difficulty = config.difficulties.add()
difficulty.levels.append(label_pb2.Label.LEVEL_1)
difficulty.levels.append(label_pb2.Label.LEVEL_2)
config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.ALL_BUT_SIGN)
difficulty = config.difficulties.add()
difficulty.levels.append(label_pb2.Label.LEVEL_1)
difficulty.levels.append(label_pb2.Label.LEVEL_2)
config.matcher_type = metrics_pb2.MatcherProto.TYPE_HUNGARIAN
config.iou_thresholds.append(0.0)
config.iou_thresholds.append(0.7)
config.iou_thresholds.append(0.5)
config.iou_thresholds.append(0.5)
config.iou_thresholds.append(0.5)
config.box_type = label_pb2.Label.Box.TYPE_2D
for i in range(100):
config.score_cutoffs.append(i * 0.01)
config.score_cutoffs.append(1.0)
return config
def _BuildWaymoMetricConfig(metadata, box_type, waymo_breakdown_metrics):
"""Build the Config proto for Waymo's metric op."""
config = metrics_pb2.Config()
# config.num_desired_score_cutoffs = metadata.NumberOfPrecisionRecallPoints()
num_pr_points = metadata.NumberOfPrecisionRecallPoints()
config.score_cutoffs.extend(
[i * 1.0 / (num_pr_points - 1) for i in range(num_pr_points)])
config.matcher_type = metrics_pb2.MatcherProto.Type.TYPE_HUNGARIAN
if box_type == '2d':
config.box_type = label_pb2.Label.Box.Type.TYPE_2D
else:
config.box_type = label_pb2.Label.Box.Type.TYPE_3D
# Default values
config.iou_thresholds[:] = [0.7, 0.7, 0.7, 0.7, 0.7]
for class_name, threshold in metadata.IoUThresholds().items():
cls_idx = metadata.ClassNames().index(class_name)
config.iou_thresholds[cls_idx] = threshold
config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.ONE_SHARD)
config.difficulties.append(metrics_pb2.Difficulty())
# Add extra breakdown metrics.
for breakdown_value in waymo_breakdown_metrics:
breakdown_id = breakdown_pb2.Breakdown.GeneratorId.Value(
breakdown_value)
config.breakdown_generator_ids.append(breakdown_id)
config.difficulties.append(metrics_pb2.Difficulty())
return config
def _build_config(self):
config = metrics_pb2.Config()
config_text = """
num_desired_score_cutoffs: 11
breakdown_generator_ids: OBJECT_TYPE
breakdown_generator_ids: ONE_SHARD
breakdown_generator_ids: RANGE
breakdown_generator_ids: RANGE
difficulties {
}
difficulties {
levels: LEVEL_1
}
difficulties {
levels: LEVEL_2
levels: LEVEL_1
}
difficulties {
levels: LEVEL_1
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
"""
text_format.Merge(config_text, config)
return config
def _BuildConfig(self):
config = metrics_pb2.Config()
# pdb.set_trace()
config_text = """
num_desired_score_cutoffs: 11
breakdown_generator_ids: OBJECT_TYPE
breakdown_generator_ids: RANGE
difficulties {
levels: 1
levels: 2
}
difficulties {
levels: 1
levels: 2
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.5
iou_thresholds: 0.7
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
"""
text_format.Merge(config_text, config)
return config
def _get_default_config(self):
"""Returns the default Config proto for detection.
This is the python version of the GetConfig() function in
metrics/tools/compute_detection_metrics_main.cc
"""
config = metrics_pb2.Config()
config.breakdown_generator_ids.append(
breakdown_pb2.Breakdown.OBJECT_TYPE)
difficulty = config.difficulties.add()
difficulty.levels.append(label_pb2.Label.LEVEL_1)
difficulty.levels.append(label_pb2.Label.LEVEL_2)
config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.RANGE)
difficulty = config.difficulties.add()
difficulty.levels.append(label_pb2.Label.LEVEL_1)
difficulty.levels.append(label_pb2.Label.LEVEL_2)
config.matcher_type = metrics_pb2.MatcherProto.TYPE_HUNGARIAN
config.iou_thresholds.append(0.0)
config.iou_thresholds.append(0.7)
config.iou_thresholds.append(0.5)
config.iou_thresholds.append(0.5)
config.iou_thresholds.append(0.5)
config.box_type = label_pb2.Label.Box.TYPE_3D
for i in range(100):
config.score_cutoffs.append(i * 0.01)
config.score_cutoffs.append(1.0)
return config
def set_config():
config = metrics_pb2.Config()
config_text = """
num_desired_score_cutoffs: 11
breakdown_generator_ids: OBJECT_TYPE
difficulties {
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
"""
text_format.Merge(config_text, config)
return config
def testWaymoAPConfig(self):
metadata = waymo_metadata.WaymoMetadata()
# Use 2D metric.
config_str = waymo_ap_metric._BuildWaymoMetricConfig(metadata, '2d')
config = metrics_pb2.Config()
config.ParseFromString(config_str)
vehicle_idx = label_pb2.Label.Type.Value('TYPE_VEHICLE')
ped_idx = label_pb2.Label.Type.Value('TYPE_PEDESTRIAN')
cyc_idx = label_pb2.Label.Type.Value('TYPE_CYCLIST')
thresholds_meta = metadata.IoUThresholds()
self.assertNear(config.iou_thresholds[vehicle_idx],
thresholds_meta['Vehicle'], 1e-6)
self.assertNear(config.iou_thresholds[ped_idx],
thresholds_meta['Pedestrian'], 1e-6)
self.assertNear(config.iou_thresholds[cyc_idx], thresholds_meta['Cyclist'],
1e-6)
def _BuildConfig(self):
""" Builds a metrics config. """
config = metrics_pb2.Config()
config_text = """
num_desired_score_cutoffs: 11
breakdown_generator_ids: ONE_SHARD
difficulties {
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
"""
text_format.Merge(config_text, config)
return config
def build_config(self):
config = metrics_pb2.Config()
config_text = """
breakdown_generator_ids: OBJECT_TYPE
difficulties {
levels:1
levels:2
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.0
iou_thresholds: 0.7
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
"""
for x in range(0, 100):
config.score_cutoffs.append(x * 0.01)
config.score_cutoffs.append(1.0)
text_format.Merge(config_text, config)
return config
