def _BuildConfig(self, additional_config_str=''): """Builds a metrics config.""" config = metrics_pb2.Config() # OBJECT_TYPE adds 4 breakdowns # RANGE adds 12 config_text = """ num_desired_score_cutoffs: 2 breakdown_generator_ids: OBJECT_TYPE difficulties { levels: LEVEL_1 } breakdown_generator_ids: RANGE difficulties { levels: LEVEL_1 } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D score_cutoffs: 0.5 score_cutoffs: 0.9 """ + additional_config_str text_format.Merge(config_text, config) return config
def get_2d_detection_default_config(): """Returns the config proto for WOD 2D detection Evaluation.""" config = metrics_pb2.Config() config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.OBJECT_TYPE) difficulty = config.difficulties.add() difficulty.levels.append(label_pb2.Label.LEVEL_1) difficulty.levels.append(label_pb2.Label.LEVEL_2) config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.ALL_BUT_SIGN) difficulty = config.difficulties.add() difficulty.levels.append(label_pb2.Label.LEVEL_1) difficulty.levels.append(label_pb2.Label.LEVEL_2) config.matcher_type = metrics_pb2.MatcherProto.TYPE_HUNGARIAN config.iou_thresholds.append(0.0) config.iou_thresholds.append(0.7) config.iou_thresholds.append(0.5) config.iou_thresholds.append(0.5) config.iou_thresholds.append(0.5) config.box_type = label_pb2.Label.Box.TYPE_2D for i in range(100): config.score_cutoffs.append(i * 0.01) config.score_cutoffs.append(1.0) return config
def _BuildWaymoMetricConfig(metadata, box_type, waymo_breakdown_metrics): """Build the Config proto for Waymo's metric op.""" config = metrics_pb2.Config() # config.num_desired_score_cutoffs = metadata.NumberOfPrecisionRecallPoints() num_pr_points = metadata.NumberOfPrecisionRecallPoints() config.score_cutoffs.extend( [i * 1.0 / (num_pr_points - 1) for i in range(num_pr_points)]) config.matcher_type = metrics_pb2.MatcherProto.Type.TYPE_HUNGARIAN if box_type == '2d': config.box_type = label_pb2.Label.Box.Type.TYPE_2D else: config.box_type = label_pb2.Label.Box.Type.TYPE_3D # Default values config.iou_thresholds[:] = [0.7, 0.7, 0.7, 0.7, 0.7] for class_name, threshold in metadata.IoUThresholds().items(): cls_idx = metadata.ClassNames().index(class_name) config.iou_thresholds[cls_idx] = threshold config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.ONE_SHARD) config.difficulties.append(metrics_pb2.Difficulty()) # Add extra breakdown metrics. for breakdown_value in waymo_breakdown_metrics: breakdown_id = breakdown_pb2.Breakdown.GeneratorId.Value( breakdown_value) config.breakdown_generator_ids.append(breakdown_id) config.difficulties.append(metrics_pb2.Difficulty()) return config
def _build_config(self): config = metrics_pb2.Config() config_text = """ num_desired_score_cutoffs: 11 breakdown_generator_ids: OBJECT_TYPE breakdown_generator_ids: ONE_SHARD breakdown_generator_ids: RANGE breakdown_generator_ids: RANGE difficulties { } difficulties { levels: LEVEL_1 } difficulties { levels: LEVEL_2 levels: LEVEL_1 } difficulties { levels: LEVEL_1 } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D """ text_format.Merge(config_text, config) return config
def _BuildConfig(self): config = metrics_pb2.Config() # pdb.set_trace() config_text = """ num_desired_score_cutoffs: 11 breakdown_generator_ids: OBJECT_TYPE breakdown_generator_ids: RANGE difficulties { levels: 1 levels: 2 } difficulties { levels: 1 levels: 2 } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.5 iou_thresholds: 0.7 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D """ text_format.Merge(config_text, config) return config
def _get_default_config(self): """Returns the default Config proto for detection. This is the python version of the GetConfig() function in metrics/tools/compute_detection_metrics_main.cc """ config = metrics_pb2.Config() config.breakdown_generator_ids.append( breakdown_pb2.Breakdown.OBJECT_TYPE) difficulty = config.difficulties.add() difficulty.levels.append(label_pb2.Label.LEVEL_1) difficulty.levels.append(label_pb2.Label.LEVEL_2) config.breakdown_generator_ids.append(breakdown_pb2.Breakdown.RANGE) difficulty = config.difficulties.add() difficulty.levels.append(label_pb2.Label.LEVEL_1) difficulty.levels.append(label_pb2.Label.LEVEL_2) config.matcher_type = metrics_pb2.MatcherProto.TYPE_HUNGARIAN config.iou_thresholds.append(0.0) config.iou_thresholds.append(0.7) config.iou_thresholds.append(0.5) config.iou_thresholds.append(0.5) config.iou_thresholds.append(0.5) config.box_type = label_pb2.Label.Box.TYPE_3D for i in range(100): config.score_cutoffs.append(i * 0.01) config.score_cutoffs.append(1.0) return config
def set_config(): config = metrics_pb2.Config() config_text = """ num_desired_score_cutoffs: 11 breakdown_generator_ids: OBJECT_TYPE difficulties { } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D """ text_format.Merge(config_text, config) return config
def testWaymoAPConfig(self): metadata = waymo_metadata.WaymoMetadata() # Use 2D metric. config_str = waymo_ap_metric._BuildWaymoMetricConfig(metadata, '2d') config = metrics_pb2.Config() config.ParseFromString(config_str) vehicle_idx = label_pb2.Label.Type.Value('TYPE_VEHICLE') ped_idx = label_pb2.Label.Type.Value('TYPE_PEDESTRIAN') cyc_idx = label_pb2.Label.Type.Value('TYPE_CYCLIST') thresholds_meta = metadata.IoUThresholds() self.assertNear(config.iou_thresholds[vehicle_idx], thresholds_meta['Vehicle'], 1e-6) self.assertNear(config.iou_thresholds[ped_idx], thresholds_meta['Pedestrian'], 1e-6) self.assertNear(config.iou_thresholds[cyc_idx], thresholds_meta['Cyclist'], 1e-6)
def _BuildConfig(self): """ Builds a metrics config. """ config = metrics_pb2.Config() config_text = """ num_desired_score_cutoffs: 11 breakdown_generator_ids: ONE_SHARD difficulties { } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D """ text_format.Merge(config_text, config) return config
def build_config(self): config = metrics_pb2.Config() config_text = """ breakdown_generator_ids: OBJECT_TYPE difficulties { levels:1 levels:2 } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.0 iou_thresholds: 0.7 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D """ for x in range(0, 100): config.score_cutoffs.append(x * 0.01) config.score_cutoffs.append(1.0) text_format.Merge(config_text, config) return config