def test_get_breakdown_names_from_config(self): config = self._build_config() clif_names = config_util.GetBreakdownNamesFromConfig(config) py_names = config_util_py.get_breakdown_names_from_config(config) self.assertEqual(len(clif_names), len(py_names)) for a, b in zip(clif_names, py_names): self.assertEqual(a, b)
def build_waymo_metric(pred_bbox, pred_class_id, pred_class_score, pred_frame_id, gt_bbox, gt_class_id, gt_frame_id, gt_speed, box_type='3d', breakdowns=None): """Build waymo evaluation metric.""" metadata = waymo_metadata.WaymoMetadata() if breakdowns is None: breakdowns = ['RANGE'] waymo_metric_config = _build_waymo_metric_config(metadata, box_type, breakdowns) ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics( prediction_bbox=pred_bbox, prediction_type=pred_class_id, prediction_score=pred_class_score, prediction_frame_id=tf.cast(pred_frame_id, tf.int64), prediction_overlap_nlz=tf.zeros_like(pred_frame_id, dtype=tf.bool), ground_truth_bbox=gt_bbox, ground_truth_type=gt_class_id, ground_truth_frame_id=tf.cast(gt_frame_id, tf.int64), ground_truth_difficulty=tf.zeros_like(gt_frame_id, dtype=tf.uint8), ground_truth_speed=gt_speed, config=waymo_metric_config.SerializeToString()) # All tensors returned by Waymo's metric op have a leading dimension # B=number of breakdowns. At this moment we always use B=1 to make # it compatible to the python code. scalar_metrics = { '%s_ap' % box_type: ap[0], '%s_ap_ha_weighted' % box_type: ap_ha[0] } curve_metrics = { '%s_pr' % box_type: pr[0], '%s_pr_ha_weighted' % box_type: pr_ha[0] } breakdown_names = config_util.get_breakdown_names_from_config( waymo_metric_config) for i, metric in enumerate(breakdown_names): # There is a scalar / curve for every breakdown. scalar_metrics['%s_ap_%s' % (box_type, metric)] = ap[i] scalar_metrics['%s_ap_ha_weighted_%s' % (box_type, metric)] = ap_ha[i] curve_metrics['%s_pr_%s' % (box_type, metric)] = pr[i] curve_metrics['%s_pr_ha_weighted_%s' % (box_type, metric)] = pr_ha[i] return scalar_metrics, curve_metrics
def __init__(self, params): super(WaymoAPMetrics, self).__init__(params) self._waymo_metric_config = _BuildWaymoMetricConfig( self.metadata, self.params.box_type, self.params.waymo_breakdown_metrics) # Compute only waymo breakdown metrics. breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) waymo_params = WaymoBreakdownMetric.Params().Set( metadata=self.metadata, breakdown_list=breakdown_names) self._breakdown_metrics['waymo'] = WaymoBreakdownMetric(waymo_params) # Remove the base metric. del self._breakdown_metrics['difficulty']
def value(self): """Returns weighted mAP over all eval classes.""" self._EvaluateIfNecessary() ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) num_sum = 0.0 denom_sum = 0.0 # Compute the average AP over all eval classes. The first breakdown # is the overall mAP. for class_index in range(len(self.metadata.EvalClassIndices())): num_sum += np.nan_to_num(ap[breakdown_names[0]][class_index]) denom_sum += 1. return num_sum / denom_sum
def Summary(self, name): """Implements custom Summary for Waymo metrics.""" self._EvaluateIfNecessary() ret = tf.Summary() # Put '.value' first (so it shows up in logs / summaries, etc). ret.value.add(tag='{}/weighted_mAP'.format(name), simple_value=self.value) ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access aph = self._breakdown_metrics['waymo']._average_precision_headings # pylint:disable=protected-access breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) for i, j in enumerate(self.metadata.EvalClassIndices()): classname = self.metadata.ClassNames()[j] for k, breakdown_name in enumerate(breakdown_names): # Skip adding entries for breakdowns that are in a different class. # # The first breakdown is the overall one, so never skip it. if k > 0 and classname.lower() not in breakdown_name.lower(): continue if k == 0: # For the overall mAP, include the class name # and set the breakdown_str to 'default' for backwards compatibility. prefix = '{}/{}'.format(name, classname) breakdown_str = 'default' else: # All breakdowns after the first one are extra and elide # the classname, since it is present in the breakdown_name. prefix = '{}_extra'.format(name) breakdown_str = breakdown_name tag_str = '{}/AP_{}'.format(prefix, breakdown_str) ap_value = ap[breakdown_name][i] ret.value.add(tag=tag_str, simple_value=ap_value) tag_str = '{}/APH_{}'.format(prefix, breakdown_str) aph_value = aph[breakdown_name][i] ret.value.add(tag=tag_str, simple_value=aph_value) image_summaries = self._breakdown_metrics['waymo'].GenerateSummaries( name) for image_summary in image_summaries: ret.value.extend(image_summary.value) return ret
def Summary(self, name): """Implements custom Summary for Waymo metrics.""" self._EvaluateIfNecessary() ret = tf.Summary() # Put '.value' first (so it shows up in logs / summaries, etc). ret.value.add(tag='{}/weighted_mAP'.format(name), simple_value=self.value) ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access aph = self._breakdown_metrics['waymo']._average_precision_headings # pylint:disable=protected-access breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) for i, class_index in enumerate(self.metadata.EvalClassIndices()): classname = self.metadata.ClassNames()[class_index] for breakdown_name in breakdown_names: # 'ONE_SHARD' breakdowns are the overall metrics (not sliced up) # So we should make that the defualt metric. if 'ONE_SHARD' in breakdown_name: # For the overall mAP, include the class name # and set the breakdown_str which will have the level prefix = '{}/{}'.format(name, classname) postfix = breakdown_name.replace('ONE_SHARD_', '') breakdown_str = postfix if postfix else 'UNKNOWN' # Otherwise check that the class we are looking at is in the breakdown. elif classname.lower() in breakdown_name.lower(): prefix = '{}_extra'.format(name) breakdown_str = breakdown_name else: continue tag_str = '{}/AP_{}'.format(prefix, breakdown_str) ap_value = ap[breakdown_name][i] ret.value.add(tag=tag_str, simple_value=ap_value) tag_str = '{}/APH_{}'.format(prefix, breakdown_str) aph_value = aph[breakdown_name][i] ret.value.add(tag=tag_str, simple_value=aph_value) image_summaries = self._breakdown_metrics['waymo'].GenerateSummaries( name) for image_summary in image_summaries: ret.value.extend(image_summary.value) return ret
def __init__(self, params): super(WaymoAPMetrics, self).__init__(params) self._waymo_metric_config = _BuildWaymoMetricConfig( self.metadata, self.params.box_type, self.params.waymo_breakdown_metrics) # Add APH metric. metrics_params = breakdown_metric.ByDifficulty.Params().Set( metadata=self.metadata, ap_key='ap_ha_weighted', pr_key='pr_ha_weighted') self._breakdown_metrics['aph'] = breakdown_metric.ByDifficulty( metrics_params) # Compute extra breakdown metrics. breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) waymo_params = WaymoBreakdownMetric.Params().Set( metadata=self.metadata, breakdown_list=breakdown_names[1:]) self._breakdown_metrics['waymo'] = WaymoBreakdownMetric(waymo_params)
def Summary(self, name): ret = super(WaymoAPMetrics, self).Summary(name) self._EvaluateIfNecessary() aph = self._breakdown_metrics['aph']._average_precisions # pylint:disable=protected-access for i, j in enumerate(self.metadata.EvalClassIndices()): classname = self.metadata.ClassNames()[j] for difficulty in self.metadata.DifficultyLevels(): tag_str = '{}/{}/APH_{}'.format(name, classname, difficulty) aph_value = aph[difficulty][i] ret.value.add(tag=tag_str, simple_value=aph_value) image_summaries = self._breakdown_metrics['aph'].GenerateSummaries(name + '_aph') for image_summary in image_summaries: ret.value.extend(image_summary.value) ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access aph = self._breakdown_metrics['waymo']._average_precision_headings # pylint:disable=protected-access breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) for i, j in enumerate(self.metadata.EvalClassIndices()): classname = self.metadata.ClassNames()[j] for breakdown_name in breakdown_names[1:]: # Skip adding entries for breakdowns that are in a different class. if classname.lower() not in breakdown_name.lower(): continue # Use a different suffix to avoid polluting the main metrics tab. tag_str = '{}_extra/AP_{}'.format(name, breakdown_name) ap_value = ap[breakdown_name][i] ret.value.add(tag=tag_str, simple_value=ap_value) tag_str = '{}_extra/APH_{}'.format(name, breakdown_name) aph_value = aph[breakdown_name][i] ret.value.add(tag=tag_str, simple_value=aph_value) return ret
def __init__(self, config=None): """Constructs WOD detection evaluation class. Args: config: The metrics config defined in protos/metrics.proto. """ if config is None: config = self._get_default_config() self._config = config # These are the keys in the metric_dict returned by evaluate. self._metric_names = [ 'average_precision', 'average_precision_ha_weighted', 'precision_recall', 'precision_recall_ha_weighted', 'breakdown', ] self._breakdown_names = config_util.get_breakdown_names_from_config( config) self._required_prediction_fields = [ 'prediction_frame_id', 'prediction_bbox', 'prediction_type', 'prediction_score', 'prediction_overlap_nlz', ] self._required_groundtruth_fields = [ 'ground_truth_frame_id', 'ground_truth_bbox', 'ground_truth_type', 'ground_truth_difficulty', ] self.reset_states()
def get_detection_metric_ops( config, prediction_frame_id, prediction_bbox, prediction_type, prediction_score, prediction_overlap_nlz, ground_truth_frame_id, ground_truth_bbox, ground_truth_type, ground_truth_difficulty, ground_truth_speed=None, recall_at_precision=None, ): """Returns dict of metric name to tuples of `(value_op, update_op)`. Each update_op accumulates the prediction and ground truth tensors to its corresponding tf variables. Each value_op computes detection metrics on all prediction and ground truth seen so far. This works similar as `tf.metrics` code. Notation: * M: number of predicted boxes. * D: number of box dimensions. The number of box dimensions can be one of the following: 4: Used for boxes with type TYPE_AA_2D (center_x, center_y, length, width) 5: Used for boxes with type TYPE_2D (center_x, center_y, length, width, heading). 7: Used for boxes with type TYPE_3D (center_x, center_y, center_z, length, width, height, heading). * N: number of ground truth boxes. Args: config: The metrics config defined in protos/metrics.proto. prediction_frame_id: [M] int64 tensor that identifies frame for each prediction. prediction_bbox: [M, D] tensor encoding the predicted bounding boxes. prediction_type: [M] tensor encoding the object type of each prediction. prediction_score: [M] tensor encoding the score of each prediciton. prediction_overlap_nlz: [M] tensor encoding whether each prediciton overlaps with any no label zone. ground_truth_frame_id: [N] int64 tensor that identifies frame for each ground truth. ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes. ground_truth_type: [N] tensor encoding the object type of each ground truth. ground_truth_difficulty: [N] tensor encoding the difficulty level of each ground truth. ground_truth_speed: [N, 2] tensor with the vx, vy velocity for each object. recall_at_precision: a float within [0,1]. If set, returns a 3rd metric that reports the recall at the given precision. Returns: A dictionary of metric names to tuple of value_op and update_op. """ if ground_truth_speed is None: num_gt_boxes = tf.shape(ground_truth_bbox)[0] ground_truth_speed = tf.zeros((num_gt_boxes, 2), tf.float32) eval_dict = { 'prediction_frame_id': (prediction_frame_id, [0], tf.int64), 'prediction_bbox': (prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32), 'prediction_type': (prediction_type, [0], tf.uint8), 'prediction_score': (prediction_score, [0], tf.float32), 'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool), 'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64), 'ground_truth_bbox': (ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32), 'ground_truth_type': (ground_truth_type, [0], tf.uint8), 'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8), 'ground_truth_speed': (ground_truth_speed, [0, 2], tf.float32), } variable_and_update_ops = {} for name, value in eval_dict.items(): update, init_shape, dtype = value variable_and_update_ops[name] = _update(name, update, init_shape, dtype) update_ops = [value[1] for value in variable_and_update_ops.values()] update_op = tf.group(update_ops) variable_map = { name: value[0] for name, value in variable_and_update_ops.items() } config_str = config.SerializeToString() ap, aph, pr, _, _ = py_metrics_ops.detection_metrics( config=config_str, **variable_map) breakdown_names = config_util.get_breakdown_names_from_config(config) metric_ops = {} for i, name in enumerate(breakdown_names): if i == 0: metric_ops['{}/AP'.format(name)] = (ap[i], update_op) else: # Set update_op to be an no-op just in case if anyone runs update_ops in # multiple session.run()s. metric_ops['{}/AP'.format(name)] = (ap[i], tf.constant([])) metric_ops['{}/APH'.format(name)] = (aph[i], tf.constant([])) if recall_at_precision is not None: precision_i_mask = pr[i, :, 0] > recall_at_precision recall_i = tf.reduce_max( tf.where(precision_i_mask, pr[i, :, 1], tf.zeros_like(pr[i, :, 1]))) metric_ops['{}/Recall@{}'.format(name, recall_at_precision)] = (recall_i, tf.constant([])) return metric_ops
def _BuildMetric(self, feed_data, classid): """Construct tensors and the feed_dict for Waymo metric op. Args: feed_data: a NestedMap returned by _GetData(). classid: integer. Returns: A tuple of 3 dicts: - scalar_metrics: a dict mapping all the metric names to fetch tensors. - curves: a dict mapping all the curve names to fetch tensors. - feed_dict: a dict mapping the tensors in feed_tensors to feed values. """ breakdown_names = config_util.get_breakdown_names_from_config( self._waymo_metric_config) if feed_data is None: dummy_scalar = tf.constant(np.nan) dummy_curve = tf.zeros( [self.metadata.NumberOfPrecisionRecallPoints(), 2], tf.float32) scalar_metrics = { 'ap': dummy_scalar, 'ap_ha_weighted': dummy_scalar } curve_metrics = {'pr': dummy_curve, 'pr_ha_weighted': dummy_curve} for i, metric in enumerate(breakdown_names): scalar_metrics['ap_%s' % metric] = dummy_scalar scalar_metrics['ap_ha_weighted_%s' % metric] = dummy_scalar curve_metrics['pr_%s' % metric] = dummy_curve curve_metrics['pr_ha_weighted_%s' % metric] = dummy_curve return py_utils.NestedMap(feed_dict={}, scalar_metrics=scalar_metrics, curve_metrics=curve_metrics) feed_dict = {} f_gt_bbox = tf.placeholder(tf.float32) feed_dict[f_gt_bbox] = feed_data.gt.bbox f_gt_imgid = tf.placeholder(tf.int32) feed_dict[f_gt_imgid] = feed_data.gt.imgid f_gt_speed = tf.placeholder(tf.float32) feed_dict[f_gt_speed] = feed_data.gt.speed f_pd_bbox = tf.placeholder(tf.float32) feed_dict[f_pd_bbox] = feed_data.pd.bbox f_pd_imgid = tf.placeholder(tf.int32) feed_dict[f_pd_imgid] = feed_data.pd.imgid f_pd_score = tf.placeholder(tf.float32) feed_dict[f_pd_score] = feed_data.pd.score num_gt_bboxes = feed_data.gt.imgid.shape[0] num_pd_bboxes = feed_data.pd.imgid.shape[0] gt_class_ids = tf.constant(classid, dtype=tf.uint8, shape=[num_gt_bboxes]) pd_class_ids = tf.constant(classid, dtype=tf.uint8, shape=[num_pd_bboxes]) ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics( prediction_bbox=f_pd_bbox, prediction_type=pd_class_ids, prediction_score=f_pd_score, prediction_frame_id=tf.cast(f_pd_imgid, tf.int64), prediction_overlap_nlz=tf.zeros_like(f_pd_imgid, dtype=tf.bool), ground_truth_bbox=f_gt_bbox, ground_truth_type=gt_class_ids, ground_truth_frame_id=tf.cast(f_gt_imgid, tf.int64), ground_truth_difficulty=tf.zeros_like(f_gt_imgid, dtype=tf.uint8), ground_truth_speed=f_gt_speed, config=self._waymo_metric_config.SerializeToString()) # All tensors returned by Waymo's metric op have a leading dimension # B=number of breakdowns. At this moment we always use B=1 to make # it compatible to the python code. scalar_metrics = {'ap': ap[0], 'ap_ha_weighted': ap_ha[0]} curve_metrics = {'pr': pr[0], 'pr_ha_weighted': pr_ha[0]} for i, metric in enumerate(breakdown_names): # There is a scalar / curve for every breakdown. scalar_metrics['ap_%s' % metric] = ap[i] scalar_metrics['ap_ha_weighted_%s' % metric] = ap_ha[i] curve_metrics['pr_%s' % metric] = pr[i] curve_metrics['pr_ha_weighted_%s' % metric] = pr_ha[i] return py_utils.NestedMap(feed_dict=feed_dict, scalar_metrics=scalar_metrics, curve_metrics=curve_metrics)
def get_tracking_metric_ops(config, prediction_bbox, prediction_type, prediction_score, prediction_frame_id, prediction_sequence_id, prediction_object_id, ground_truth_bbox, ground_truth_type, ground_truth_frame_id, ground_truth_sequence_id, ground_truth_object_id, ground_truth_difficulty, prediction_overlap_nlz=None, ground_truth_speed=None): """Returns dict of metric name to tuples of `(value_op, update_op)`. Each update_op accumulates the prediction and ground truth tensors to its corresponding tf variables. Each value_op computes tracking metrics on all prediction and ground truth seen so far. This works similar as `tf.metrics` code. Notation: * M: number of predicted boxes. * D: number of box dimensions. The number of box dimensions can be one of the following: 4: Used for boxes with type TYPE_AA_2D (center_x, center_y, length, width) 5: Used for boxes with type TYPE_2D (center_x, center_y, length, width, heading). 7: Used for boxes with type TYPE_3D (center_x, center_y, center_z, length, width, height, heading). * N: number of ground truth boxes. Args: config: The metrics config defined in protos/metrics.proto. prediction_bbox: [M, D] tensor encoding the predicted bounding boxes. prediction_type: [M] tensor encoding the object type of each prediction. prediction_score: [M] tensor encoding the score of each prediction. prediction_frame_id: [M] int64 tensor that identifies frame for each prediction. prediction_sequence_id: [M] int64 tensor that identifies sequence for each prediction. prediction_object_id: [M] int64 tensor that identifies object for each prediction. ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes. ground_truth_type: [N] tensor encoding the object type of each ground truth. ground_truth_frame_id: [N] int64 tensor that identifies frame for each ground truth. ground_truth_sequence_id: [M] int64 tensor that identifies sequence for each ground truth. ground_truth_object_id: [M] int64 tensor that identifies object for each ground truth. ground_truth_difficulty: [N] tensor encoding the difficulty level of each ground truth. prediction_overlap_nlz: [M] tensor encoding whether each prediction overlaps with any no label zone. ground_truth_speed: [N, 2] tensor with the vx, vy velocity for each object. Returns: A dictionary of metric names to tuple of value_op and update_op. """ if ground_truth_speed is None: num_gt_boxes = tf.shape(ground_truth_bbox)[0] ground_truth_speed = tf.zeros((num_gt_boxes, 2), tf.float32) if prediction_overlap_nlz is None: prediction_overlap_nlz = tf.zeros_like(prediction_frame_id, dtype=tf.bool) eval_dict = { 'prediction_bbox': (prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32), 'prediction_type': (prediction_type, [0], tf.uint8), 'prediction_score': (prediction_score, [0], tf.float32), 'prediction_frame_id': (prediction_frame_id, [0], tf.int64), 'prediction_sequence_id': (prediction_sequence_id, [0], tf.string), 'prediction_object_id': (prediction_object_id, [0], tf.int64), 'ground_truth_bbox': (ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32), 'ground_truth_type': (ground_truth_type, [0], tf.uint8), 'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64), 'ground_truth_sequence_id': (ground_truth_sequence_id, [0], tf.string), 'ground_truth_object_id': (ground_truth_object_id, [0], tf.int64), 'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8), 'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool), 'ground_truth_speed': (ground_truth_speed, [0, 2], tf.float32), } variable_and_update_ops = { name: _update(name, update, init_shape, dtype) for name, (update, init_shape, dtype) in eval_dict.items() } update_ops = [value[1] for value in variable_and_update_ops.values()] update_op = tf.group(update_ops) variable_map = { name: value[0] for name, value in variable_and_update_ops.items() } config_str = config.SerializeToString() mota, motp, miss, mismatch, fp, _, _ = py_metrics_ops.tracking_metrics( config=config_str, **variable_map) breakdown_names = config_util.get_breakdown_names_from_config(config) metric_ops = {} for i, name in enumerate(breakdown_names): if i == 0: metric_ops['{}/MOTA'.format(name)] = (mota[i], update_op) else: # Set update_op to be an no-op just in case if anyone runs update_ops in # multiple session.run()s. metric_ops['{}/MOTA'.format(name)] = (mota[i], tf.constant([])) metric_ops['{}/MOTP'.format(name)] = (motp[i], tf.constant([])) metric_ops['{}/MISS'.format(name)] = (miss[i], tf.constant([])) metric_ops['{}/MISMATCH'.format(name)] = (mismatch[i], tf.constant([])) metric_ops['{}/FP'.format(name)] = (fp[i], tf.constant([])) return metric_ops
def get_detection_metric_ops( config, prediction_frame_id, prediction_bbox, prediction_type, prediction_score, prediction_overlap_nlz, ground_truth_frame_id, ground_truth_bbox, ground_truth_type, ground_truth_difficulty, ): """Returns dict of metric name to tuples of `(value_op, update_op)`. Each update_op accumulates the prediction and ground truth tensors to its corresponding tf variables. Each value_op computes detection metrics on all prediction and groud truth seen so far. This works similar as `tf.metrics` code. Notation: * M: number of predicted boxes. * D: number of box dimensions (4, 5 or 7). * N: number of ground truth boxes. Args: prediction_frame_id: [M] int64 tensor that identifies frame for each prediction. prediction_bbox: [M, D] tensor encoding the predicted bounding boxes. prediction_type: [M] tensor encoding the object type of each prediction. prediction_score: [M] tensor encoding the score of each prediciton. prediction_overlap_nlz: [M] tensor encoding whether each prediciton overlaps with any no label zone. ground_truth_frame_id: [N] int64 tensor that identifies frame for each ground truth. ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes. ground_truth_type: [N] tensor encoding the object type of each ground truth. ground_truth_difficulty: [N] tensor encoding the difficulty level of each ground truth. Returns: A dictionary of metric names to tuple of value_op and update_op. """ eval_dict = { 'prediction_frame_id': (prediction_frame_id, [0], tf.int64), 'prediction_bbox': (prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32), 'prediction_type': (prediction_type, [0], tf.uint8), 'prediction_score': (prediction_score, [0], tf.float32), 'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool), 'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64), 'ground_truth_bbox': (ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32), 'ground_truth_type': (ground_truth_type, [0], tf.uint8), 'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8), } variable_and_update_ops = {} for name, value in eval_dict.items(): update, init_shape, dtype = value variable_and_update_ops[name] = _update(name, update, init_shape, dtype) update_ops = [value[1] for value in variable_and_update_ops.values()] update_op = tf.group(update_ops) variable_map = { name: value[0] for name, value in variable_and_update_ops.items() } config_str = config.SerializeToString() ap, aph, _, _, _ = py_metrics_ops.detection_metrics(config=config_str, **variable_map) breakdown_names = config_util.get_breakdown_names_from_config(config) metric_ops = {} for i, name in enumerate(breakdown_names): if i == 0: metric_ops['{}/AP'.format(name)] = (ap[i], update_op) else: # Set update_op to be an no-op just in case if anyone runs update_ops in # multiple session.run()s. metric_ops['{}/AP'.format(name)] = (ap[i], tf.constant([])) metric_ops['{}/APH'.format(name)] = (aph[i], tf.constant([])) return metric_ops