def test_get_breakdown_names_from_config(self):
config = self._build_config()
clif_names = config_util.GetBreakdownNamesFromConfig(config)
py_names = config_util_py.get_breakdown_names_from_config(config)
self.assertEqual(len(clif_names), len(py_names))
for a, b in zip(clif_names, py_names):
self.assertEqual(a, b)
def build_waymo_metric(pred_bbox,
pred_class_id,
pred_class_score,
pred_frame_id,
gt_bbox,
gt_class_id,
gt_frame_id,
gt_speed,
box_type='3d',
breakdowns=None):
"""Build waymo evaluation metric."""
metadata = waymo_metadata.WaymoMetadata()
if breakdowns is None:
breakdowns = ['RANGE']
waymo_metric_config = _build_waymo_metric_config(metadata, box_type,
breakdowns)
ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics(
prediction_bbox=pred_bbox,
prediction_type=pred_class_id,
prediction_score=pred_class_score,
prediction_frame_id=tf.cast(pred_frame_id, tf.int64),
prediction_overlap_nlz=tf.zeros_like(pred_frame_id, dtype=tf.bool),
ground_truth_bbox=gt_bbox,
ground_truth_type=gt_class_id,
ground_truth_frame_id=tf.cast(gt_frame_id, tf.int64),
ground_truth_difficulty=tf.zeros_like(gt_frame_id, dtype=tf.uint8),
ground_truth_speed=gt_speed,
config=waymo_metric_config.SerializeToString())
# All tensors returned by Waymo's metric op have a leading dimension
# B=number of breakdowns. At this moment we always use B=1 to make
# it compatible to the python code.
scalar_metrics = {
'%s_ap' % box_type: ap[0],
'%s_ap_ha_weighted' % box_type: ap_ha[0]
}
curve_metrics = {
'%s_pr' % box_type: pr[0],
'%s_pr_ha_weighted' % box_type: pr_ha[0]
}
breakdown_names = config_util.get_breakdown_names_from_config(
waymo_metric_config)
for i, metric in enumerate(breakdown_names):
# There is a scalar / curve for every breakdown.
scalar_metrics['%s_ap_%s' % (box_type, metric)] = ap[i]
scalar_metrics['%s_ap_ha_weighted_%s' % (box_type, metric)] = ap_ha[i]
curve_metrics['%s_pr_%s' % (box_type, metric)] = pr[i]
curve_metrics['%s_pr_ha_weighted_%s' % (box_type, metric)] = pr_ha[i]
return scalar_metrics, curve_metrics
def __init__(self, params):
super(WaymoAPMetrics, self).__init__(params)
self._waymo_metric_config = _BuildWaymoMetricConfig(
self.metadata, self.params.box_type,
self.params.waymo_breakdown_metrics)
# Compute only waymo breakdown metrics.
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
waymo_params = WaymoBreakdownMetric.Params().Set(
metadata=self.metadata, breakdown_list=breakdown_names)
self._breakdown_metrics['waymo'] = WaymoBreakdownMetric(waymo_params)
# Remove the base metric.
del self._breakdown_metrics['difficulty']
def value(self):
"""Returns weighted mAP over all eval classes."""
self._EvaluateIfNecessary()
ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
num_sum = 0.0
denom_sum = 0.0
# Compute the average AP over all eval classes. The first breakdown
# is the overall mAP.
for class_index in range(len(self.metadata.EvalClassIndices())):
num_sum += np.nan_to_num(ap[breakdown_names[0]][class_index])
denom_sum += 1.
return num_sum / denom_sum
def Summary(self, name):
"""Implements custom Summary for Waymo metrics."""
self._EvaluateIfNecessary()
ret = tf.Summary()
# Put '.value' first (so it shows up in logs / summaries, etc).
ret.value.add(tag='{}/weighted_mAP'.format(name),
simple_value=self.value)
ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access
aph = self._breakdown_metrics['waymo']._average_precision_headings # pylint:disable=protected-access
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
for i, j in enumerate(self.metadata.EvalClassIndices()):
classname = self.metadata.ClassNames()[j]
for k, breakdown_name in enumerate(breakdown_names):
# Skip adding entries for breakdowns that are in a different class.
#
# The first breakdown is the overall one, so never skip it.
if k > 0 and classname.lower() not in breakdown_name.lower():
continue
if k == 0:
# For the overall mAP, include the class name
# and set the breakdown_str to 'default' for backwards compatibility.
prefix = '{}/{}'.format(name, classname)
breakdown_str = 'default'
else:
# All breakdowns after the first one are extra and elide
# the classname, since it is present in the breakdown_name.
prefix = '{}_extra'.format(name)
breakdown_str = breakdown_name
tag_str = '{}/AP_{}'.format(prefix, breakdown_str)
ap_value = ap[breakdown_name][i]
ret.value.add(tag=tag_str, simple_value=ap_value)
tag_str = '{}/APH_{}'.format(prefix, breakdown_str)
aph_value = aph[breakdown_name][i]
ret.value.add(tag=tag_str, simple_value=aph_value)
image_summaries = self._breakdown_metrics['waymo'].GenerateSummaries(
name)
for image_summary in image_summaries:
ret.value.extend(image_summary.value)
return ret
def Summary(self, name):
"""Implements custom Summary for Waymo metrics."""
self._EvaluateIfNecessary()
ret = tf.Summary()
# Put '.value' first (so it shows up in logs / summaries, etc).
ret.value.add(tag='{}/weighted_mAP'.format(name),
simple_value=self.value)
ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access
aph = self._breakdown_metrics['waymo']._average_precision_headings # pylint:disable=protected-access
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
for i, class_index in enumerate(self.metadata.EvalClassIndices()):
classname = self.metadata.ClassNames()[class_index]
for breakdown_name in breakdown_names:
# 'ONE_SHARD' breakdowns are the overall metrics (not sliced up)
# So we should make that the defualt metric.
if 'ONE_SHARD' in breakdown_name:
# For the overall mAP, include the class name
# and set the breakdown_str which will have the level
prefix = '{}/{}'.format(name, classname)
postfix = breakdown_name.replace('ONE_SHARD_', '')
breakdown_str = postfix if postfix else 'UNKNOWN'
# Otherwise check that the class we are looking at is in the breakdown.
elif classname.lower() in breakdown_name.lower():
prefix = '{}_extra'.format(name)
breakdown_str = breakdown_name
else:
continue
tag_str = '{}/AP_{}'.format(prefix, breakdown_str)
ap_value = ap[breakdown_name][i]
ret.value.add(tag=tag_str, simple_value=ap_value)
tag_str = '{}/APH_{}'.format(prefix, breakdown_str)
aph_value = aph[breakdown_name][i]
ret.value.add(tag=tag_str, simple_value=aph_value)
image_summaries = self._breakdown_metrics['waymo'].GenerateSummaries(
name)
for image_summary in image_summaries:
ret.value.extend(image_summary.value)
return ret
def __init__(self, params):
super(WaymoAPMetrics, self).__init__(params)
self._waymo_metric_config = _BuildWaymoMetricConfig(
self.metadata, self.params.box_type,
self.params.waymo_breakdown_metrics)
# Add APH metric.
metrics_params = breakdown_metric.ByDifficulty.Params().Set(
metadata=self.metadata,
ap_key='ap_ha_weighted',
pr_key='pr_ha_weighted')
self._breakdown_metrics['aph'] = breakdown_metric.ByDifficulty(
metrics_params)
# Compute extra breakdown metrics.
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
waymo_params = WaymoBreakdownMetric.Params().Set(
metadata=self.metadata, breakdown_list=breakdown_names[1:])
self._breakdown_metrics['waymo'] = WaymoBreakdownMetric(waymo_params)
def Summary(self, name):
ret = super(WaymoAPMetrics, self).Summary(name)
self._EvaluateIfNecessary()
aph = self._breakdown_metrics['aph']._average_precisions # pylint:disable=protected-access
for i, j in enumerate(self.metadata.EvalClassIndices()):
classname = self.metadata.ClassNames()[j]
for difficulty in self.metadata.DifficultyLevels():
tag_str = '{}/{}/APH_{}'.format(name, classname, difficulty)
aph_value = aph[difficulty][i]
ret.value.add(tag=tag_str, simple_value=aph_value)
image_summaries = self._breakdown_metrics['aph'].GenerateSummaries(name +
'_aph')
for image_summary in image_summaries:
ret.value.extend(image_summary.value)
ap = self._breakdown_metrics['waymo']._average_precisions # pylint:disable=protected-access
aph = self._breakdown_metrics['waymo']._average_precision_headings # pylint:disable=protected-access
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
for i, j in enumerate(self.metadata.EvalClassIndices()):
classname = self.metadata.ClassNames()[j]
for breakdown_name in breakdown_names[1:]:
# Skip adding entries for breakdowns that are in a different class.
if classname.lower() not in breakdown_name.lower():
continue
# Use a different suffix to avoid polluting the main metrics tab.
tag_str = '{}_extra/AP_{}'.format(name, breakdown_name)
ap_value = ap[breakdown_name][i]
ret.value.add(tag=tag_str, simple_value=ap_value)
tag_str = '{}_extra/APH_{}'.format(name, breakdown_name)
aph_value = aph[breakdown_name][i]
ret.value.add(tag=tag_str, simple_value=aph_value)
return ret
def __init__(self, config=None):
"""Constructs WOD detection evaluation class.
Args:
config: The metrics config defined in protos/metrics.proto.
"""
if config is None:
config = self._get_default_config()
self._config = config
# These are the keys in the metric_dict returned by evaluate.
self._metric_names = [
'average_precision',
'average_precision_ha_weighted',
'precision_recall',
'precision_recall_ha_weighted',
'breakdown',
]
self._breakdown_names = config_util.get_breakdown_names_from_config(
config)
self._required_prediction_fields = [
'prediction_frame_id',
'prediction_bbox',
'prediction_type',
'prediction_score',
'prediction_overlap_nlz',
]
self._required_groundtruth_fields = [
'ground_truth_frame_id',
'ground_truth_bbox',
'ground_truth_type',
'ground_truth_difficulty',
]
self.reset_states()
def get_detection_metric_ops(
config,
prediction_frame_id,
prediction_bbox,
prediction_type,
prediction_score,
prediction_overlap_nlz,
ground_truth_frame_id,
ground_truth_bbox,
ground_truth_type,
ground_truth_difficulty,
ground_truth_speed=None,
recall_at_precision=None,
):
"""Returns dict of metric name to tuples of `(value_op, update_op)`.
Each update_op accumulates the prediction and ground truth tensors to its
corresponding tf variables. Each value_op computes detection metrics on all
prediction and ground truth seen so far. This works similar as `tf.metrics`
code.
Notation:
* M: number of predicted boxes.
* D: number of box dimensions. The number of box dimensions can be one of
the following:
4: Used for boxes with type TYPE_AA_2D (center_x, center_y, length,
width)
5: Used for boxes with type TYPE_2D (center_x, center_y, length,
width, heading).
7: Used for boxes with type TYPE_3D (center_x, center_y, center_z,
length, width, height, heading).
* N: number of ground truth boxes.
Args:
config: The metrics config defined in protos/metrics.proto.
prediction_frame_id: [M] int64 tensor that identifies frame for each
prediction.
prediction_bbox: [M, D] tensor encoding the predicted bounding boxes.
prediction_type: [M] tensor encoding the object type of each prediction.
prediction_score: [M] tensor encoding the score of each prediciton.
prediction_overlap_nlz: [M] tensor encoding whether each prediciton overlaps
with any no label zone.
ground_truth_frame_id: [N] int64 tensor that identifies frame for each
ground truth.
ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes.
ground_truth_type: [N] tensor encoding the object type of each ground truth.
ground_truth_difficulty: [N] tensor encoding the difficulty level of each
ground truth.
ground_truth_speed: [N, 2] tensor with the vx, vy velocity for each object.
recall_at_precision: a float within [0,1]. If set, returns a 3rd metric that
reports the recall at the given precision.
Returns:
A dictionary of metric names to tuple of value_op and update_op.
"""
if ground_truth_speed is None:
num_gt_boxes = tf.shape(ground_truth_bbox)[0]
ground_truth_speed = tf.zeros((num_gt_boxes, 2), tf.float32)
eval_dict = {
'prediction_frame_id': (prediction_frame_id, [0], tf.int64),
'prediction_bbox':
(prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'prediction_type': (prediction_type, [0], tf.uint8),
'prediction_score': (prediction_score, [0], tf.float32),
'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool),
'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64),
'ground_truth_bbox':
(ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'ground_truth_type': (ground_truth_type, [0], tf.uint8),
'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8),
'ground_truth_speed': (ground_truth_speed, [0, 2], tf.float32),
}
variable_and_update_ops = {}
for name, value in eval_dict.items():
update, init_shape, dtype = value
variable_and_update_ops[name] = _update(name, update, init_shape, dtype)
update_ops = [value[1] for value in variable_and_update_ops.values()]
update_op = tf.group(update_ops)
variable_map = {
name: value[0] for name, value in variable_and_update_ops.items()
}
config_str = config.SerializeToString()
ap, aph, pr, _, _ = py_metrics_ops.detection_metrics(
config=config_str, **variable_map)
breakdown_names = config_util.get_breakdown_names_from_config(config)
metric_ops = {}
for i, name in enumerate(breakdown_names):
if i == 0:
metric_ops['{}/AP'.format(name)] = (ap[i], update_op)
else:
# Set update_op to be an no-op just in case if anyone runs update_ops in
# multiple session.run()s.
metric_ops['{}/AP'.format(name)] = (ap[i], tf.constant([]))
metric_ops['{}/APH'.format(name)] = (aph[i], tf.constant([]))
if recall_at_precision is not None:
precision_i_mask = pr[i, :, 0] > recall_at_precision
recall_i = tf.reduce_max(
tf.where(precision_i_mask, pr[i, :, 1], tf.zeros_like(pr[i, :, 1])))
metric_ops['{}/Recall@{}'.format(name,
recall_at_precision)] = (recall_i,
tf.constant([]))
return metric_ops
def _BuildMetric(self, feed_data, classid):
"""Construct tensors and the feed_dict for Waymo metric op.
Args:
feed_data: a NestedMap returned by _GetData().
classid: integer.
Returns:
A tuple of 3 dicts:
- scalar_metrics: a dict mapping all the metric names to fetch tensors.
- curves: a dict mapping all the curve names to fetch tensors.
- feed_dict: a dict mapping the tensors in feed_tensors to feed values.
"""
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
if feed_data is None:
dummy_scalar = tf.constant(np.nan)
dummy_curve = tf.zeros(
[self.metadata.NumberOfPrecisionRecallPoints(), 2], tf.float32)
scalar_metrics = {
'ap': dummy_scalar,
'ap_ha_weighted': dummy_scalar
}
curve_metrics = {'pr': dummy_curve, 'pr_ha_weighted': dummy_curve}
for i, metric in enumerate(breakdown_names):
scalar_metrics['ap_%s' % metric] = dummy_scalar
scalar_metrics['ap_ha_weighted_%s' % metric] = dummy_scalar
curve_metrics['pr_%s' % metric] = dummy_curve
curve_metrics['pr_ha_weighted_%s' % metric] = dummy_curve
return py_utils.NestedMap(feed_dict={},
scalar_metrics=scalar_metrics,
curve_metrics=curve_metrics)
feed_dict = {}
f_gt_bbox = tf.placeholder(tf.float32)
feed_dict[f_gt_bbox] = feed_data.gt.bbox
f_gt_imgid = tf.placeholder(tf.int32)
feed_dict[f_gt_imgid] = feed_data.gt.imgid
f_gt_speed = tf.placeholder(tf.float32)
feed_dict[f_gt_speed] = feed_data.gt.speed
f_pd_bbox = tf.placeholder(tf.float32)
feed_dict[f_pd_bbox] = feed_data.pd.bbox
f_pd_imgid = tf.placeholder(tf.int32)
feed_dict[f_pd_imgid] = feed_data.pd.imgid
f_pd_score = tf.placeholder(tf.float32)
feed_dict[f_pd_score] = feed_data.pd.score
num_gt_bboxes = feed_data.gt.imgid.shape[0]
num_pd_bboxes = feed_data.pd.imgid.shape[0]
gt_class_ids = tf.constant(classid,
dtype=tf.uint8,
shape=[num_gt_bboxes])
pd_class_ids = tf.constant(classid,
dtype=tf.uint8,
shape=[num_pd_bboxes])
ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics(
prediction_bbox=f_pd_bbox,
prediction_type=pd_class_ids,
prediction_score=f_pd_score,
prediction_frame_id=tf.cast(f_pd_imgid, tf.int64),
prediction_overlap_nlz=tf.zeros_like(f_pd_imgid, dtype=tf.bool),
ground_truth_bbox=f_gt_bbox,
ground_truth_type=gt_class_ids,
ground_truth_frame_id=tf.cast(f_gt_imgid, tf.int64),
ground_truth_difficulty=tf.zeros_like(f_gt_imgid, dtype=tf.uint8),
ground_truth_speed=f_gt_speed,
config=self._waymo_metric_config.SerializeToString())
# All tensors returned by Waymo's metric op have a leading dimension
# B=number of breakdowns. At this moment we always use B=1 to make
# it compatible to the python code.
scalar_metrics = {'ap': ap[0], 'ap_ha_weighted': ap_ha[0]}
curve_metrics = {'pr': pr[0], 'pr_ha_weighted': pr_ha[0]}
for i, metric in enumerate(breakdown_names):
# There is a scalar / curve for every breakdown.
scalar_metrics['ap_%s' % metric] = ap[i]
scalar_metrics['ap_ha_weighted_%s' % metric] = ap_ha[i]
curve_metrics['pr_%s' % metric] = pr[i]
curve_metrics['pr_ha_weighted_%s' % metric] = pr_ha[i]
return py_utils.NestedMap(feed_dict=feed_dict,
scalar_metrics=scalar_metrics,
curve_metrics=curve_metrics)
def get_tracking_metric_ops(config,
prediction_bbox,
prediction_type,
prediction_score,
prediction_frame_id,
prediction_sequence_id,
prediction_object_id,
ground_truth_bbox,
ground_truth_type,
ground_truth_frame_id,
ground_truth_sequence_id,
ground_truth_object_id,
ground_truth_difficulty,
prediction_overlap_nlz=None,
ground_truth_speed=None):
"""Returns dict of metric name to tuples of `(value_op, update_op)`.
Each update_op accumulates the prediction and ground truth tensors to its
corresponding tf variables. Each value_op computes tracking metrics on all
prediction and ground truth seen so far. This works similar as `tf.metrics`
code.
Notation:
* M: number of predicted boxes.
* D: number of box dimensions. The number of box dimensions can be one of
the following:
4: Used for boxes with type TYPE_AA_2D (center_x, center_y, length,
width)
5: Used for boxes with type TYPE_2D (center_x, center_y, length,
width, heading).
7: Used for boxes with type TYPE_3D (center_x, center_y, center_z,
length, width, height, heading).
* N: number of ground truth boxes.
Args:
config: The metrics config defined in protos/metrics.proto.
prediction_bbox: [M, D] tensor encoding the predicted bounding boxes.
prediction_type: [M] tensor encoding the object type of each prediction.
prediction_score: [M] tensor encoding the score of each prediction.
prediction_frame_id: [M] int64 tensor that identifies frame for each
prediction.
prediction_sequence_id: [M] int64 tensor that identifies sequence for each
prediction.
prediction_object_id: [M] int64 tensor that identifies object for each
prediction.
ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes.
ground_truth_type: [N] tensor encoding the object type of each ground truth.
ground_truth_frame_id: [N] int64 tensor that identifies frame for each
ground truth.
ground_truth_sequence_id: [M] int64 tensor that identifies sequence for each
ground truth.
ground_truth_object_id: [M] int64 tensor that identifies object for each
ground truth.
ground_truth_difficulty: [N] tensor encoding the difficulty level of each
ground truth.
prediction_overlap_nlz: [M] tensor encoding whether each prediction overlaps
with any no label zone.
ground_truth_speed: [N, 2] tensor with the vx, vy velocity for each object.
Returns:
A dictionary of metric names to tuple of value_op and update_op.
"""
if ground_truth_speed is None:
num_gt_boxes = tf.shape(ground_truth_bbox)[0]
ground_truth_speed = tf.zeros((num_gt_boxes, 2), tf.float32)
if prediction_overlap_nlz is None:
prediction_overlap_nlz = tf.zeros_like(prediction_frame_id,
dtype=tf.bool)
eval_dict = {
'prediction_bbox':
(prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'prediction_type': (prediction_type, [0], tf.uint8),
'prediction_score': (prediction_score, [0], tf.float32),
'prediction_frame_id': (prediction_frame_id, [0], tf.int64),
'prediction_sequence_id': (prediction_sequence_id, [0], tf.string),
'prediction_object_id': (prediction_object_id, [0], tf.int64),
'ground_truth_bbox':
(ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'ground_truth_type': (ground_truth_type, [0], tf.uint8),
'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64),
'ground_truth_sequence_id': (ground_truth_sequence_id, [0], tf.string),
'ground_truth_object_id': (ground_truth_object_id, [0], tf.int64),
'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8),
'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool),
'ground_truth_speed': (ground_truth_speed, [0, 2], tf.float32),
}
variable_and_update_ops = {
name: _update(name, update, init_shape, dtype)
for name, (update, init_shape, dtype) in eval_dict.items()
}
update_ops = [value[1] for value in variable_and_update_ops.values()]
update_op = tf.group(update_ops)
variable_map = {
name: value[0]
for name, value in variable_and_update_ops.items()
}
config_str = config.SerializeToString()
mota, motp, miss, mismatch, fp, _, _ = py_metrics_ops.tracking_metrics(
config=config_str, **variable_map)
breakdown_names = config_util.get_breakdown_names_from_config(config)
metric_ops = {}
for i, name in enumerate(breakdown_names):
if i == 0:
metric_ops['{}/MOTA'.format(name)] = (mota[i], update_op)
else:
# Set update_op to be an no-op just in case if anyone runs update_ops in
# multiple session.run()s.
metric_ops['{}/MOTA'.format(name)] = (mota[i], tf.constant([]))
metric_ops['{}/MOTP'.format(name)] = (motp[i], tf.constant([]))
metric_ops['{}/MISS'.format(name)] = (miss[i], tf.constant([]))
metric_ops['{}/MISMATCH'.format(name)] = (mismatch[i], tf.constant([]))
metric_ops['{}/FP'.format(name)] = (fp[i], tf.constant([]))
return metric_ops
def get_detection_metric_ops(
config,
prediction_frame_id,
prediction_bbox,
prediction_type,
prediction_score,
prediction_overlap_nlz,
ground_truth_frame_id,
ground_truth_bbox,
ground_truth_type,
ground_truth_difficulty,
):
"""Returns dict of metric name to tuples of `(value_op, update_op)`.
Each update_op accumulates the prediction and ground truth tensors to its
corresponding tf variables. Each value_op computes detection metrics on all
prediction and groud truth seen so far. This works similar as `tf.metrics`
code.
Notation:
* M: number of predicted boxes.
* D: number of box dimensions (4, 5 or 7).
* N: number of ground truth boxes.
Args:
prediction_frame_id: [M] int64 tensor that identifies frame for each
prediction.
prediction_bbox: [M, D] tensor encoding the predicted bounding boxes.
prediction_type: [M] tensor encoding the object type of each prediction.
prediction_score: [M] tensor encoding the score of each prediciton.
prediction_overlap_nlz: [M] tensor encoding whether each prediciton overlaps
with any no label zone.
ground_truth_frame_id: [N] int64 tensor that identifies frame for each
ground truth.
ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes.
ground_truth_type: [N] tensor encoding the object type of each ground truth.
ground_truth_difficulty: [N] tensor encoding the difficulty level of each
ground truth.
Returns:
A dictionary of metric names to tuple of value_op and update_op.
"""
eval_dict = {
'prediction_frame_id': (prediction_frame_id, [0], tf.int64),
'prediction_bbox':
(prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'prediction_type': (prediction_type, [0], tf.uint8),
'prediction_score': (prediction_score, [0], tf.float32),
'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool),
'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64),
'ground_truth_bbox':
(ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'ground_truth_type': (ground_truth_type, [0], tf.uint8),
'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8),
}
variable_and_update_ops = {}
for name, value in eval_dict.items():
update, init_shape, dtype = value
variable_and_update_ops[name] = _update(name, update, init_shape,
dtype)
update_ops = [value[1] for value in variable_and_update_ops.values()]
update_op = tf.group(update_ops)
variable_map = {
name: value[0]
for name, value in variable_and_update_ops.items()
}
config_str = config.SerializeToString()
ap, aph, _, _, _ = py_metrics_ops.detection_metrics(config=config_str,
**variable_map)
breakdown_names = config_util.get_breakdown_names_from_config(config)
metric_ops = {}
for i, name in enumerate(breakdown_names):
if i == 0:
metric_ops['{}/AP'.format(name)] = (ap[i], update_op)
else:
# Set update_op to be an no-op just in case if anyone runs update_ops in
# multiple session.run()s.
metric_ops['{}/AP'.format(name)] = (ap[i], tf.constant([]))
metric_ops['{}/APH'.format(name)] = (aph[i], tf.constant([]))
return metric_ops
