def _GetAP(self,
pd_bbox,
pd_types,
pd_frameid,
pd_score,
gt_bbox,
gt_types,
gt_frameid,
gt_speed,
additional_config_str=''):
""" Calls detection metrics op to compute detection metrics. """
g = tf.Graph()
with g.as_default():
ap, aph, pr, prh, breakdown = py_metrics_ops.detection_metrics(
prediction_bbox=pd_bbox,
prediction_type=pd_types,
prediction_score=pd_score,
prediction_frame_id=pd_frameid,
prediction_overlap_nlz=tf.zeros_like(pd_frameid,
dtype=tf.bool),
ground_truth_bbox=gt_bbox,
ground_truth_type=gt_types,
ground_truth_frame_id=gt_frameid,
ground_truth_difficulty=tf.ones_like(gt_frameid,
dtype=tf.uint8),
ground_truth_speed=gt_speed,
config=self._BuildConfig(
additional_config_str).SerializeToString())
with self.test_session(graph=g) as sess:
val = sess.run([ap, aph, pr, prh, breakdown])
return val
def build_waymo_metric(pred_bbox,
pred_class_id,
pred_class_score,
pred_frame_id,
gt_bbox,
gt_class_id,
gt_frame_id,
gt_speed,
box_type='3d',
breakdowns=None):
"""Build waymo evaluation metric."""
metadata = waymo_metadata.WaymoMetadata()
if breakdowns is None:
breakdowns = ['RANGE']
waymo_metric_config = _build_waymo_metric_config(metadata, box_type,
breakdowns)
ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics(
prediction_bbox=pred_bbox,
prediction_type=pred_class_id,
prediction_score=pred_class_score,
prediction_frame_id=tf.cast(pred_frame_id, tf.int64),
prediction_overlap_nlz=tf.zeros_like(pred_frame_id, dtype=tf.bool),
ground_truth_bbox=gt_bbox,
ground_truth_type=gt_class_id,
ground_truth_frame_id=tf.cast(gt_frame_id, tf.int64),
ground_truth_difficulty=tf.zeros_like(gt_frame_id, dtype=tf.uint8),
ground_truth_speed=gt_speed,
config=waymo_metric_config.SerializeToString())
# All tensors returned by Waymo's metric op have a leading dimension
# B=number of breakdowns. At this moment we always use B=1 to make
# it compatible to the python code.
scalar_metrics = {
'%s_ap' % box_type: ap[0],
'%s_ap_ha_weighted' % box_type: ap_ha[0]
}
curve_metrics = {
'%s_pr' % box_type: pr[0],
'%s_pr_ha_weighted' % box_type: pr_ha[0]
}
breakdown_names = config_util.get_breakdown_names_from_config(
waymo_metric_config)
for i, metric in enumerate(breakdown_names):
# There is a scalar / curve for every breakdown.
scalar_metrics['%s_ap_%s' % (box_type, metric)] = ap[i]
scalar_metrics['%s_ap_ha_weighted_%s' % (box_type, metric)] = ap_ha[i]
curve_metrics['%s_pr_%s' % (box_type, metric)] = pr[i]
curve_metrics['%s_pr_ha_weighted_%s' % (box_type, metric)] = pr_ha[i]
return scalar_metrics, curve_metrics
def evaluate(self):
"""Evaluates with detections from all images with WOD API.
Returns:
metric_dict: dictionary to float numpy array representing the wod
evaluation metrics. Keys in metric dictionary:
- average_precision
- average_precision_ha_weighted
- precision_recall
- precision_recall_ha_weighted
- breakdown
"""
metric_dict = py_metrics_ops.detection_metrics(
prediction_bbox=tf.concat(self._predictions['prediction_bbox'],
axis=0),
prediction_type=tf.concat(self._predictions['prediction_type'],
axis=0),
prediction_score=tf.concat(self._predictions['prediction_score'],
axis=0),
prediction_frame_id=tf.concat(
self._predictions['prediction_frame_id'], axis=0),
prediction_overlap_nlz=tf.concat(
self._predictions['prediction_overlap_nlz'], axis=0),
ground_truth_bbox=tf.concat(
self._groundtruths['ground_truth_bbox'], axis=0),
ground_truth_type=tf.concat(
self._groundtruths['ground_truth_type'], axis=0),
ground_truth_frame_id=tf.concat(
self._groundtruths['ground_truth_frame_id'], axis=0),
ground_truth_difficulty=tf.concat(
self._groundtruths['ground_truth_difficulty'], axis=0),
config=self._config.SerializeToString(),
ground_truth_speed=(
tf.concat(self._groundtruths['ground_truth_speed'], axis=0)
if 'ground_truth_speed' in self._groundtruths else None),
)
return metric_dict
def get_detection_metric_ops(
config,
prediction_frame_id,
prediction_bbox,
prediction_type,
prediction_score,
prediction_overlap_nlz,
ground_truth_frame_id,
ground_truth_bbox,
ground_truth_type,
ground_truth_difficulty,
ground_truth_speed=None,
recall_at_precision=None,
):
"""Returns dict of metric name to tuples of `(value_op, update_op)`.
Each update_op accumulates the prediction and ground truth tensors to its
corresponding tf variables. Each value_op computes detection metrics on all
prediction and ground truth seen so far. This works similar as `tf.metrics`
code.
Notation:
* M: number of predicted boxes.
* D: number of box dimensions. The number of box dimensions can be one of
the following:
4: Used for boxes with type TYPE_AA_2D (center_x, center_y, length,
width)
5: Used for boxes with type TYPE_2D (center_x, center_y, length,
width, heading).
7: Used for boxes with type TYPE_3D (center_x, center_y, center_z,
length, width, height, heading).
* N: number of ground truth boxes.
Args:
config: The metrics config defined in protos/metrics.proto.
prediction_frame_id: [M] int64 tensor that identifies frame for each
prediction.
prediction_bbox: [M, D] tensor encoding the predicted bounding boxes.
prediction_type: [M] tensor encoding the object type of each prediction.
prediction_score: [M] tensor encoding the score of each prediciton.
prediction_overlap_nlz: [M] tensor encoding whether each prediciton overlaps
with any no label zone.
ground_truth_frame_id: [N] int64 tensor that identifies frame for each
ground truth.
ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes.
ground_truth_type: [N] tensor encoding the object type of each ground truth.
ground_truth_difficulty: [N] tensor encoding the difficulty level of each
ground truth.
ground_truth_speed: [N, 2] tensor with the vx, vy velocity for each object.
recall_at_precision: a float within [0,1]. If set, returns a 3rd metric that
reports the recall at the given precision.
Returns:
A dictionary of metric names to tuple of value_op and update_op.
"""
if ground_truth_speed is None:
num_gt_boxes = tf.shape(ground_truth_bbox)[0]
ground_truth_speed = tf.zeros((num_gt_boxes, 2), tf.float32)
eval_dict = {
'prediction_frame_id': (prediction_frame_id, [0], tf.int64),
'prediction_bbox':
(prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'prediction_type': (prediction_type, [0], tf.uint8),
'prediction_score': (prediction_score, [0], tf.float32),
'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool),
'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64),
'ground_truth_bbox':
(ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'ground_truth_type': (ground_truth_type, [0], tf.uint8),
'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8),
'ground_truth_speed': (ground_truth_speed, [0, 2], tf.float32),
}
variable_and_update_ops = {}
for name, value in eval_dict.items():
update, init_shape, dtype = value
variable_and_update_ops[name] = _update(name, update, init_shape, dtype)
update_ops = [value[1] for value in variable_and_update_ops.values()]
update_op = tf.group(update_ops)
variable_map = {
name: value[0] for name, value in variable_and_update_ops.items()
}
config_str = config.SerializeToString()
ap, aph, pr, _, _ = py_metrics_ops.detection_metrics(
config=config_str, **variable_map)
breakdown_names = config_util.get_breakdown_names_from_config(config)
metric_ops = {}
for i, name in enumerate(breakdown_names):
if i == 0:
metric_ops['{}/AP'.format(name)] = (ap[i], update_op)
else:
# Set update_op to be an no-op just in case if anyone runs update_ops in
# multiple session.run()s.
metric_ops['{}/AP'.format(name)] = (ap[i], tf.constant([]))
metric_ops['{}/APH'.format(name)] = (aph[i], tf.constant([]))
if recall_at_precision is not None:
precision_i_mask = pr[i, :, 0] > recall_at_precision
recall_i = tf.reduce_max(
tf.where(precision_i_mask, pr[i, :, 1], tf.zeros_like(pr[i, :, 1])))
metric_ops['{}/Recall@{}'.format(name,
recall_at_precision)] = (recall_i,
tf.constant([]))
return metric_ops
def _BuildMetric(self, feed_data, classid):
"""Construct tensors and the feed_dict for Waymo metric op.
Args:
feed_data: a NestedMap returned by _GetData().
classid: integer.
Returns:
A tuple of 3 dicts:
- scalar_metrics: a dict mapping all the metric names to fetch tensors.
- curves: a dict mapping all the curve names to fetch tensors.
- feed_dict: a dict mapping the tensors in feed_tensors to feed values.
"""
breakdown_names = config_util.get_breakdown_names_from_config(
self._waymo_metric_config)
if feed_data is None:
dummy_scalar = tf.constant(np.nan)
dummy_curve = tf.zeros(
[self.metadata.NumberOfPrecisionRecallPoints(), 2], tf.float32)
scalar_metrics = {
'ap': dummy_scalar,
'ap_ha_weighted': dummy_scalar
}
curve_metrics = {'pr': dummy_curve, 'pr_ha_weighted': dummy_curve}
for i, metric in enumerate(breakdown_names):
scalar_metrics['ap_%s' % metric] = dummy_scalar
scalar_metrics['ap_ha_weighted_%s' % metric] = dummy_scalar
curve_metrics['pr_%s' % metric] = dummy_curve
curve_metrics['pr_ha_weighted_%s' % metric] = dummy_curve
return py_utils.NestedMap(feed_dict={},
scalar_metrics=scalar_metrics,
curve_metrics=curve_metrics)
feed_dict = {}
f_gt_bbox = tf.placeholder(tf.float32)
feed_dict[f_gt_bbox] = feed_data.gt.bbox
f_gt_imgid = tf.placeholder(tf.int32)
feed_dict[f_gt_imgid] = feed_data.gt.imgid
f_gt_speed = tf.placeholder(tf.float32)
feed_dict[f_gt_speed] = feed_data.gt.speed
f_pd_bbox = tf.placeholder(tf.float32)
feed_dict[f_pd_bbox] = feed_data.pd.bbox
f_pd_imgid = tf.placeholder(tf.int32)
feed_dict[f_pd_imgid] = feed_data.pd.imgid
f_pd_score = tf.placeholder(tf.float32)
feed_dict[f_pd_score] = feed_data.pd.score
num_gt_bboxes = feed_data.gt.imgid.shape[0]
num_pd_bboxes = feed_data.pd.imgid.shape[0]
gt_class_ids = tf.constant(classid,
dtype=tf.uint8,
shape=[num_gt_bboxes])
pd_class_ids = tf.constant(classid,
dtype=tf.uint8,
shape=[num_pd_bboxes])
ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics(
prediction_bbox=f_pd_bbox,
prediction_type=pd_class_ids,
prediction_score=f_pd_score,
prediction_frame_id=tf.cast(f_pd_imgid, tf.int64),
prediction_overlap_nlz=tf.zeros_like(f_pd_imgid, dtype=tf.bool),
ground_truth_bbox=f_gt_bbox,
ground_truth_type=gt_class_ids,
ground_truth_frame_id=tf.cast(f_gt_imgid, tf.int64),
ground_truth_difficulty=tf.zeros_like(f_gt_imgid, dtype=tf.uint8),
ground_truth_speed=f_gt_speed,
config=self._waymo_metric_config.SerializeToString())
# All tensors returned by Waymo's metric op have a leading dimension
# B=number of breakdowns. At this moment we always use B=1 to make
# it compatible to the python code.
scalar_metrics = {'ap': ap[0], 'ap_ha_weighted': ap_ha[0]}
curve_metrics = {'pr': pr[0], 'pr_ha_weighted': pr_ha[0]}
for i, metric in enumerate(breakdown_names):
# There is a scalar / curve for every breakdown.
scalar_metrics['ap_%s' % metric] = ap[i]
scalar_metrics['ap_ha_weighted_%s' % metric] = ap_ha[i]
curve_metrics['pr_%s' % metric] = pr[i]
curve_metrics['pr_ha_weighted_%s' % metric] = pr_ha[i]
return py_utils.NestedMap(feed_dict=feed_dict,
scalar_metrics=scalar_metrics,
curve_metrics=curve_metrics)
def _BuildMetric(self, feed_data, classid):
"""Construct tensors and the feed_dict for Waymo metric op.
Args:
feed_data: a NestedMap returned by _GetData().
classid: integer.
Returns:
A tuple of 3 dicts:
- scalar_metrics: a dict mapping all the metric names to fetch tensors.
- curves: a dict mapping all the curve names to fetch tensors.
- feed_dict: a dict mapping the tensors in feed_tensors to feed values.
"""
if feed_data is None:
dummy_scalar = tf.constant(np.nan)
dummy_curve = tf.zeros(
[self.metadata.NumberOfPrecisionRecallPoints(), 2], tf.float32)
scalar_metrics = {
'ap': dummy_scalar,
'ap_ha_weighted': dummy_scalar
}
curve_metrics = {'pr': dummy_curve, 'pr_ha_weighted': dummy_curve}
return scalar_metrics, curve_metrics, {}
feed_dict = {}
f_gt_bbox = tf.placeholder(tf.float32)
feed_dict[f_gt_bbox] = feed_data.gt.bbox
f_gt_imgid = tf.placeholder(tf.int32)
feed_dict[f_gt_imgid] = feed_data.gt.imgid
f_pd_bbox = tf.placeholder(tf.float32)
feed_dict[f_pd_bbox] = feed_data.pd.bbox
f_pd_imgid = tf.placeholder(tf.int32)
feed_dict[f_pd_imgid] = feed_data.pd.imgid
f_pd_score = tf.placeholder(tf.float32)
feed_dict[f_pd_score] = feed_data.pd.score
num_gt_bboxes = feed_data.gt.imgid.shape[0]
num_pd_bboxes = feed_data.pd.imgid.shape[0]
gt_class_ids = tf.constant(classid,
dtype=tf.uint8,
shape=[num_gt_bboxes])
pd_class_ids = tf.constant(classid,
dtype=tf.uint8,
shape=[num_pd_bboxes])
ap, ap_ha, pr, pr_ha, _ = py_metrics_ops.detection_metrics(
prediction_bbox=f_pd_bbox,
prediction_type=pd_class_ids,
prediction_score=f_pd_score,
prediction_frame_id=tf.to_int64(f_pd_imgid),
prediction_overlap_nlz=tf.zeros_like(f_pd_imgid, dtype=tf.bool),
ground_truth_bbox=f_gt_bbox,
ground_truth_type=gt_class_ids,
ground_truth_frame_id=tf.to_int64(f_gt_imgid),
ground_truth_difficulty=tf.zeros_like(f_gt_imgid, dtype=tf.uint8),
config=self._waymo_metric_config)
# All tensors returned by Waymo's metric op have a leading dimension
# B=number of breakdowns. At this moment we always use B=1 to make
# it compatible to the python code.
scalar_metrics = {'ap': ap[0], 'ap_ha_weighted': ap_ha[0]}
curve_metrics = {'pr': pr[0], 'pr_ha_weighted': pr_ha[0]}
return scalar_metrics, curve_metrics, feed_dict
def get_detection_metric_ops(
config,
prediction_frame_id,
prediction_bbox,
prediction_type,
prediction_score,
prediction_overlap_nlz,
ground_truth_frame_id,
ground_truth_bbox,
ground_truth_type,
ground_truth_difficulty,
):
"""Returns dict of metric name to tuples of `(value_op, update_op)`.
Each update_op accumulates the prediction and ground truth tensors to its
corresponding tf variables. Each value_op computes detection metrics on all
prediction and groud truth seen so far. This works similar as `tf.metrics`
code.
Notation:
* M: number of predicted boxes.
* D: number of box dimensions (4, 5 or 7).
* N: number of ground truth boxes.
Args:
prediction_frame_id: [M] int64 tensor that identifies frame for each
prediction.
prediction_bbox: [M, D] tensor encoding the predicted bounding boxes.
prediction_type: [M] tensor encoding the object type of each prediction.
prediction_score: [M] tensor encoding the score of each prediciton.
prediction_overlap_nlz: [M] tensor encoding whether each prediciton overlaps
with any no label zone.
ground_truth_frame_id: [N] int64 tensor that identifies frame for each
ground truth.
ground_truth_bbox: [N, D] tensor encoding the ground truth bounding boxes.
ground_truth_type: [N] tensor encoding the object type of each ground truth.
ground_truth_difficulty: [N] tensor encoding the difficulty level of each
ground truth.
Returns:
A dictionary of metric names to tuple of value_op and update_op.
"""
eval_dict = {
'prediction_frame_id': (prediction_frame_id, [0], tf.int64),
'prediction_bbox':
(prediction_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'prediction_type': (prediction_type, [0], tf.uint8),
'prediction_score': (prediction_score, [0], tf.float32),
'prediction_overlap_nlz': (prediction_overlap_nlz, [0], tf.bool),
'ground_truth_frame_id': (ground_truth_frame_id, [0], tf.int64),
'ground_truth_bbox':
(ground_truth_bbox, [0, _get_box_dof(config.box_type)], tf.float32),
'ground_truth_type': (ground_truth_type, [0], tf.uint8),
'ground_truth_difficulty': (ground_truth_difficulty, [0], tf.uint8),
}
variable_and_update_ops = {}
for name, value in eval_dict.items():
update, init_shape, dtype = value
variable_and_update_ops[name] = _update(name, update, init_shape,
dtype)
update_ops = [value[1] for value in variable_and_update_ops.values()]
update_op = tf.group(update_ops)
variable_map = {
name: value[0]
for name, value in variable_and_update_ops.items()
}
config_str = config.SerializeToString()
ap, aph, _, _, _ = py_metrics_ops.detection_metrics(config=config_str,
**variable_map)
breakdown_names = config_util.get_breakdown_names_from_config(config)
metric_ops = {}
for i, name in enumerate(breakdown_names):
if i == 0:
metric_ops['{}/AP'.format(name)] = (ap[i], update_op)
else:
# Set update_op to be an no-op just in case if anyone runs update_ops in
# multiple session.run()s.
metric_ops['{}/AP'.format(name)] = (ap[i], tf.constant([]))
metric_ops['{}/APH'.format(name)] = (aph[i], tf.constant([]))
return metric_ops
