def _BuildGraph(self, graph):
with graph.as_default():
self._pd_frame_id = tf.compat.v1.placeholder(dtype=tf.int64)
self._pd_bbox = tf.compat.v1.placeholder(dtype=tf.float32)
self._pd_type = tf.compat.v1.placeholder(dtype=tf.uint8)
self._pd_score = tf.compat.v1.placeholder(dtype=tf.float32)
self._gt_frame_id = tf.compat.v1.placeholder(dtype=tf.int64)
self._gt_bbox = tf.compat.v1.placeholder(dtype=tf.float32)
self._gt_type = tf.compat.v1.placeholder(dtype=tf.uint8)
self._gt_difficulty = tf.compat.v1.placeholder(dtype=tf.uint8)
metrics = detection_metrics.get_detection_metric_ops(
config=self._BuildConfig(),
prediction_frame_id=self._pd_frame_id,
prediction_bbox=self._pd_bbox,
prediction_type=self._pd_type,
prediction_score=self._pd_score,
prediction_overlap_nlz=tf.zeros_like(self._pd_frame_id,
dtype=tf.bool),
ground_truth_bbox=self._gt_bbox,
ground_truth_type=self._gt_type,
ground_truth_frame_id=self._gt_frame_id,
# ground_truth_difficulty=tf.ones_like(self._gt_frame_id, dtype=tf.uint8),
ground_truth_difficulty=self._gt_difficulty,
recall_at_precision=0.95,
)
return metrics
def build_graph(self, graph):
with graph.as_default():
self._pd_frame_id = tf.compat.v1.placeholder(dtype=tf.int64)
self._pd_bbox = tf.compat.v1.placeholder(dtype=tf.float32)
self._pd_type = tf.compat.v1.placeholder(dtype=tf.uint8)
self._pd_score = tf.compat.v1.placeholder(dtype=tf.float32)
self._pd_overlap_nlz = tf.compat.v1.placeholder(dtype=tf.bool)
self._gt_frame_id = tf.compat.v1.placeholder(dtype=tf.int64)
self._gt_bbox = tf.compat.v1.placeholder(dtype=tf.float32)
self._gt_type = tf.compat.v1.placeholder(dtype=tf.uint8)
self._gt_difficulty = tf.compat.v1.placeholder(dtype=tf.uint8)
metrics = detection_metrics.get_detection_metric_ops(
config=self.build_config(),
prediction_frame_id=self._pd_frame_id,
prediction_bbox=self._pd_bbox,
prediction_type=self._pd_type,
prediction_score=self._pd_score,
prediction_overlap_nlz=self._pd_overlap_nlz,
ground_truth_bbox=self._gt_bbox,
ground_truth_type=self._gt_type,
ground_truth_frame_id=self._gt_frame_id,
ground_truth_difficulty=self._gt_difficulty,
)
return metrics
def cal_mAP(prediction_file,
gt_file,
ignore_box_by_point_num=-1,
fov_only=False,
fov=[-115.2, 115.2]):
import tensorflow as tf
from google.protobuf import text_format
from waymo_open_dataset import label_pb2
from waymo_open_dataset import dataset_pb2
from waymo_open_dataset.protos import metrics_pb2
from waymo_open_dataset.metrics.python import detection_metrics
streams = open(prediction_file, 'rb').read()
objects = metrics_pb2.Objects()
objects.ParseFromString(bytearray(streams))
pred_objects = objects
streams = open(gt_file, 'rb').read()
objects = metrics_pb2.Objects()
objects.ParseFromString(bytearray(streams))
gt_objects = objects
no_label_zone_objects = gt_objects.no_label_zone_objects
def set_config():
config = metrics_pb2.Config()
config_text = """
num_desired_score_cutoffs: 11
breakdown_generator_ids: OBJECT_TYPE
difficulties {
}
matcher_type: TYPE_HUNGARIAN
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
iou_thresholds: 0.5
box_type: TYPE_3D
"""
text_format.Merge(config_text, config)
return config
def check_in_no_label_zone():
return False
def filter_by_camera_fov(center_x, center_y):
assert isinstance(fov, list)
fov_neg_angle, fov_pos_angle = fov[0], fov[1]
fov_neg_cos, fov_pos_cos = np.cos(
fov_neg_angle / 180 * np.pi), np.cos(fov_pos_angle / 180 *
np.pi)
x, y = center_x, center_y
fov_flag = np.logical_or(
(x / np.linalg.norm(
(x, y), ord=2, axis=0) > fov_neg_cos) & (y < 0),
(x / np.linalg.norm(
(x, y), ord=2, axis=0) > fov_pos_cos) & (y > 0))
return fov_flag
config = set_config()
frame_ids_dict = {} # key:frame_name value:frame_id
prediction_frame_id = []
prediction_bbox = []
prediction_type = []
prediction_score = []
prediction_overlap_nlz = []
for _object in pred_objects.objects:
context_name = _object.context_name
frame_timestamp_micros = _object.frame_timestamp_micros
label = _object.object
score = _object.score
frame_id = '%s_%s' % (context_name, frame_timestamp_micros)
if frame_id not in frame_ids_dict:
frame_ids_dict[frame_id] = len(frame_ids_dict) + 1
prediction_frame_id.append(frame_ids_dict[frame_id])
prediction_bbox.append([
label.box.center_x, label.box.center_y, label.box.center_z,
label.box.length, label.box.width, label.box.height,
label.box.heading
])
prediction_type.append(label.type)
prediction_score.append(score)
prediction_overlap_nlz.append(check_in_no_label_zone())
ground_truth_frame_id = []
ground_truth_bbox = []
ground_truth_type = []
ground_truth_difficulty = []
for _object in gt_objects.objects:
context_name = _object.context_name
frame_timestamp_micros = _object.frame_timestamp_micros
label = _object.object
frame_id = '%s_%s' % (context_name, frame_timestamp_micros)
num_lidar_points_in_box = label.num_lidar_points_in_box
# detection_difficulty_level = _object.detection_difficulty_level
detection_difficulty_level = 0
if frame_id not in frame_ids_dict:
continue
if num_lidar_points_in_box < ignore_box_by_point_num:
continue
in_fov = True
if fov_only:
in_fov = filter_by_camera_fov(center_x=label.box.center_x,
center_y=label.box.center_y)
if not in_fov:
continue
ground_truth_frame_id.append(frame_ids_dict[frame_id])
ground_truth_bbox.append([
label.box.center_x, label.box.center_y, label.box.center_z,
label.box.length, label.box.width, label.box.height,
label.box.heading
])
ground_truth_type.append(label.type)
ground_truth_difficulty.append(detection_difficulty_level)
prediction_frame_id = tf.convert_to_tensor(prediction_frame_id,
dtype=tf.int64)
prediction_bbox = tf.convert_to_tensor(prediction_bbox,
dtype=tf.float32)
prediction_type = tf.convert_to_tensor(prediction_type, dtype=tf.int64)
prediction_score = tf.convert_to_tensor(prediction_score,
dtype=tf.float32)
prediction_overlap_nlz = tf.convert_to_tensor(prediction_overlap_nlz,
dtype=tf.bool)
ground_truth_frame_id = tf.convert_to_tensor(ground_truth_frame_id,
dtype=tf.int64)
ground_truth_bbox = tf.convert_to_tensor(ground_truth_bbox,
dtype=tf.float32)
ground_truth_type = tf.convert_to_tensor(ground_truth_type,
dtype=tf.int64)
ground_truth_difficulty = tf.convert_to_tensor(ground_truth_difficulty,
dtype=tf.int64)
ground_truth_speed = None
recall_at_precision = None
results = \
detection_metrics.get_detection_metric_ops(
config,
prediction_frame_id,
prediction_bbox,
prediction_type,
prediction_score,
prediction_overlap_nlz,
ground_truth_frame_id,
ground_truth_bbox,
ground_truth_type,
ground_truth_difficulty,
ground_truth_speed,
recall_at_precision,
)
return results