def _BuildGraph(self, graph): with graph.as_default(): self._pd_frame_id = tf.compat.v1.placeholder(dtype=tf.int64) self._pd_bbox = tf.compat.v1.placeholder(dtype=tf.float32) self._pd_type = tf.compat.v1.placeholder(dtype=tf.uint8) self._pd_score = tf.compat.v1.placeholder(dtype=tf.float32) self._gt_frame_id = tf.compat.v1.placeholder(dtype=tf.int64) self._gt_bbox = tf.compat.v1.placeholder(dtype=tf.float32) self._gt_type = tf.compat.v1.placeholder(dtype=tf.uint8) self._gt_difficulty = tf.compat.v1.placeholder(dtype=tf.uint8) metrics = detection_metrics.get_detection_metric_ops( config=self._BuildConfig(), prediction_frame_id=self._pd_frame_id, prediction_bbox=self._pd_bbox, prediction_type=self._pd_type, prediction_score=self._pd_score, prediction_overlap_nlz=tf.zeros_like(self._pd_frame_id, dtype=tf.bool), ground_truth_bbox=self._gt_bbox, ground_truth_type=self._gt_type, ground_truth_frame_id=self._gt_frame_id, # ground_truth_difficulty=tf.ones_like(self._gt_frame_id, dtype=tf.uint8), ground_truth_difficulty=self._gt_difficulty, recall_at_precision=0.95, ) return metrics
def build_graph(self, graph): with graph.as_default(): self._pd_frame_id = tf.compat.v1.placeholder(dtype=tf.int64) self._pd_bbox = tf.compat.v1.placeholder(dtype=tf.float32) self._pd_type = tf.compat.v1.placeholder(dtype=tf.uint8) self._pd_score = tf.compat.v1.placeholder(dtype=tf.float32) self._pd_overlap_nlz = tf.compat.v1.placeholder(dtype=tf.bool) self._gt_frame_id = tf.compat.v1.placeholder(dtype=tf.int64) self._gt_bbox = tf.compat.v1.placeholder(dtype=tf.float32) self._gt_type = tf.compat.v1.placeholder(dtype=tf.uint8) self._gt_difficulty = tf.compat.v1.placeholder(dtype=tf.uint8) metrics = detection_metrics.get_detection_metric_ops( config=self.build_config(), prediction_frame_id=self._pd_frame_id, prediction_bbox=self._pd_bbox, prediction_type=self._pd_type, prediction_score=self._pd_score, prediction_overlap_nlz=self._pd_overlap_nlz, ground_truth_bbox=self._gt_bbox, ground_truth_type=self._gt_type, ground_truth_frame_id=self._gt_frame_id, ground_truth_difficulty=self._gt_difficulty, ) return metrics
def cal_mAP(prediction_file, gt_file, ignore_box_by_point_num=-1, fov_only=False, fov=[-115.2, 115.2]): import tensorflow as tf from google.protobuf import text_format from waymo_open_dataset import label_pb2 from waymo_open_dataset import dataset_pb2 from waymo_open_dataset.protos import metrics_pb2 from waymo_open_dataset.metrics.python import detection_metrics streams = open(prediction_file, 'rb').read() objects = metrics_pb2.Objects() objects.ParseFromString(bytearray(streams)) pred_objects = objects streams = open(gt_file, 'rb').read() objects = metrics_pb2.Objects() objects.ParseFromString(bytearray(streams)) gt_objects = objects no_label_zone_objects = gt_objects.no_label_zone_objects def set_config(): config = metrics_pb2.Config() config_text = """ num_desired_score_cutoffs: 11 breakdown_generator_ids: OBJECT_TYPE difficulties { } matcher_type: TYPE_HUNGARIAN iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 iou_thresholds: 0.5 box_type: TYPE_3D """ text_format.Merge(config_text, config) return config def check_in_no_label_zone(): return False def filter_by_camera_fov(center_x, center_y): assert isinstance(fov, list) fov_neg_angle, fov_pos_angle = fov[0], fov[1] fov_neg_cos, fov_pos_cos = np.cos( fov_neg_angle / 180 * np.pi), np.cos(fov_pos_angle / 180 * np.pi) x, y = center_x, center_y fov_flag = np.logical_or( (x / np.linalg.norm( (x, y), ord=2, axis=0) > fov_neg_cos) & (y < 0), (x / np.linalg.norm( (x, y), ord=2, axis=0) > fov_pos_cos) & (y > 0)) return fov_flag config = set_config() frame_ids_dict = {} # key:frame_name value:frame_id prediction_frame_id = [] prediction_bbox = [] prediction_type = [] prediction_score = [] prediction_overlap_nlz = [] for _object in pred_objects.objects: context_name = _object.context_name frame_timestamp_micros = _object.frame_timestamp_micros label = _object.object score = _object.score frame_id = '%s_%s' % (context_name, frame_timestamp_micros) if frame_id not in frame_ids_dict: frame_ids_dict[frame_id] = len(frame_ids_dict) + 1 prediction_frame_id.append(frame_ids_dict[frame_id]) prediction_bbox.append([ label.box.center_x, label.box.center_y, label.box.center_z, label.box.length, label.box.width, label.box.height, label.box.heading ]) prediction_type.append(label.type) prediction_score.append(score) prediction_overlap_nlz.append(check_in_no_label_zone()) ground_truth_frame_id = [] ground_truth_bbox = [] ground_truth_type = [] ground_truth_difficulty = [] for _object in gt_objects.objects: context_name = _object.context_name frame_timestamp_micros = _object.frame_timestamp_micros label = _object.object frame_id = '%s_%s' % (context_name, frame_timestamp_micros) num_lidar_points_in_box = label.num_lidar_points_in_box # detection_difficulty_level = _object.detection_difficulty_level detection_difficulty_level = 0 if frame_id not in frame_ids_dict: continue if num_lidar_points_in_box < ignore_box_by_point_num: continue in_fov = True if fov_only: in_fov = filter_by_camera_fov(center_x=label.box.center_x, center_y=label.box.center_y) if not in_fov: continue ground_truth_frame_id.append(frame_ids_dict[frame_id]) ground_truth_bbox.append([ label.box.center_x, label.box.center_y, label.box.center_z, label.box.length, label.box.width, label.box.height, label.box.heading ]) ground_truth_type.append(label.type) ground_truth_difficulty.append(detection_difficulty_level) prediction_frame_id = tf.convert_to_tensor(prediction_frame_id, dtype=tf.int64) prediction_bbox = tf.convert_to_tensor(prediction_bbox, dtype=tf.float32) prediction_type = tf.convert_to_tensor(prediction_type, dtype=tf.int64) prediction_score = tf.convert_to_tensor(prediction_score, dtype=tf.float32) prediction_overlap_nlz = tf.convert_to_tensor(prediction_overlap_nlz, dtype=tf.bool) ground_truth_frame_id = tf.convert_to_tensor(ground_truth_frame_id, dtype=tf.int64) ground_truth_bbox = tf.convert_to_tensor(ground_truth_bbox, dtype=tf.float32) ground_truth_type = tf.convert_to_tensor(ground_truth_type, dtype=tf.int64) ground_truth_difficulty = tf.convert_to_tensor(ground_truth_difficulty, dtype=tf.int64) ground_truth_speed = None recall_at_precision = None results = \ detection_metrics.get_detection_metric_ops( config, prediction_frame_id, prediction_bbox, prediction_type, prediction_score, prediction_overlap_nlz, ground_truth_frame_id, ground_truth_bbox, ground_truth_type, ground_truth_difficulty, ground_truth_speed, recall_at_precision, ) return results