def get_max_ious_3d(all_gt_boxes_3d, pred_boxes_3d): """Helper function to calculate 3D IoU for the given predictions. Args: all_gt_boxes_3d: A list of the same ground-truth boxes in box_3d format. pred_boxes_3d: A list of predictions in box_3d format. """ # Only calculate ious if there are predictions if pred_boxes_3d: # Convert to iou format gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( all_gt_boxes_3d) pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( pred_boxes_3d) max_ious_3d = np.zeros(len(all_gt_boxes_3d)) for gt_obj_idx in range(len(all_gt_boxes_3d)): gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx] ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt, pred_objs_iou_fmt) max_ious_3d[gt_obj_idx] = np.amax(ious_3d) else: # No detections, all ious = 0 max_ious_3d = np.zeros(len(all_gt_boxes_3d)) return max_ious_3d
def draw_prediction_info(ax, x, y, pred_obj, pred_class_idx, pred_box, ground_truth, draw_score, draw_iou, gt_classes, iou_3d): label = "" if draw_score: label += "{:.2f}".format(pred_obj.score) if draw_iou and len(ground_truth) > 0: if draw_score: label += ', ' if iou_3d: iou = evaluation.three_d_iou(pred_box, ground_truth) if len(iou.shape) == 0: iou = np.array([iou]) else: iou = evaluation.two_d_iou(pred_box, ground_truth) label += "{:.3f}".format(max(iou)) box_cls = gt_classes[int(pred_class_idx)] ax.text( x, y - 4, gt_classes[int(pred_class_idx)] + '\n' + label, verticalalignment='bottom', horizontalalignment='center', color=BOX_COLOUR_SCHEME[box_cls], fontsize=15, fontweight='bold', path_effects=[patheffects.withStroke(linewidth=2, foreground='black')])
def get_iou3d_matches(ego_objs, objs_perspectives, iou_thresh): all_3d_ious = [] if len(ego_objs) > 0 and \ objs_perspectives is not None and len(objs_perspectives) > 0: ego_objs_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(ego_obj) for ego_obj in ego_objs ] perspect_objs_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(objs_perspective) for objs_perspective in objs_perspectives ] # Convert to iou format ego_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( ego_objs_boxes_3d) perspect_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( perspect_objs_boxes_3d) max_ious_3d = np.zeros(len(objs_perspectives)) max_iou_pred_indices = -np.ones(len(objs_perspectives)) for det_idx in range(len(objs_perspectives)): perspect_obj_iou_fmt = perspect_objs_iou_fmt[det_idx] ious_3d = evaluation.three_d_iou(perspect_obj_iou_fmt, ego_objs_iou_fmt) max_iou_3d = np.amax(ious_3d) max_ious_3d[det_idx] = max_iou_3d if max_iou_3d > iou_thresh: max_iou_pred_indices[det_idx] = np.argmax(ious_3d) return max_ious_3d, max_iou_pred_indices
def _calculate_anchors_info(self, all_anchor_boxes_3d, empty_anchor_filter, gt_labels): """Calculates the list of anchor information in the format: N x 8 [max_gt_2d_iou, max_gt_3d_iou, (6 x offsets), class_index] max_gt_out - highest 3D iou with any ground truth box offsets - encoded offsets [dx, dy, dz, d_dimx, d_dimy, d_dimz] class_index - the anchor's class as an index (e.g. 0 or 1, for "Background" or "Car") Args: all_anchor_boxes_3d: list of anchors in box_3d format N x [x, y, z, l, w, h, ry] empty_anchor_filter: boolean mask of which anchors are non empty gt_labels: list of Object Label data format containing ground truth labels to generate positives/negatives from. Returns: list of anchor info """ # Check for ground truth objects if len(gt_labels) == 0: raise Warning("No valid ground truth label to generate anchors.") kitti_utils = self._dataset.kitti_utils # Filter empty anchors anchor_indices = np.where(empty_anchor_filter)[0] anchor_boxes_3d = all_anchor_boxes_3d[empty_anchor_filter] # Convert anchor_boxes_3d to anchor format anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) # Convert gt to boxes_3d -> anchors -> iou format gt_boxes_3d = np.asarray([ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in gt_labels ]) gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d, ortho_rotate=True) rpn_iou_type = self.mini_batch_utils.rpn_iou_type if rpn_iou_type == '2d': # Convert anchors to 2d iou format anchors_for_2d_iou, _ = np.asarray( anchor_projector.project_to_bev(anchors, kitti_utils.bev_extents)) gt_boxes_for_2d_iou, _ = anchor_projector.project_to_bev( gt_anchors, kitti_utils.bev_extents) elif rpn_iou_type == '3d': # Convert anchors to 3d iou format for calculation anchors_for_3d_iou = box_3d_encoder.box_3d_to_3d_iou_format( anchor_boxes_3d) gt_boxes_for_3d_iou = \ box_3d_encoder.box_3d_to_3d_iou_format(gt_boxes_3d) else: raise ValueError('Invalid rpn_iou_type {}', rpn_iou_type) # Initialize sample and offset lists num_anchors = len(anchor_boxes_3d) all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length)) # Update anchor indices all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices # For each of the labels, generate samples for gt_idx in range(len(gt_labels)): gt_obj = gt_labels[gt_idx] gt_box_3d = gt_boxes_3d[gt_idx] # Get 2D or 3D IoU for every anchor if self.mini_batch_utils.rpn_iou_type == '2d': gt_box_for_2d_iou = gt_boxes_for_2d_iou[gt_idx] ious = evaluation.two_d_iou(gt_box_for_2d_iou, anchors_for_2d_iou) elif self.mini_batch_utils.rpn_iou_type == '3d': gt_box_for_3d_iou = gt_boxes_for_3d_iou[gt_idx] ious = evaluation.three_d_iou(gt_box_for_3d_iou, anchors_for_3d_iou) # Only update indices with a higher iou than before update_indices = np.greater( ious, all_info[:, self.mini_batch_utils.col_ious]) # Get ious to update ious_to_update = ious[update_indices] # Calculate offsets, use 3D iou to get highest iou anchors_to_update = anchors[update_indices] gt_anchor = box_3d_encoder.box_3d_to_anchor(gt_box_3d, ortho_rotate=True) offsets = anchor_encoder.anchor_to_offset(anchors_to_update, gt_anchor) # Convert gt type to index class_idx = kitti_utils.class_str_to_index(gt_obj.type) # Update anchors info (indices already updated) # [index, iou, (offsets), class_index] all_info[update_indices, self.mini_batch_utils.col_ious] = ious_to_update all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self. mini_batch_utils.col_offsets_hi] = offsets all_info[update_indices, self.mini_batch_utils.col_class_idx] = class_idx return all_info
def generate_negative_3d_bb(obj_label, boxes3d, iou_threshold_min, iou_threshold_max, samples): """Generates negative 3D bounding boxes. This is the 3D version of generate_negative_3d_bb. Keyword arguments: obj_label: single label for a detected 3D object boxes3d: a list of numpy array representing all 3D bounding boxes in the image iou_threshold_min: determines the min variation between the original iou and the negative samples iou_threshold_max: determines the max variation between the original iou and the negative samples samples: number of negative samples per detected object Returns: a list of generated ObjectLabels """ box_corners = od.compute_box_corners_3d(obj_label) # make sure this is not empty assert (len(box_corners) > 0) P1 = box_corners[:, 0] P2 = box_corners[:, 1] P4 = box_corners[:, 3] current_samples = 0 new_objects = [] while current_samples < samples: # Generate random 3D point inside the box # we keep the same y, only generate x and z new_xp = float(np.random.uniform(P1[0], P4[0], 1)) new_zp = float(np.random.uniform(P1[2], P2[2], 1)) # create a new ObjectLabel new_obj = copy.copy(obj_label) # update the new obj.t # everything else is the same, only changing the # centroid point t which remains the same along # the y direction new_obj.t = (new_xp, obj_label.t[1], new_zp) _, box_to_test, _ = od.build_bbs_from_objects([new_obj], 'All') assert (len(box_to_test) == 1) # we are dealing with one box here so its the first element iou_3d = evaluation.three_d_iou(box_to_test[0], boxes3d) # check if iou_3d is a list, take the largest # this compares to all the boxes if isinstance(iou_3d, np.ndarray): iou_max = np.max(iou_3d) else: iou_max = iou_3d if iou_threshold_min < iou_max < iou_threshold_max: # keep the new object label new_objects.append(new_obj) current_samples += 1 return new_objects
def test_3d_iou(self): iou = evaluation.three_d_iou(self.gtbb_3d, self.test_cases_3d) np.testing.assert_almost_equal(self.gt_iou_3d, iou, 3) iou = evaluation.three_d_iou(self.gtbb_3d, self.test_cases_3d[4]) np.testing.assert_almost_equal(self.gt_iou_3d[4], iou, 3)
def main(): """Plots detection errors for xyz, lwh, ry, and shows 3D IoU with ground truth boxes """ dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL_HALF, use_defaults=True) difficulty = 2 # Full path to kitti predictions # (e.g. '.../data/outputs/mlod_exp_example/predictions/' # 'kitti_predictions_3d/val/0.1/100000/data' predictions_data_path = 'path_to_detections/data' # Loop through split and save ious and errors all_3d_ious = [] all_errors = [] for sample_idx in range(dataset.num_samples): sys.stdout.write('\r{} / {}'.format(sample_idx + 1, dataset.num_samples)) sample_name = dataset.sample_names[sample_idx] img_idx = int(sample_name) # Get filtered ground truth all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx) all_gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs, difficulty=difficulty) pred_objs = obj_utils.read_labels(predictions_data_path, img_idx) ############################## # Check IoUs ############################## if len(all_gt_objs) > 0 and \ pred_objs is not None and len(pred_objs) > 0: all_gt_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in all_gt_objs ] pred_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(pred_obj) for pred_obj in pred_objs ] # Convert to iou format gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( all_gt_boxes_3d) pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( pred_boxes_3d) max_ious_3d = np.zeros(len(all_gt_objs)) max_iou_pred_indices = -np.ones(len(all_gt_objs)) for gt_obj_idx in range(len(all_gt_objs)): gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx] ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt, pred_objs_iou_fmt) max_iou_3d = np.amax(ious_3d) max_ious_3d[gt_obj_idx] = max_iou_3d if max_iou_3d > 0.0: max_iou_pred_indices[gt_obj_idx] = np.argmax(ious_3d) for gt_obj_idx in range(len(all_gt_objs)): max_iou_pred_idx = int(max_iou_pred_indices[gt_obj_idx]) if max_iou_pred_idx >= 0: error = all_gt_boxes_3d[gt_obj_idx] - \ pred_boxes_3d[max_iou_pred_idx] all_errors.append(error) all_3d_ious.extend(max_ious_3d) print('Done') all_errors = np.asarray(all_errors) # Plot Data Histograms f, ax_arr = plt.subplots(3, 3) xyzlwh_bins = 51 ry_bins = 31 iou_bins = 51 # xyz ax_arr[0, 0].hist(all_errors[:, 0], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[0, 1].hist(all_errors[:, 1], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[0, 2].hist(all_errors[:, 2], xyzlwh_bins, facecolor='green', alpha=0.75) # lwh ax_arr[1, 0].hist(all_errors[:, 3], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[1, 1].hist(all_errors[:, 4], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[1, 2].hist(all_errors[:, 5], xyzlwh_bins, facecolor='green', alpha=0.75) # orientation ax_arr[2, 0].hist(all_errors[:, 6], ry_bins, facecolor='green', alpha=0.75) # iou ax_arr[2, 2].hist(all_3d_ious, iou_bins, facecolor='green', alpha=0.75) plt.show()