def test_2d_iou(self): iou = evaluation.two_d_iou(self.gtbb_2d, self.test_cases_2d) np.testing.assert_allclose(self.gt_iou_2d, iou, 0, 0.01) gt_box = np.asarray([0.0, 0.0, 0.5, 0.5]) test_boxes = np.asarray([ [-0.5, 0.0, 0.0, 0.5], [-0.25, 0.0, 0.25, 0.5], [0.0, 0.0, 0.5, 0.5], [0.25, 0.0, 0.75, 0.5], [0.5, 0.0, 1.0, 0.5], ]) exp_ious_2d = [0.0, 0.333, 1.0, 0.333, 0.0] ious_2d = evaluation.two_d_iou(gt_box, test_boxes) np.testing.assert_almost_equal(ious_2d, exp_ious_2d)
def draw_proposals_info(ax, x, y, proposal_scores, proposal_box_2d, ground_truth, draw_score, draw_iou): label = "" # print('proposal_scores = ', proposal_scores) if draw_score: label += "sc:{:.2f}".format(proposal_scores) # print('proposal_box_2d = ', proposal_box_2d) # print('ground_truth = ', ground_truth) if draw_iou and len(ground_truth) > 0: if draw_score: label += ', ' iou = evaluation.two_d_iou(proposal_box_2d, ground_truth) label += "iou:{:.3f}".format(max(iou)) ax.text( x, y + 150, label, verticalalignment='bottom', horizontalalignment='center', color='cornflowerblue', fontsize=10, fontweight='bold', path_effects=[patheffects.withStroke(linewidth=5, foreground='black')])
def draw_prediction_info(ax, x, y, pred_obj, pred_class_idx, pred_box_2d, ground_truth, draw_score, draw_iou, gt_classes): label = "" if draw_score: label += "sc:{:.2f}".format(pred_obj.score) if draw_iou and len(ground_truth) > 0: if draw_score: label += ', ' iou = evaluation.two_d_iou(pred_box_2d, ground_truth) label += "iou:{:.3f}".format(max(iou)) box_cls = gt_classes[int(pred_class_idx)] ax.text( x, y + 54, gt_classes[int(pred_class_idx)] + '\n' + label, verticalalignment='bottom', horizontalalignment='center', color=BOX_COLOUR_SCHEME[box_cls], fontsize=10, fontweight='bold', path_effects=[patheffects.withStroke(linewidth=5, foreground='black')])
def generate_negative_2d_bb(obj_label, boxes2d, iou_threshold_min, iou_threshold_max, samples): """Generates negative 2D bounding boxes. This is computed in a brute force fashion. For any given bounding box, we randomly shift the centroid and generate new bounding boxes and if it lies within the desired IoU threshold bound, we will keep it. Otherwise it is thrown out and this is repeated until number of valid samples is satisfied. Keyword arguments: obj_label: single label for a detected 2D object boxes2d: a list of numpy array representing all bounding boxes in the image iou_threshold_min: determines the min variation between the original iou and the negative samples iou_threshold_max: determines the max variation between the original iou and the negative samples samples: number of negative samples per detected object Returns: a list of generated ObjectLabels """ x1 = obj_label.x1 y1 = obj_label.y1 x2 = obj_label.x2 y2 = obj_label.y2 diff_x = (x2 - x1) / 2 diff_y = (y2 - y1) / 2 current_samples = 0 new_objects = [] while current_samples < samples: # Keep trying to generate samples that # lie within reasonable bound new_xp = np.random.uniform(x1, x2, 1) new_yp = np.random.uniform(y1, y2, 1) new_x1 = float(new_xp - diff_x) new_x2 = float(new_xp + diff_x) new_y1 = float(new_yp - diff_y) new_y2 = float(new_yp + diff_y) new_obj = od.ObjectLabel() new_obj.x1 = new_x1 new_obj.x2 = new_x2 new_obj.y1 = new_y1 new_obj.y2 = new_y2 new_box = np.array([new_x1, new_y1, new_x2, new_y2]) # calculate the IoU iou = evaluation.two_d_iou(new_box, boxes2d) if iou_threshold_min < max(iou) < iou_threshold_max: # keep the new object label current_samples += 1 new_objects.append(new_obj) return new_objects
def draw_prediction_info(ax, x, y, pred_obj, pred_class_idx, pred_box, ground_truth, draw_score, draw_iou, gt_classes, iou_3d): label = "" if draw_score: label += "{:.2f}".format(pred_obj.score) if draw_iou and len(ground_truth) > 0: if draw_score: label += ', ' if iou_3d: iou = evaluation.three_d_iou(pred_box, ground_truth) if len(iou.shape) == 0: iou = np.array([iou]) else: iou = evaluation.two_d_iou(pred_box, ground_truth) label += "{:.3f}".format(max(iou)) box_cls = gt_classes[int(pred_class_idx)] ax.text( x, y - 4, gt_classes[int(pred_class_idx)] + '\n' + label, verticalalignment='bottom', horizontalalignment='center', color=BOX_COLOUR_SCHEME[box_cls], fontsize=15, fontweight='bold', path_effects=[patheffects.withStroke(linewidth=2, foreground='black')])
def _calculate_anchors_info(self, all_anchor_boxes_3d, empty_anchor_filter, gt_labels): """Calculates the list of anchor information in the format: N x 8 [max_gt_2d_iou, max_gt_3d_iou, (6 x offsets), class_index] max_gt_out - highest 3D iou with any ground truth box offsets - encoded offsets [dx, dy, dz, d_dimx, d_dimy, d_dimz] class_index - the anchor's class as an index (e.g. 0 or 1, for "Background" or "Car") Args: all_anchor_boxes_3d: list of anchors in box_3d format N x [x, y, z, l, w, h, ry] empty_anchor_filter: boolean mask of which anchors are non empty gt_labels: list of Object Label data format containing ground truth labels to generate positives/negatives from. Returns: list of anchor info """ # Check for ground truth objects if len(gt_labels) == 0: raise Warning("No valid ground truth label to generate anchors.") kitti_utils = self._dataset.kitti_utils # Filter empty anchors anchor_indices = np.where(empty_anchor_filter)[0] anchor_boxes_3d = all_anchor_boxes_3d[empty_anchor_filter] # Convert anchor_boxes_3d to anchor format anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) # Convert gt to boxes_3d -> anchors -> iou format gt_boxes_3d = np.asarray([ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in gt_labels ]) gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d, ortho_rotate=True) rpn_iou_type = self.mini_batch_utils.rpn_iou_type if rpn_iou_type == '2d': # Convert anchors to 2d iou format anchors_for_2d_iou, _ = np.asarray( anchor_projector.project_to_bev(anchors, kitti_utils.bev_extents)) gt_boxes_for_2d_iou, _ = anchor_projector.project_to_bev( gt_anchors, kitti_utils.bev_extents) elif rpn_iou_type == '3d': # Convert anchors to 3d iou format for calculation anchors_for_3d_iou = box_3d_encoder.box_3d_to_3d_iou_format( anchor_boxes_3d) gt_boxes_for_3d_iou = \ box_3d_encoder.box_3d_to_3d_iou_format(gt_boxes_3d) else: raise ValueError('Invalid rpn_iou_type {}', rpn_iou_type) # Initialize sample and offset lists num_anchors = len(anchor_boxes_3d) all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length)) # Update anchor indices all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices # For each of the labels, generate samples for gt_idx in range(len(gt_labels)): gt_obj = gt_labels[gt_idx] gt_box_3d = gt_boxes_3d[gt_idx] # Get 2D or 3D IoU for every anchor if self.mini_batch_utils.rpn_iou_type == '2d': gt_box_for_2d_iou = gt_boxes_for_2d_iou[gt_idx] ious = evaluation.two_d_iou(gt_box_for_2d_iou, anchors_for_2d_iou) elif self.mini_batch_utils.rpn_iou_type == '3d': gt_box_for_3d_iou = gt_boxes_for_3d_iou[gt_idx] ious = evaluation.three_d_iou(gt_box_for_3d_iou, anchors_for_3d_iou) # Only update indices with a higher iou than before update_indices = np.greater( ious, all_info[:, self.mini_batch_utils.col_ious]) # Get ious to update ious_to_update = ious[update_indices] # Calculate offsets, use 3D iou to get highest iou anchors_to_update = anchors[update_indices] gt_anchor = box_3d_encoder.box_3d_to_anchor(gt_box_3d, ortho_rotate=True) offsets = anchor_encoder.anchor_to_offset(anchors_to_update, gt_anchor) # Convert gt type to index class_idx = kitti_utils.class_str_to_index(gt_obj.type) # Update anchors info (indices already updated) # [index, iou, (offsets), class_index] all_info[update_indices, self.mini_batch_utils.col_ious] = ious_to_update all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self. mini_batch_utils.col_offsets_hi] = offsets all_info[update_indices, self.mini_batch_utils.col_class_idx] = class_idx return all_info
def calculate_negative_2d_bb(obj_label, boxes2d, iou_threshold_min, iou_threshold_max, samples, rand_sampl=False): """Generates negative 2D bounding boxes. This is computed in a semi-brute force fashion. For any given bounding box, we first try to calculate the desired shift based on the selected IoU. If this failes, we just randomly shift the centroid and generate new bounding boxes. If it lies within the desired IoU threshold bound, we will keep it. Otherwise it is thrown out and this is repeated until number of valid samples is satisfied. Keyword arguments: obj_label: single label for a detected 2D object boxes2d: a list of numpy array representing all bounding boxes in the image iou_threshold_min: determines the min variation between the original iou and the negative samples iou_threshold_max: determines the max variation between the original iou and the negative samples samples: number of negative samples per detected object rand_sampl: Flag to switch between calculation vs pure random sampling. For speed testing purposes. Returns: new_objects: a list of randomly generated ObjectLabels failed_cases: int number of cases it failed to calculate and opted in for random sampling. """ x1 = obj_label.x1 y1 = obj_label.y1 x2 = obj_label.x2 y2 = obj_label.y2 width = x2 - x1 length = y2 - y1 half_w = width / 2 half_l = length / 2 miscalc = False current_samples = 0 new_objects = [] failed_cases = 0 while current_samples < samples: # Keep trying to generate samples that # lie within reasonable bound if not miscalc and not rand_sampl: # we will try to to this by calculating # the shift along l or w given the desired # IoU and fixing either w or l shift. # this is sub-optimal since it does not # guarantee to achive the actual desired # IoU since we keep one variable fixed! # First let's try to generate bounding box # by randomly selecting a desirable IoU possible_ious = np.linspace(iou_threshold_min, iou_threshold_max, 10) # pick one randomly rand_iou = random.choice(possible_ious) # assuming l and w here are equal, given IoU # by fixing either delta_w or delta_l we can # calculate the other. This way we *guess* # the generated box will satify the IoU bound # constraint l_fixed = random.choice([True, False]) if l_fixed: # Lets keep delta_l fixed # It just needs to keep it within 0 - l delta_l = random.uniform(0, length / 2) l_shift = length - delta_l w_shift = (rand_iou * length * width) / (l_shift) delta_w = width - w_shift else: # keep delta_w fixed delta_w = random.uniform(0, width / 2) w_shift = length - delta_w l_shift = (rand_iou * length * width) / (w_shift) delta_l = length - l_shift # now just shift the l and w new_xp = x1 + delta_w new_yp = y1 + delta_l else: # that didn't work in the previous iteration # try generating random points new_xp = np.random.uniform(x1, x2, 1) new_yp = np.random.uniform(y1, y2, 1) # give it another chance in the next iteration miscalc = False new_obj, new_box = _construct_new_2d_object(new_xp, half_w, new_yp, half_l) # calculate the IoU iou = evaluation.two_d_iou(new_box, boxes2d) # check if it generated the desired IoU if iou_threshold_min < max(iou) < iou_threshold_max: # keep the new object label current_samples += 1 new_objects.append(new_obj) else: failed_cases += 1 miscalc = True return new_objects, failed_cases
def _calculate_img_anchors_info(self, all_anchor_boxes_3d, empty_anchor_filter, gt_labels, stereo_calib_p2, image_size): """Calculates the list of anchor information in the format: N x 6 [max_gt_img_iou, (4 x offsets), class_index] max_gt_img_out - highest img iou with any ground truth box offsets - encoded offsets [dx, dy, d_dimx, d_dimy] class_index - the anchor's class as an index (e.g. 0 or 1, for "Background" or "Car") Args: all_anchor_boxes_3d: list of anchors in box_3d format N x [x, y, z, l, w, h, ry] empty_anchor_filter: boolean mask of which anchors are non empty gt_labels: list of Object Label data format containing ground truth labels to generate positives/negatives from. image_size: [h, w] Returns: list of anchor info """ # Check for ground truth objects if len(gt_labels) == 0: raise Warning("No valid ground truth label to generate anchors.") kitti_utils = self._dataset.kitti_utils # Filter empty anchors anchor_indices = np.where(empty_anchor_filter)[0] anchor_boxes_3d = all_anchor_boxes_3d[empty_anchor_filter] # Convert anchor_boxes_3d to anchor format [x,y,z,dx,dy,dz] anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) ## benz, project anchor_boxes_3d into image plane instead of using anchors, TODO remove out_of_bounding points ## img_boxes: [x1, y1, x2, y2] # img_boxes_anchors, img_boxes_anchors_norm = anchor_projector.project_to_image_space(anchors, stereo_calib_p2, image_size) img_boxes_anchors, img_boxes_anchors_norm = anchor_projector.project_3d_box_to_image_space( anchor_boxes_3d, stereo_calib_p2, image_size) ## project 3d_boxes # Convert gt to boxes_3d -> anchors -> iou format #gt_boxes_3d = np.asarray( # [box_3d_encoder.object_label_to_box_3d(gt_obj) # for gt_obj in gt_labels]) ## benz, gt_boxes_3d [x1,y1,x2,y2] gt_boxes_2d = np.asarray([ box_2d_encoder.object_label_to_box_2d(gt_obj) for gt_obj in gt_labels ]) #gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d, # ortho_rotate=True) rpn_iou_type = self.mini_batch_utils.rpn_iou_type #if rpn_iou_type == '2d': # # Convert anchors to 2d iou format # #anchors_for_2d_iou, _ = np.asarray(anchor_projector.project_to_bev( # # anchors, kitti_utils.bev_extents)) # gt_boxes_for_2d_iou, _ = anchor_projector.project_to_bev( # gt_anchors, kitti_utils.bev_extents) #elif rpn_iou_type == '3d': # # Convert anchors to 3d iou format for calculation # anchors_for_3d_iou = box_3d_encoder.box_3d_to_3d_iou_format( # anchor_boxes_3d) # gt_boxes_for_3d_iou = \ # box_3d_encoder.box_3d_to_3d_iou_format(gt_boxes_3d) #else: # raise ValueError('Invalid rpn_iou_type {}', rpn_iou_type) # Initialize sample and offset lists num_anchors = len(anchor_boxes_3d) all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length)) # Update anchor indices all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices # For each of the labels, generate samples for gt_idx in range(len(gt_labels)): gt_obj = gt_labels[gt_idx] gt_box_2d = gt_boxes_2d[gt_idx] # gt_box_for_2d_iou = gt_boxes_for_2d_iou[gt_idx] ious = evaluation.two_d_iou(gt_box_2d, img_boxes_anchors) # Only update indices with a higher iou than before update_indices = np.greater( ious, all_info[:, self.mini_batch_utils.col_ious]) # Get ious to update ious_to_update = ious[update_indices] # Calculate offsets, use 3D iou to get highest iou anchors_to_update = img_boxes_anchors[update_indices] #gt_anchor = box_3d_encoder.box_3d_to_anchor(gt_box_2d, # ortho_rotate=True) offsets = anchor_encoder.np_2d_box_to_offset( anchors_to_update, gt_box_2d) # Convert gt type to index class_idx = kitti_utils.class_str_to_index(gt_obj.type) # Update anchors info (indices already updated) # [index, iou, (offsets), class_index] all_info[update_indices, self.mini_batch_utils.col_ious] = ious_to_update all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self. mini_batch_utils.col_offsets_hi] = offsets all_info[update_indices, self.mini_batch_utils.col_class_idx] = class_idx return all_info, img_boxes_anchors, img_boxes_anchors_norm
def _calculate_anchors_info(self, all_anchor_boxes_bev, empty_anchor_filter, gt_labels): """Calculates the list of anchor information in the format: N x 8 [max_gt_2d_iou_r, max_gt_2d_iou_h, (6 x offsets), class_index] max_gt_out - highest 2D iou with any ground truth box, using [anchor_r vs gt_r] or [anchor_h vs gt_h] offsets - encoded offsets [dx, dy, d_dimx, d_dimy, d_angle, angle_face_class_index, (-180,0) or (0,180)] class_index - the anchor's class as an index (e.g. 0 or 1, for "Background" or "Car") Args: all_anchor_boxes_3d: list of anchors in box_3d format N x [xc, yc, w, h, angle] empty_anchor_filter: boolean mask of which anchors are non empty gt_labels: list of Object Label data format containing ground truth labels to generate positives/negatives from. Returns: list of anchor info """ # Check for ground truth objects if len(gt_labels) == 0: raise Warning("No valid ground truth label to generate anchors.") kitti_utils = self._dataset.kitti_utils # Filter empty anchors anchor_indices = np.where(empty_anchor_filter)[0] anchors = all_anchor_boxes_bev[empty_anchor_filter] # Convert anchor_boxes_3d to anchor format #anchors = box_bev_encoder.box_bev_to_anchor(anchor_boxes_bev) # Convert gt to boxes_3d -> anchors -> iou format gt_boxes_3d = np.asarray([ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in gt_labels ]) gt_anchors_norm, _ = box_3d_projector.project_to_bev_box( gt_boxes_3d, self._area_extents[[0, 2]]) #bev_image_size = kitti_utils.area_extents / kitti_utils.voxel_size bev_map_h, bev_map_w = self._bev_shape #(N, 5) , (5, ) coorespondence element multiplification gt_anchors = np.multiply( gt_anchors_norm, np.array([bev_map_w, bev_map_h, bev_map_w, bev_map_h, 1])) iou_type = self.mini_batch_utils.retinanet_iou_type if iou_type == '2d_rotate': # Convert anchors to 2d iou format anchors_for_2d_iou_r = anchors gt_boxes_for_2d_iou_r = gt_anchors elif iou_type == '2d': # Convert anchors to 3d iou format for calculation anchors_for_2d_iou_h = box_bev_encoder.box_bev_to_iou_h_format( anchors) anchors_for_2d_iou_h = anchors_for_2d_iou_h.astype(np.int32) gt_boxes_for_2d_iou_h = box_bev_encoder.box_bev_to_iou_h_format( gt_anchors) gt_boxes_for_2d_iou_h = gt_boxes_for_2d_iou_h.astype(np.int32) else: raise ValueError('Invalid retinanet iou_type {}', iou_type) # Initialize sample and offset lists num_anchors = len(anchors) all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length)) # Update anchor indices all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices # For each of the labels, generate samples for gt_idx in range(len(gt_labels)): gt_obj = gt_labels[gt_idx] gt_box_3d = box_3d_encoder.object_label_to_box_3d(gt_obj) # Get 2D or 3D IoU for every anchor if self.mini_batch_utils.retinanet_iou_type == '2d': gt_box_for_2d_iou_h = gt_boxes_for_2d_iou_h[gt_idx] ious = evaluation.two_d_iou(gt_box_for_2d_iou_h, anchors_for_2d_iou_h) elif self.mini_batch_utils.retinanet_iou_type == '2d_rotate': gt_box_for_2d_iou_r = gt_boxes_for_2d_iou_r[gt_idx] ious = evaluation.two_d_rotate_iou(gt_box_for_2d_iou_r, anchors_for_2d_iou_r) # Only update indices with a higher iou than before update_indices = np.greater( ious, all_info[:, self.mini_batch_utils.col_ious]) # Get ious to update ious_to_update = ious[update_indices] # Calculate offsets, use 3D iou to get highest iou anchors_to_update = anchors[update_indices] facing_obj_head = gt_obj.ry >= 0 #camera facing object's head. gt_anchor = gt_anchors[gt_idx] #turns (-pi, pi) to (-pi, 0) for gt_anchor's angle if facing_obj_head: gt_anchor[-1] -= np.pi offsets_boxes = anchor_bev_encoder.anchor_to_offset( anchors_to_update, gt_anchor) gt_anchor_pred = anchor_bev_encoder.offset_to_anchor( anchors_to_update, offsets_boxes) #y axis 3d value n_anchor = offsets_boxes.shape[0] anchor_h = anchor_bev_encoder.get_default_anchor_h(n_anchor, 'np') gt_h = [gt_obj.t[1], gt_obj.h] offsets_h = anchor_bev_encoder.anchor_to_offset_h(anchor_h, gt_h) gt_anchors_angle = np.zeros_like(offsets_boxes[:, 0], dtype=np.int) + gt_obj.ry offsets_angle_cls = orientation_encoder.orientation_to_angle_cls( gt_anchors_angle) offsets = np.hstack( [offsets_boxes, offsets_h, offsets_angle_cls[:, np.newaxis]]) # Convert gt type to index class_idx = kitti_utils.class_str_to_index(gt_obj.type) # Update anchors info (indices already updated) # [index, iou, (offsets), class_index] all_info[update_indices, self.mini_batch_utils.col_ious] = ious_to_update all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self. mini_batch_utils.col_offsets_hi] = offsets all_info[update_indices, self.mini_batch_utils.col_class_idx] = class_idx debug = False #True if debug: print(f'gt obj:{gt_box_3d}, gt anchor bev: {gt_anchor}') print(f'anchors_to_update: {anchors_to_update[:1]}') print(f'update at all_info: \n{all_info[update_indices][:1]}') print(f'gt_from_anchor_offsets:\n{gt_anchor_pred[:1]}') return all_info