def test_different_iou_threshold(self): boxes = np.array([[0, 0, 20, 100], [0, 0, 20, 80], [200, 200, 210, 300], [200, 200, 210, 250]], dtype=float) boxlist = np_box_list.BoxList(boxes) boxlist.add_field('scores', np.array([0.9, 0.8, 0.7, 0.6])) max_output_size = 4 iou_threshold = .4 expected_boxes = np.array([[0, 0, 20, 100], [200, 200, 210, 300],], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes) iou_threshold = .5 expected_boxes = np.array([[0, 0, 20, 100], [200, 200, 210, 300], [200, 200, 210, 250]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes) iou_threshold = .8 expected_boxes = np.array([[0, 0, 20, 100], [0, 0, 20, 80], [200, 200, 210, 300], [200, 200, 210, 250]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_different_iou_threshold(self): boxes = np.array([[0, 0, 20, 100], [0, 0, 20, 80], [200, 200, 210, 300], [200, 200, 210, 250]], dtype=float) boxlist = np_box_list.BoxList(boxes) boxlist.add_field('scores', np.array([0.9, 0.8, 0.7, 0.6])) max_output_size = 4 iou_threshold = .4 expected_boxes = np.array([[0, 0, 20, 100], [200, 200, 210, 300],], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes) iou_threshold = .5 expected_boxes = np.array([[0, 0, 20, 100], [200, 200, 210, 300], [200, 200, 210, 250]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes) iou_threshold = .8 expected_boxes = np.array([[0, 0, 20, 100], [0, 0, 20, 80], [200, 200, 210, 300], [200, 200, 210, 250]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_with_no_scores_field(self): boxlist = np_box_list.BoxList(self._boxes) max_output_size = 3 iou_threshold = 0.5 with self.assertRaises(ValueError): np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold)
def test_with_no_scores_field(self): boxlist = np_box_list.BoxList(self._boxes) max_output_size = 3 iou_threshold = 0.5 with self.assertRaises(ValueError): np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold)
def test_select_from_ten_indentical_boxes(self): boxes = np.array(10 * [[0, 0, 1, 1]], dtype=float) boxlist = np_box_list.BoxList(boxes) boxlist.add_field('scores', np.array(10 * [0.8])) iou_threshold = .5 max_output_size = 3 expected_boxes = np.array([[0, 0, 1, 1]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_select_from_ten_indentical_boxes(self): boxes = np.array(10 * [[0, 0, 1, 1]], dtype=float) boxlist = np_box_list.BoxList(boxes) boxlist.add_field('scores', np.array(10 * [0.8])) iou_threshold = .5 max_output_size = 3 expected_boxes = np.array([[0, 0, 1, 1]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_select_at_most_two_from_three_clusters(self): boxlist = np_box_list.BoxList(self._boxes) boxlist.add_field('scores', np.array([.9, .75, .6, .95, .5, .3], dtype=float)) max_output_size = 2 iou_threshold = 0.5 expected_boxes = np.array([[0, 10, 1, 11], [0, 0, 1, 1]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_select_at_most_two_from_three_clusters(self): boxlist = np_box_list.BoxList(self._boxes) boxlist.add_field('scores', np.array([.9, .75, .6, .95, .5, .3], dtype=float)) max_output_size = 2 iou_threshold = 0.5 expected_boxes = np.array([[0, 10, 1, 11], [0, 0, 1, 1]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_nms_disabled_max_output_size_equals_three(self): boxlist = np_box_list.BoxList(self._boxes) boxlist.add_field('scores', np.array([.9, .75, .6, .95, .2, .3], dtype=float)) max_output_size = 3 iou_threshold = 1. # No NMS expected_boxes = np.array([[0, 10, 1, 11], [0, 0, 1, 1], [0, 0.1, 1, 1.1]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def test_nms_disabled_max_output_size_equals_three(self): boxlist = np_box_list.BoxList(self._boxes) boxlist.add_field('scores', np.array([.9, .75, .6, .95, .2, .3], dtype=float)) max_output_size = 3 iou_threshold = 1. # No NMS expected_boxes = np.array([[0, 10, 1, 11], [0, 0, 1, 1], [0, 0.1, 1, 1.1]], dtype=float) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes)
def _compute_tp_fp_for_single_class(self, detected_boxes, detected_scores, groundtruth_boxes, groundtruth_is_difficult_list): """Labels boxes detected with the same class from the same image as tp/fp. Args: detected_boxes: A numpy array of shape [N, 4] representing detected box coordinates detected_scores: A 1-d numpy array of length N representing classification score groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth box coordinates groundtruth_is_difficult_list: A boolean numpy array of length M denoting whether a ground truth box is a difficult instance or not Returns: scores: A numpy array representing the detection scores tp_fp_labels: a boolean numpy array indicating whether a detection is a true positive. """ if detected_boxes.size == 0: return np.array([], dtype=float), np.array([], dtype=bool) detected_boxlist = np_box_list.BoxList(detected_boxes) detected_boxlist.add_field('scores', detected_scores) detected_boxlist = np_box_list_ops.non_max_suppression( detected_boxlist, self.nms_max_output_boxes, self.nms_iou_threshold) scores = detected_boxlist.get_field('scores') if groundtruth_boxes.size == 0: return scores, np.zeros(detected_boxlist.num_boxes(), dtype=bool) gt_boxlist = np_box_list.BoxList(groundtruth_boxes) iou = np_box_list_ops.iou(detected_boxlist, gt_boxlist) max_overlap_gt_ids = np.argmax(iou, axis=1) is_gt_box_detected = np.zeros(gt_boxlist.num_boxes(), dtype=bool) tp_fp_labels = np.zeros(detected_boxlist.num_boxes(), dtype=bool) is_matched_to_difficult_box = np.zeros(detected_boxlist.num_boxes(), dtype=bool) for i in range(detected_boxlist.num_boxes()): gt_id = max_overlap_gt_ids[i] if iou[i, gt_id] >= self.matching_iou_threshold: if not groundtruth_is_difficult_list[gt_id]: if not is_gt_box_detected[gt_id]: tp_fp_labels[i] = True is_gt_box_detected[gt_id] = True else: is_matched_to_difficult_box[i] = True return scores[~is_matched_to_difficult_box], tp_fp_labels[ ~is_matched_to_difficult_box]
def _compute_tp_fp_for_single_class(self, detected_boxes, detected_scores, groundtruth_boxes, groundtruth_is_difficult_list): """Labels boxes detected with the same class from the same image as tp/fp. Args: detected_boxes: A numpy array of shape [N, 4] representing detected box coordinates detected_scores: A 1-d numpy array of length N representing classification score groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth box coordinates groundtruth_is_difficult_list: A boolean numpy array of length M denoting whether a ground truth box is a difficult instance or not Returns: scores: A numpy array representing the detection scores tp_fp_labels: a boolean numpy array indicating whether a detection is a true positive. """ if detected_boxes.size == 0: return np.array([], dtype=float), np.array([], dtype=bool) detected_boxlist = np_box_list.BoxList(detected_boxes) detected_boxlist.add_field('scores', detected_scores) detected_boxlist = np_box_list_ops.non_max_suppression( detected_boxlist, self.nms_max_output_boxes, self.nms_iou_threshold) scores = detected_boxlist.get_field('scores') if groundtruth_boxes.size == 0: return scores, np.zeros(detected_boxlist.num_boxes(), dtype=bool) gt_boxlist = np_box_list.BoxList(groundtruth_boxes) iou = np_box_list_ops.iou(detected_boxlist, gt_boxlist) max_overlap_gt_ids = np.argmax(iou, axis=1) is_gt_box_detected = np.zeros(gt_boxlist.num_boxes(), dtype=bool) tp_fp_labels = np.zeros(detected_boxlist.num_boxes(), dtype=bool) is_matched_to_difficult_box = np.zeros( detected_boxlist.num_boxes(), dtype=bool) for i in range(detected_boxlist.num_boxes()): gt_id = max_overlap_gt_ids[i] if iou[i, gt_id] >= self.matching_iou_threshold: if not groundtruth_is_difficult_list[gt_id]: if not is_gt_box_detected[gt_id]: tp_fp_labels[i] = True is_gt_box_detected[gt_id] = True else: is_matched_to_difficult_box[i] = True return scores[~is_matched_to_difficult_box], tp_fp_labels[ ~is_matched_to_difficult_box]
def _get_overlaps_and_scores_box_mode(self, detected_boxes, detected_scores, groundtruth_boxes, groundtruth_is_group_of_list): """Computes overlaps and scores between detected and groudntruth boxes. Args: detected_boxes: A numpy array of shape [N, 4] representing detected box coordinates detected_scores: A 1-d numpy array of length N representing classification score groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth box coordinates groundtruth_is_group_of_list: A boolean numpy array of length M denoting whether a ground truth box has group-of tag. If a groundtruth box is group-of box, every detection matching this box is ignored. Returns: iou: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If gt_non_group_of_boxlist.num_boxes() == 0 it will be None. ioa: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If gt_group_of_boxlist.num_boxes() == 0 it will be None. scores: The score of the detected boxlist. num_boxes: Number of non-maximum suppressed detected boxes. """ detected_boxlist = np_box_list.BoxList(detected_boxes) #print(detected_boxes) #print('no of detected boxes22', len(detected_boxes)) #print('detected_boxlist size', detected_boxlist.num_boxes()) #print('detected_boxlist.data',detected_boxlist.data['boxes']) detected_boxlist.add_field('scores', detected_scores) detected_boxlist = np_box_list_ops.non_max_suppression( detected_boxlist, self.nms_max_output_boxes, 0.5) #print('detected_boxlist size after nms', detected_boxlist.num_boxes()) gt_non_group_of_boxlist = np_box_list.BoxList( groundtruth_boxes[~groundtruth_is_group_of_list]) gt_group_of_boxlist = np_box_list.BoxList( groundtruth_boxes[groundtruth_is_group_of_list]) iou = np_box_list_ops.iou(detected_boxlist, gt_non_group_of_boxlist) ioa = np.transpose( np_box_list_ops.ioa(gt_group_of_boxlist, detected_boxlist)) scores = detected_boxlist.get_field('scores') num_boxes = detected_boxlist.num_boxes() return iou, ioa, scores, num_boxes
def _get_overlaps_and_scores_box_mode( self, detected_boxes, detected_scores, groundtruth_boxes, groundtruth_is_group_of_list): """Computes overlaps and scores between detected and groudntruth boxes. Args: detected_boxes: A numpy array of shape [N, 4] representing detected box coordinates detected_scores: A 1-d numpy array of length N representing classification score groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth box coordinates groundtruth_is_group_of_list: A boolean numpy array of length M denoting whether a ground truth box has group-of tag. If a groundtruth box is group-of box, every detection matching this box is ignored. Returns: iou: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If gt_non_group_of_boxlist.num_boxes() == 0 it will be None. ioa: A float numpy array of size [num_detected_boxes, num_gt_boxes]. If gt_group_of_boxlist.num_boxes() == 0 it will be None. scores: The score of the detected boxlist. num_boxes: Number of non-maximum suppressed detected boxes. """ detected_boxlist = np_box_list.BoxList(detected_boxes) detected_boxlist.add_field('scores', detected_scores) detected_boxlist = np_box_list_ops.non_max_suppression( detected_boxlist, self.nms_max_output_boxes, self.nms_iou_threshold) gt_non_group_of_boxlist = np_box_list.BoxList( groundtruth_boxes[~groundtruth_is_group_of_list]) gt_group_of_boxlist = np_box_list.BoxList( groundtruth_boxes[groundtruth_is_group_of_list]) iou = np_box_list_ops.iou(detected_boxlist, gt_non_group_of_boxlist) ioa = np.transpose( np_box_list_ops.ioa(gt_group_of_boxlist, detected_boxlist)) scores = detected_boxlist.get_field('scores') num_boxes = detected_boxlist.num_boxes() return iou, ioa, scores, num_boxes
def multi_class_non_max_suppression(boxlist, score_thresh, iou_thresh, max_output_size): if not 0 <= iou_thresh <= 1.0: raise ValueError('thresh must be between 0 and 1') scores = boxlist.get_field('scores') classes = boxlist.get_field('classes') num_boxes = boxlist.num_boxes() selected_boxes_list = [] for cls in np.unique(classes): inds = np.where(cls == classes)[0] subboxlist = gather(boxlist, inds) boxlist_filt = filter_scores_greater_than(subboxlist, score_thresh) nms_result = non_max_suppression(boxlist_filt, max_output_size=max_output_size, iou_threshold=iou_thresh, score_threshold=score_thresh) selected_boxes_list.append(nms_result) selected_boxes = concatenate(selected_boxes_list) sorted_boxes = sort_by_field(selected_boxes, 'scores') return sorted_boxes
def multi_class_non_max_suppression_with_extra_fields( boxlist: BoxList, score_thresh: float, iou_thresh: float, max_output_size: int) -> BoxList: """ Is the same method as `object_detection.utils.np_box_list_ops.multi_class_non_max_suppression` but also preserves all the extra fields of the boxlist after nms """ # Modification: omit all the assertions # End of modification scores = boxlist.get_field('scores') num_scores = scores.shape[0] num_classes = scores.shape[1] selected_boxes_list = [] for class_idx in range(num_classes): boxlist_and_class_scores = np_box_list.BoxList(boxlist.get()) class_scores = np.reshape(scores[0:num_scores, class_idx], [-1]) boxlist_and_class_scores.add_field('scores', class_scores) # Modification: add extra fields to boxlist_and_class_scores for each_extra_field in boxlist.get_extra_fields(): if (not boxlist_and_class_scores.has_field(each_extra_field) and each_extra_field != "classes"): boxlist_and_class_scores.add_field( each_extra_field, boxlist.get_field(each_extra_field)) # End of modification boxlist_filt = np_box_list_ops.filter_scores_greater_than( boxlist_and_class_scores, score_thresh) nms_result = np_box_list_ops.non_max_suppression( boxlist_filt, max_output_size=max_output_size, iou_threshold=iou_thresh, score_threshold=score_thresh) nms_result.add_field( 'classes', np.zeros_like(nms_result.get_field('scores')) + class_idx) selected_boxes_list.append(nms_result) selected_boxes = np_box_list_ops.concatenate(selected_boxes_list) sorted_boxes = np_box_list_ops.sort_by_field(selected_boxes, 'scores') return sorted_boxes
def prune_duplicates(labels, score_thresh, merge_thresh): """Remove duplicate boxes. Runs non-maximum suppression to remove duplicate boxes that result from sliding window prediction algorithm. Args: labels: ObjectDetectionLabels score_thresh: the minimum allowed score of boxes merge_thresh: the minimum IOA allowed when merging two boxes together Returns: ObjectDetectionLabels """ from object_detection.utils.np_box_list_ops import non_max_suppression max_output_size = 1000000 pruned_boxlist = non_max_suppression(labels.boxlist, max_output_size=max_output_size, iou_threshold=merge_thresh, score_threshold=score_thresh) return ObjectDetectionLabels.from_boxlist(pruned_boxlist)
def test_soft_nms(self): boxes = np.array([[0, 0, 20, 20], [10, 0, 30, 20], [0, 10, 20, 30], [10, 10, 30, 30]], dtype=np.float32) boxlist = np_box_list.BoxList(boxes) boxlist.add_field('scores', np.array([0.9, 0.8, 0.7, 0.6])) max_output_size = 4 iou_threshold = 0.3 expected_boxes = np.array([[0, 0, 20, 20], [10, 10, 30, 30]], dtype=np.float32) nms_boxlist = np_box_list_ops.non_max_suppression( boxlist, max_output_size, iou_threshold) self.assertAllClose(nms_boxlist.get(), expected_boxes) nms_boxlist = np_box_list_ops.soft_non_max_suppression( boxlist, max_output_size, iou_threshold) expected_scores = np.flip( np.sort([ 0.9, 0.8 * (1 - 1. / 3) * (1 - 1. / 3), 0.7 * (1 - 1. / 3) * (1 - 1. / 3), 0.6 ]), 0) self.assertAllClose(nms_boxlist.get_field('scores'), expected_scores)
def _compute_tp_fp_for_single_class(self, detected_boxes, detected_scores, groundtruth_boxes, groundtruth_is_difficult_list, groundtruth_is_group_of_list): """Labels boxes detected with the same class from the same image as tp/fp. Args: detected_boxes: A numpy array of shape [N, 4] representing detected box coordinates detected_scores: A 1-d numpy array of length N representing classification score groundtruth_boxes: A numpy array of shape [M, 4] representing ground truth box coordinates groundtruth_is_difficult_list: A boolean numpy array of length M denoting whether a ground truth box is a difficult instance or not. If a groundtruth box is difficult, every detection matching this box is ignored. groundtruth_is_group_of_list: A boolean numpy array of length M denoting whether a ground truth box has group-of tag. If a groundtruth box is group-of box, every detection matching this box is ignored. Returns: Two arrays of the same size, containing all boxes that were evaluated as being true positives or false positives; if a box matched to a difficult box or to a group-of box, it is ignored. scores: A numpy array representing the detection scores. tp_fp_labels: a boolean numpy array indicating whether a detection is a true positive. """ if detected_boxes.size == 0: return np.array([], dtype=float), np.array([], dtype=bool) detected_boxlist = np_box_list.BoxList(detected_boxes) detected_boxlist.add_field('scores', detected_scores) detected_boxlist = np_box_list_ops.non_max_suppression( detected_boxlist, self.nms_max_output_boxes, self.nms_iou_threshold) scores = detected_boxlist.get_field('scores') if groundtruth_boxes.size == 0: return scores, np.zeros(detected_boxlist.num_boxes(), dtype=bool) tp_fp_labels = np.zeros(detected_boxlist.num_boxes(), dtype=bool) is_matched_to_difficult_box = np.zeros(detected_boxlist.num_boxes(), dtype=bool) is_matched_to_group_of_box = np.zeros(detected_boxlist.num_boxes(), dtype=bool) # The evaluation is done in two stages: # 1. All detections are matched to non group-of boxes; true positives are # determined and detections matched to difficult boxes are ignored. # 2. Detections that are determined as false positives are matched against # group-of boxes and ignored if matched. # Tp-fp evaluation for non-group of boxes (if any). gt_non_group_of_boxlist = np_box_list.BoxList( groundtruth_boxes[~groundtruth_is_group_of_list, :]) if gt_non_group_of_boxlist.num_boxes() > 0: groundtruth_nongroup_of_is_difficult_list = groundtruth_is_difficult_list[ ~groundtruth_is_group_of_list] iou = np_box_list_ops.iou(detected_boxlist, gt_non_group_of_boxlist) max_overlap_gt_ids = np.argmax(iou, axis=1) is_gt_box_detected = np.zeros(gt_non_group_of_boxlist.num_boxes(), dtype=bool) for i in range(detected_boxlist.num_boxes()): gt_id = max_overlap_gt_ids[i] if iou[i, gt_id] >= self.matching_iou_threshold: if not groundtruth_nongroup_of_is_difficult_list[gt_id]: if not is_gt_box_detected[gt_id]: tp_fp_labels[i] = True is_gt_box_detected[gt_id] = True else: is_matched_to_difficult_box[i] = True # Tp-fp evaluation for group of boxes. gt_group_of_boxlist = np_box_list.BoxList( groundtruth_boxes[groundtruth_is_group_of_list, :]) if gt_group_of_boxlist.num_boxes() > 0: ioa = np_box_list_ops.ioa(gt_group_of_boxlist, detected_boxlist) max_overlap_group_of_gt = np.max(ioa, axis=0) for i in range(detected_boxlist.num_boxes()): if (not tp_fp_labels[i] and not is_matched_to_difficult_box[i] and max_overlap_group_of_gt[i] >= self.matching_iou_threshold): is_matched_to_group_of_box[i] = True return scores[~is_matched_to_difficult_box & ~is_matched_to_group_of_box], tp_fp_labels[ ~is_matched_to_difficult_box & ~is_matched_to_group_of_box]
def check_for_duplicate_annotations(dataset, iou_threshold=0.9, remove_boxes=False): """ Unusually high IOU values between boxes can be a sign of duplicate annotations on the same instance. Returns: A list of image ids that may contain duplicate annotations """ if np_box_ops is None: print( "WARNING: `np_box_ops` failed to import, can't check for duplicates." ) return [] image_id_to_annos = {image['id']: [] for image in dataset['images']} for anno in dataset['annotations']: if 'bbox' in anno: image_id_to_annos[anno['image_id']].append(anno) image_ids_with_duplicates = [] anno_ids_to_remove = set() for image_id, annos in image_id_to_annos.items(): if len(annos) > 1: boxes = [] ids = [] for anno in annos: x, y, w, h = anno['bbox'] x2 = x + w y2 = y + h boxes.append([y, x, y2, x2]) ids.append(anno['id']) boxes = np.array(boxes).astype(np.float) scores = np.ones(len(ids), dtype=np.float32) ids = np.array(ids) try: boxlist = np_box_list.BoxList(boxes) except: print(image_id) print(annos) print(boxes) raise boxlist.add_field("ids", ids) boxlist.add_field("scores", scores) nms = np_box_list_ops.non_max_suppression( boxlist, iou_threshold=iou_threshold) selected_ids = nms.get_field("ids").tolist() removed_ids = [aid for aid in ids if aid not in selected_ids] if len(removed_ids): anno_ids_to_remove.update(removed_ids) image_ids_with_duplicates.append(image_id) if remove_boxes: dataset['annotations'] = [ anno for anno in dataset['annotations'] if anno['id'] not in anno_ids_to_remove ] return image_ids_with_duplicates