def test_merge_internal_state(self): # Test that if initial state is merged, the results of the evaluation are # the same. od_eval_state = self.od_eval.get_internal_state() copy_od_eval = object_detection_evaluation.ObjectDetectionEvaluation( self.od_eval.num_class) copy_od_eval.merge_internal_state(od_eval_state) (average_precision_per_class, mean_ap, precisions_per_class, recalls_per_class, corloc_per_class, mean_corloc) = self.od_eval.evaluate() (copy_average_precision_per_class, copy_mean_ap, copy_precisions_per_class, copy_recalls_per_class, copy_corloc_per_class, copy_mean_corloc) = copy_od_eval.evaluate() for i in range(self.od_eval.num_class): self.assertTrue( np.allclose(copy_precisions_per_class[i], precisions_per_class[i])) self.assertTrue( np.allclose(copy_recalls_per_class[i], recalls_per_class[i])) self.assertTrue( np.allclose(copy_average_precision_per_class, average_precision_per_class)) self.assertTrue(np.allclose(copy_corloc_per_class, corloc_per_class)) self.assertAlmostEqual(copy_mean_ap, mean_ap) self.assertAlmostEqual(copy_mean_corloc, mean_corloc)
def setUp(self): num_groundtruth_classes = 3 self.od_eval = object_detection_evaluation.ObjectDetectionEvaluation( num_groundtruth_classes) image_key1 = "img1" groundtruth_boxes1 = np.array([[0, 0, 1, 1], [0, 0, 2, 2], [0, 0, 3, 3]], dtype=float) groundtruth_class_labels1 = np.array([0, 2, 0], dtype=int) self.od_eval.add_single_ground_truth_image_info( image_key1, groundtruth_boxes1, groundtruth_class_labels1) image_key2 = "img2" groundtruth_boxes2 = np.array([[10, 10, 11, 11], [500, 500, 510, 510], [10, 10, 12, 12]], dtype=float) groundtruth_class_labels2 = np.array([0, 0, 2], dtype=int) groundtruth_is_difficult_list2 = np.array([False, True, False], dtype=bool) self.od_eval.add_single_ground_truth_image_info( image_key2, groundtruth_boxes2, groundtruth_class_labels2, groundtruth_is_difficult_list2) image_key3 = "img3" groundtruth_boxes3 = np.array([[0, 0, 1, 1]], dtype=float) groundtruth_class_labels3 = np.array([1], dtype=int) self.od_eval.add_single_ground_truth_image_info( image_key3, groundtruth_boxes3, groundtruth_class_labels3) image_key = "img2" detected_boxes = np.array( [[10, 10, 11, 11], [100, 100, 120, 120], [100, 100, 220, 220]], dtype=float) detected_class_labels = np.array([0, 0, 2], dtype=int) detected_scores = np.array([0.7, 0.8, 0.9], dtype=float) self.od_eval.add_single_detected_image_info( image_key, detected_boxes, detected_scores, detected_class_labels)
def _build_od_evaluation(self): # per image evaluation is not done inside of accumulator def _per_image_eval_class(**kwargs): # pylint: disable=unused-argument return None self._od_evaluation = od_eval.ObjectDetectionEvaluation( num_groundtruth_classes=self.num_classes, label_id_offset=1, use_weighted_mean_ap=self.use_weighted_mean_ap, per_image_eval_class=_per_image_eval_class)
def compute_od_eval(ground_truth_labels, prediction_labels, nb_classes): # Lazy import of TFOD from object_detection.utils import object_detection_evaluation matching_iou_threshold = 0.5 od_eval = object_detection_evaluation.ObjectDetectionEvaluation( nb_classes, matching_iou_threshold=matching_iou_threshold) image_key = 'image' od_eval.add_single_ground_truth_image_info( image_key, ground_truth_labels.get_npboxes(), ground_truth_labels.get_class_ids() - 1) od_eval.add_single_detected_image_info( image_key, prediction_labels.get_npboxes(), prediction_labels.get_scores(), prediction_labels.get_class_ids() - 1) od_eval.evaluate() return od_eval
def evaluate_detection_results_pascal_voc(result_lists, categories, label_id_offset=0, iou_thres=0.5, corloc_summary=False): """Computes Pascal VOC detection metrics given groundtruth and detections. This function computes Pascal VOC metrics. This function by default takes detections and groundtruth boxes encoded in result_lists and writes evaluation results to tf summaries which can be viewed on tensorboard. Args: result_lists: a dictionary holding lists of groundtruth and detection data corresponding to each image being evaluated. The following keys are required: 'image_id': a list of string ids 'detection_boxes': a list of float32 numpy arrays of shape [N, 4] 'detection_scores': a list of float32 numpy arrays of shape [N] 'detection_classes': a list of int32 numpy arrays of shape [N] 'groundtruth_boxes': a list of float32 numpy arrays of shape [M, 4] 'groundtruth_classes': a list of int32 numpy arrays of shape [M] and the remaining fields below are optional: 'difficult': a list of boolean arrays of shape [M] indicating the difficulty of groundtruth boxes. Some datasets like PASCAL VOC provide this information and it is used to remove difficult examples from eval in order to not penalize the models on them. Note that it is okay to have additional fields in result_lists --- they are simply ignored. categories: a list of dictionaries representing all possible categories. Each dict in this list has the following keys: 'id': (required) an integer id uniquely identifying this category 'name': (required) string representing category name e.g., 'cat', 'dog', 'pizza' label_id_offset: an integer offset for the label space. iou_thres: float determining the IoU threshold at which a box is considered correct. Defaults to the standard 0.5. corloc_summary: boolean. If True, also outputs CorLoc metrics. Returns: A dictionary of metric names to scalar values. Raises: ValueError: if the set of keys in result_lists is not a superset of the expected list of keys. Unexpected keys are ignored. ValueError: if the lists in result_lists have inconsistent sizes. """ # check for expected keys in result_lists expected_keys = [ 'detection_boxes', 'detection_scores', 'detection_classes', 'image_id' ] expected_keys += ['groundtruth_boxes', 'groundtruth_classes'] if not set(expected_keys).issubset(set(result_lists.keys())): raise ValueError('result_lists does not have expected key set.') num_results = len(result_lists[expected_keys[0]]) for key in expected_keys: if len(result_lists[key]) != num_results: raise ValueError('Inconsistent list sizes in result_lists') # Pascal VOC evaluator assumes foreground index starts from zero. categories = copy.deepcopy(categories) for idx in range(len(categories)): categories[idx]['id'] -= label_id_offset # num_classes (maybe encoded as categories) num_classes = max([cat['id'] for cat in categories]) + 1 logging.info('Computing Pascal VOC metrics on results.') if all(image_id.isdigit() for image_id in result_lists['image_id']): image_ids = [int(image_id) for image_id in result_lists['image_id']] else: image_ids = range(num_results) evaluator = object_detection_evaluation.ObjectDetectionEvaluation( num_classes, matching_iou_threshold=iou_thres) difficult_lists = None if 'difficult' in result_lists and result_lists['difficult']: difficult_lists = result_lists['difficult'] for idx, image_id in enumerate(image_ids): difficult = None if difficult_lists is not None and difficult_lists[idx].size: difficult = difficult_lists[idx].astype(np.bool) evaluator.add_single_ground_truth_image_info( image_id, result_lists['groundtruth_boxes'][idx], result_lists['groundtruth_classes'][idx] - label_id_offset, difficult) evaluator.add_single_detected_image_info( image_id, result_lists['detection_boxes'][idx], result_lists['detection_scores'][idx], result_lists['detection_classes'][idx] - label_id_offset) per_class_ap, mean_ap, _, _, per_class_corloc, mean_corloc = ( evaluator.evaluate()) metrics = {'Precision/mAP@{}IOU'.format(iou_thres): mean_ap} category_index = label_map_util.create_category_index(categories) for idx in range(per_class_ap.size): if idx in category_index: display_name = ('PerformanceByCategory/mAP@{}IOU/{}'.format( iou_thres, category_index[idx]['name'])) metrics[display_name] = per_class_ap[idx] if corloc_summary: metrics['CorLoc/CorLoc@{}IOU'.format(iou_thres)] = mean_corloc for idx in range(per_class_corloc.size): if idx in category_index: display_name = ('PerformanceByCategory/CorLoc@{}IOU/{}'.format( iou_thres, category_index[idx]['name'])) metrics[display_name] = per_class_corloc[idx] return metrics
def test_value_error_on_zero_classes(self): with self.assertRaises(ValueError): object_detection_evaluation.ObjectDetectionEvaluation( num_groundtruth_classes=0)