def testMissingDetectionResults(self):
"""Tests if groundtrue is missing, raises ValueError."""
category_list = [{'id': 0, 'name': 'dog'}]
video_evaluator = coco_evaluation_all_frames.CocoEvaluationAllFrames(
category_list)
video_evaluator.add_single_ground_truth_image_info(
image_id='image1',
groundtruth_dict=[{
standard_fields.InputDataFields.groundtruth_boxes:
np.array([[100., 100., 200., 200.]]),
standard_fields.InputDataFields.groundtruth_classes:
np.array([1])
}])
with self.assertRaisesRegexp(ValueError,
r'Missing groundtruth for image-frame id:.*'):
video_evaluator.add_single_detected_image_info(
image_id='image3',
detections_dict=[{
standard_fields.DetectionResultFields.detection_boxes:
np.array([[100., 100., 200., 200.]]),
standard_fields.DetectionResultFields.detection_scores:
np.array([.8]),
standard_fields.DetectionResultFields.detection_classes:
np.array([1])
}])
def testGroundtruthAndDetectionsDisagreeOnAllFrames(self):
"""Tests that mAP is calculated on several different frame results."""
category_list = [{'id': 0, 'name': 'dog'}, {'id': 1, 'name': 'cat'}]
video_evaluator = coco_evaluation_all_frames.CocoEvaluationAllFrames(
category_list)
video_evaluator.add_single_ground_truth_image_info(
image_id='image1',
groundtruth_dict=[{
standard_fields.InputDataFields.groundtruth_boxes:
np.array([[50., 50., 200., 200.]]),
standard_fields.InputDataFields.groundtruth_classes:
np.array([1])
}, {
standard_fields.InputDataFields.groundtruth_boxes:
np.array([[50., 50., 100., 100.]]),
standard_fields.InputDataFields.groundtruth_classes:
np.array([1])
}])
video_evaluator.add_single_detected_image_info(
image_id='image1',
# A different groundtruth box on the frame other than the last one.
detections_dict=[{
standard_fields.DetectionResultFields.detection_boxes:
np.array([[100., 100., 200., 200.]]),
standard_fields.DetectionResultFields.detection_scores:
np.array([.8]),
standard_fields.DetectionResultFields.detection_classes:
np.array([1])
}, {
standard_fields.DetectionResultFields.detection_boxes:
np.array([[50., 50., 100., 100.]]),
standard_fields.DetectionResultFields.detection_scores:
np.array([.8]),
standard_fields.DetectionResultFields.detection_classes:
np.array([1])
}])
metrics = video_evaluator.evaluate()
self.assertNotEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)
def testGroundtruthAndDetections(self):
"""Tests that mAP is calculated correctly on GT and Detections."""
category_list = [{'id': 0, 'name': 'dog'}, {'id': 1, 'name': 'cat'}]
video_evaluator = coco_evaluation_all_frames.CocoEvaluationAllFrames(
category_list)
video_evaluator.add_single_ground_truth_image_info(
image_id='image1',
groundtruth_dict=[{
standard_fields.InputDataFields.groundtruth_boxes:
np.array([[100., 100., 200., 200.]]),
standard_fields.InputDataFields.groundtruth_classes:
np.array([1])
}])
video_evaluator.add_single_ground_truth_image_info(
image_id='image2',
groundtruth_dict=[{
standard_fields.InputDataFields.groundtruth_boxes:
np.array([[50., 50., 100., 100.]]),
standard_fields.InputDataFields.groundtruth_classes:
np.array([1])
}])
video_evaluator.add_single_ground_truth_image_info(
image_id='image3',
groundtruth_dict=[{
standard_fields.InputDataFields.groundtruth_boxes:
np.array([[50., 100., 100., 120.]]),
standard_fields.InputDataFields.groundtruth_classes:
np.array([1])
}])
video_evaluator.add_single_detected_image_info(
image_id='image1',
detections_dict=[{
standard_fields.DetectionResultFields.detection_boxes:
np.array([[100., 100., 200., 200.]]),
standard_fields.DetectionResultFields.detection_scores:
np.array([.8]),
standard_fields.DetectionResultFields.detection_classes:
np.array([1])
}])
video_evaluator.add_single_detected_image_info(
image_id='image2',
detections_dict=[{
standard_fields.DetectionResultFields.detection_boxes:
np.array([[50., 50., 100., 100.]]),
standard_fields.DetectionResultFields.detection_scores:
np.array([.8]),
standard_fields.DetectionResultFields.detection_classes:
np.array([1])
}])
video_evaluator.add_single_detected_image_info(
image_id='image3',
detections_dict=[{
standard_fields.DetectionResultFields.detection_boxes:
np.array([[50., 100., 100., 120.]]),
standard_fields.DetectionResultFields.detection_scores:
np.array([.8]),
standard_fields.DetectionResultFields.detection_classes:
np.array([1])
}])
metrics = video_evaluator.evaluate()
self.assertAlmostEqual(metrics['DetectionBoxes_Precision/mAP'], 1.0)