def __init__(self, **kwargs):
super(BoxEvaluator, self).__init__(**kwargs)
self.image_ids = get_image_ids(metadata=self.metadata)
self.resize_length = _RESIZE_LENGTH
self.cnt = 0
self.num_correct = np.zeros(len(self.threshold_list))
self.original_bboxes = get_bounding_boxes(self.metadata)
self.image_sizes = get_image_sizes(self.metadata)
self.gt_bboxes = self._load_resized_boxes(self.original_bboxes)
def evaluate_wsol(scoremap_root,
metadata_root,
mask_root,
dataset_name,
split,
cam_curve_interval=.001):
"""
Compute WSOL performances of predicted heatmaps against ground truth
boxes (CUB, ILSVRC) or masks (OpenImages). For boxes, we compute the
gt-known box accuracy (IoU>=0.5) at the optimal heatmap threshold.
For masks, we compute the area-under-curve of the pixel-wise precision-
recall curve.
Args:
scoremap_root: string. Score maps for each eval image are saved under
the output_path, with the name corresponding to their image_ids.
For example, the heatmap for the image "123/456.JPEG" is expected
to be located at "{output_path}/123/456.npy".
The heatmaps must be numpy arrays of type np.float, with 2
dimensions corresponding to height and width. The height and width
must be identical to those of the original image. The heatmap values
must be in the [0, 1] range. The map must attain values 0.0 and 1.0.
See check_scoremap_validity() in util.py for the exact requirements.
metadata_root: string.
mask_root: string.
dataset_name: string. Supports [CUB, ILSVRC, and OpenImages].
split: string. Supports [train, val, test].
cam_curve_interval: float. Default 0.001. At which threshold intervals
will the heatmaps be evaluated?
Returns:
performance: float. For CUB and ILSVRC, maxboxacc is returned.
For OpenImages, area-under-curve of the precision-recall curve
is returned.
"""
print("Loading and evaluating cams.")
metadata = configure_metadata(metadata_root)
image_ids = get_image_ids(metadata)
cam_threshold_list = list(np.arange(0, 1, cam_curve_interval))
evaluator = {
"OpenImages": MaskEvaluator,
"CUB": BoxEvaluator,
"ILSVRC": BoxEvaluator
}[dataset_name](metadata=metadata,
dataset_name=dataset_name,
split=split,
cam_threshold_list=cam_threshold_list,
mask_root=mask_root)
cam_loader = _get_cam_loader(image_ids, scoremap_root)
for cams, image_ids in cam_loader:
for cam, image_id in zip(cams, image_ids):
evaluator.accumulate(t2n(cam), image_id)
performance = evaluator.compute()
return performance
def test_box_evaluator_boxes_and_sizes(self):
for dataset_name in self._DATASET_NAMES:
for split in self._SPLITS:
metadata = set_metadata(dataset_name=dataset_name, split=split)
evaluator = load_evaluator(
BoxEvaluator,
dataset_name=dataset_name,
split=split,
cam_curve_interval=self._CAM_CURVE_INTERVAL)
image_ids = get_image_ids(metadata)
test_boxes = [
evaluator.original_bboxes[image_id]
for image_id in image_ids[:3]
]
test_sizes = [
evaluator.image_sizes[image_id]
for image_id in image_ids[:3]
]
self.assertEqual(test_boxes,
self._BOX_VERIFICATION[dataset_name][split])
self.assertEqual(test_sizes,
self._SIZE_VERIFICATION[dataset_name][split])
self._check_box_sizes(image_ids, evaluator)