def test_eval_detection_voc(self):
pred_bboxes = self.result['bboxes']
pred_labels = self.result['labels']
pred_scores = self.result['scores']
gt_bboxes = self.dataset['bboxes']
gt_labels = self.dataset['labels']
gt_areas = self.dataset['areas']
gt_crowdeds = self.dataset['crowdeds']
result = eval_detection_coco(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_areas, gt_crowdeds)
expected = {
'map/iou=0.50:0.95/area=all/maxDets=100': 0.5069852,
'map/iou=0.50/area=all/maxDets=100': 0.69937725,
'map/iou=0.75/area=all/maxDets=100': 0.57538619,
'map/iou=0.50:0.95/area=small/maxDets=100': 0.58562572,
'map/iou=0.50:0.95/area=medium/maxDets=100': 0.51939969,
'map/iou=0.50:0.95/area=large/maxDets=100': 0.5013979,
'mar/iou=0.50:0.95/area=all/maxDets=1': 0.38919373,
'mar/iou=0.50:0.95/area=all/maxDets=10': 0.59606053,
'mar/iou=0.50:0.95/area=all/maxDets=100': 0.59773394,
'mar/iou=0.50:0.95/area=small/maxDets=100': 0.63981096,
'mar/iou=0.50:0.95/area=medium/maxDets=100': 0.5664206,
'mar/iou=0.50:0.95/area=large/maxDets=100': 0.5642906
}
for key, item in expected.items():
non_mean_key = key[1:]
self.assertIsInstance(result[non_mean_key], np.ndarray)
self.assertEqual(result[non_mean_key].shape, (76,))
np.testing.assert_almost_equal(
result[key], expected[key], decimal=5)
def eval_boxes(predictions: List[Dict], gts: List[Dict]) -> Dict:
"""Returns the coco evaluation metric for box detection.
Parameters
----------
predictions: List[Dict]
The predictions. Length of the list indicates the number of samples.
Each element in the list are the predictions. Keys must be 'boxes',
'scores', and 'labels'.
gts: List[Dict]
The gts. Length of the list indicates the number of samples.
Each element in the list are the predictions. Keys must be 'boxes',
'scores', and 'labels'.
Returns
-------
eval: Dict:
The results according to the coco metric. At IoU=0.5: VOC metric.
"""
pred_boxes, pred_labels, pred_scores = [], [], []
gt_boxes, gt_labels, gt_areas = [], [], []
for prediction, gt in zip(predictions, gts):
pred_boxes.append(_convert_box(prediction['boxes']))
pred_labels.append(np.array(prediction['labels'], dtype=np.int32))
pred_scores.append(np.array(prediction['scores']))
gt_boxes.append(_convert_box(gt['boxes']))
gt_labels.append(np.array(gt['labels'], dtype=np.int32))
gt_areas.append(np.array(gt['area'], dtype=np.float32))
res = eval_detection_coco(pred_boxes, pred_labels, pred_scores,
gt_boxes, gt_labels, gt_areas)
return res
def eva_coco(dataset, func, limit = 1000, preset = 'evaluate'):
total = limit if limit else len(dataset)
orig_ids = dataset.ids.copy()
dataset.ids = dataset.ids[:total]
iterator = iterators.SerialIterator(dataset, 1, repeat=False, shuffle=False)
in_values, out_values, rest_values = apply_to_iterator(func,
iterator,
hook=ProgressHook(len(dataset)))
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_areas, gt_crowdeds = rest_values
result = eval_detection_coco(pred_bboxes, pred_labels, pred_scores, gt_bboxes, gt_labels, gt_areas, gt_crowdeds)
keys = [
'map/iou=0.50:0.95/area=all/max_dets=100',
'map/iou=0.50/area=all/max_dets=100',
'map/iou=0.75/area=all/max_dets=100',
'map/iou=0.50:0.95/area=small/max_dets=100',
'map/iou=0.50:0.95/area=medium/max_dets=100',
'map/iou=0.50:0.95/area=large/max_dets=100',
'mar/iou=0.50:0.95/area=all/max_dets=1',
'mar/iou=0.50:0.95/area=all/max_dets=10',
'mar/iou=0.50:0.95/area=all/max_dets=100',
'mar/iou=0.50:0.95/area=small/max_dets=100',
'mar/iou=0.50:0.95/area=medium/max_dets=100',
'mar/iou=0.50:0.95/area=large/max_dets=100',
]
print('')
results = []
for key in keys:
print('{:s}: {:f}'.format(key, result[key]))
results.append(result[key])
dataset.ids = orig_ids
return results
def test_area_not_supplied(self):
result = eval_detection_coco(self.pred_bboxes, self.pred_labels,
self.pred_scores, self.gt_bboxes,
self.gt_labels)
self.assertFalse('map/iou=0.50:0.95/area=small/max_dets=100' in result)
self.assertFalse(
'map/iou=0.50:0.95/area=medium/max_dets=100' in result)
self.assertFalse('map/iou=0.50:0.95/area=large/max_dets=100' in result)
def evaluate(self):
target = self._targets['main']
if self.comm is not None and self.comm.rank != 0:
apply_to_iterator(target.predict, None, comm=self.comm)
return {}
iterator = self._iterators['main']
if hasattr(iterator, 'reset'):
iterator.reset()
it = iterator
else:
it = copy.copy(iterator)
in_values, out_values, rest_values = apply_to_iterator(target.predict,
it,
comm=self.comm)
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
if len(rest_values) == 2:
gt_bboxes, gt_labels = rest_values
gt_areas = None
gt_crowdeds = None
elif len(rest_values) == 4:
gt_bboxes, gt_labels, gt_areas, gt_crowdeds =\
rest_values
else:
raise ValueError('the dataset should return '
'sets of (img, bbox, label) or sets of '
'(img, bbox, label, area, crowded).')
result = eval_detection_coco(pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_areas,
gt_crowdeds)
report = {}
for key in result.keys():
if key.startswith('map') or key.startswith('mar'):
report[key] = result[key]
if self.label_names is not None:
for key in result.keys():
if key.startswith('ap') or key.startswith('ar'):
for l, label_name in enumerate(self.label_names):
report_key = '{}/{:s}'.format(key, label_name)
try:
report[report_key] = result[key][l]
except IndexError:
report[report_key] = np.nan
observation = {}
with reporter.report_scope(observation):
reporter.report(report, target)
return observation
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--gpu', type=int, default=-1)
parser.add_argument('--model', choices=('resnet50', 'resnet101'))
parser.add_argument(
'--mean', choices=('chainercv', 'detectron'), default='chainercv')
parser.add_argument('--batchsize', type=int, default=1)
group = parser.add_mutually_exclusive_group()
group.add_argument('--pretrained-model')
group.add_argument('--snapshot')
args = parser.parse_args()
if args.model == 'resnet50':
model = FasterRCNNFPNResNet50(n_fg_class=len(coco_bbox_label_names),
mean=args.mean)
elif args.model == 'resnet101':
model = FasterRCNNFPNResNet101(n_fg_class=len(coco_bbox_label_names),
mean=args.mean)
if args.pretrained_model:
chainer.serializers.load_npz(args.pretrained_model, model)
elif args.snapshot:
chainer.serializers.load_npz(
args.snapshot, model, path='updater/model:main/model/')
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
model.use_preset('evaluate')
dataset = COCOBboxDataset(
split='minival',
use_crowded=True,
return_area=True,
return_crowded=True)
iterator = iterators.MultithreadIterator(
dataset, args.batchsize, repeat=False, shuffle=False)
in_values, out_values, rest_values = apply_to_iterator(
model.predict, iterator, hook=ProgressHook(len(dataset)))
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_area, gt_crowded = rest_values
result = eval_detection_coco(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_area, gt_crowded)
print()
for area in ('all', 'large', 'medium', 'small'):
print('mmAP ({}):'.format(area),
result['map/iou=0.50:0.95/area={}/max_dets=100'.format(area)])
def test(self):
result = eval_detection_coco(self.pred_bboxes, self.pred_labels,
self.pred_scores, self.gt_bboxes,
self.gt_labels)
self.assertEqual(
result['ap/iou=0.50:0.95/area=all/max_dets=100'].shape, (3, ))
self.assertTrue(
np.isnan(result['ap/iou=0.50:0.95/area=all/max_dets=100'][0]))
self.assertEqual(
np.nanmean(result['ap/iou=0.50:0.95/area=all/max_dets=100'][1:]),
result['map/iou=0.50:0.95/area=all/max_dets=100'])
def eval_coco(out_values, rest_values):
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_area, gt_crowded = rest_values
result = eval_detection_coco(pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_area, gt_crowded)
print()
for area in ('all', 'large', 'medium', 'small'):
print('mmAP ({}):'.format(area),
result['map/iou=0.50:0.95/area={}/max_dets=100'.format(area)])
def test_area_specified(self):
result = eval_detection_coco(self.pred_bboxes, self.pred_labels,
self.pred_scores,
self.gt_bboxes, self.gt_labels,
gt_areas=[[2048]])
self.assertFalse(
np.isnan(result['map/iou=0.50:0.95/area=medium/max_dets=100']))
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=small/max_dets=100']))
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=large/max_dets=100']))
def test_crowded(self):
result = eval_detection_coco(self.pred_bboxes, self.pred_labels,
self.pred_scores,
self.gt_bboxes, self.gt_labels,
gt_crowdeds=[[True]])
# When the only ground truth is crowded, nothing is evaluated.
# In that case, all the results are nan.
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=small/maxDets=100']))
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=medium/maxDets=100']))
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=large/maxDets=100']))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--pretrained-model', default='coco')
parser.add_argument('--gpu', type=int, default=-1)
args = parser.parse_args()
model = LightHeadRCNNResNet101(
n_fg_class=len(coco_bbox_label_names),
pretrained_model=args.pretrained_model)
model.use_preset('evaluate')
if args.gpu >= 0:
chainer.cuda.get_device_from_id(args.gpu).use()
model.to_gpu()
dataset = COCOBboxDataset(
split='minival', use_crowded=True,
return_crowded=True, return_area=True)
iterator = iterators.SerialIterator(
dataset, 1, repeat=False, shuffle=False)
in_values, out_values, rest_values = apply_to_iterator(
model.predict, iterator, hook=ProgressHook(len(dataset)))
# delete unused iterators explicitly
del in_values
pred_bboxes, pred_labels, pred_scores = out_values
gt_bboxes, gt_labels, gt_areas, gt_crowdeds = rest_values
result = eval_detection_coco(
pred_bboxes, pred_labels, pred_scores,
gt_bboxes, gt_labels, gt_areas, gt_crowdeds)
keys = [
'map/iou=0.50:0.95/area=all/max_dets=100',
'map/iou=0.50/area=all/max_dets=100',
'map/iou=0.75/area=all/max_dets=100',
'map/iou=0.50:0.95/area=small/max_dets=100',
'map/iou=0.50:0.95/area=medium/max_dets=100',
'map/iou=0.50:0.95/area=large/max_dets=100',
'mar/iou=0.50:0.95/area=all/max_dets=1',
'mar/iou=0.50:0.95/area=all/max_dets=10',
'mar/iou=0.50:0.95/area=all/max_dets=100',
'mar/iou=0.50:0.95/area=small/max_dets=100',
'mar/iou=0.50:0.95/area=medium/max_dets=100',
'mar/iou=0.50:0.95/area=large/max_dets=100',
]
print('')
for key in keys:
print('{:s}: {:f}'.format(key, result[key]))
def test_area_default(self):
result = eval_detection_coco(self.pred_bboxes, self.pred_labels,
self.pred_scores,
self.gt_bboxes, self.gt_labels)
# Test that the original bbox area is used, which is 90.
# In that case, the ground truth bounding box is assigned to segment
# "small".
# Therefore, the score for segments "medium" and "large" will be nan.
self.assertFalse(
np.isnan(result['map/iou=0.50:0.95/area=small/maxDets=100']))
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=medium/maxDets=100']))
self.assertTrue(
np.isnan(result['map/iou=0.50:0.95/area=large/maxDets=100']))
def eval_boxes(predictions, dataset, indices):
"""Returns the coco evaluation metric for box detection.
Parameters
----------
predictions: List[Dict]
The predictions. Length of the list indicates the number of samples.
Each element in the list are the predictions. Keys must be 'boxes',
'scores', and 'labels'.
gts: List[Dict]
The gts. Length of the list indicates the number of samples.
Each element in the list are the predictions. Keys must be 'boxes',
'scores', and 'labels'.
Returns
-------
eval: Dict:
The results according to the coco metric. At IoU=0.5: VOC metric.
"""
# import pdb; pdb.set_trace()
pred_boxes, pred_labels, pred_scores = [], [], []
gt_boxes, gt_labels = [], []
predictions = list(zip(*predictions))
gts = [dataset[i][1] for i in indices]
# sizes = [dataset.pull_item(i)[2:] for i in range(len(dataset))]
images = [dataset[i][0] for i in indices]
for pred, gt in zip(predictions, gts):
tmp_boxes, tmp_labels, tmp_scores = convert_to_box(pred)
pred_boxes.append(tmp_boxes)
pred_labels.append(tmp_labels)
pred_scores.append(tmp_scores)
tmp_boxes, tmp_labels = convert_gt(np.array(gt))
gt_labels.append(tmp_labels)
gt_boxes.append(tmp_boxes)
res = eval_detection_coco(pred_boxes, pred_labels, pred_scores, gt_boxes,
gt_labels)
return res
