def eval_masks(predictions: List[Dict], gts: List[Dict]) -> Dict: """Returns the coco evaluation metric for instance segmentation. Parameters ---------- predictions: List[Dict] The predictions. Length of the list indicates the number of samples. Each element in the list are the predictions. Keys must be 'boxes', 'scores', and 'labels'. gts: List[Dict] The gts. Length of the list indicates the number of samples. Each element in the list are the predictions. Keys must be 'boxes', 'scores', and 'labels'. Returns ------- eval: Dict: The results according to the coco metric """ pred_masks, pred_labels, pred_scores = [], [], [] gt_masks, gt_labels, gt_area = [], [], [] for prediction, gt in zip(predictions, gts): pred_masks.append(_convert_mask(prediction['masks'])) pred_labels.append(np.array(prediction['labels'], dtype=np.int32)) pred_scores.append(np.array(prediction['scores'])) gt_masks.append(_convert_mask(gt['masks'])) gt_labels.append(np.array(gt['labels'], dtype=np.int32)) gt_area.append(np.array(gt['area'], dtype=np.float32)) res = eval_instance_segmentation_coco(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, gt_area) return res
def my_evaluate(self): self._logger.info("Evaluating {} using {} metric. ".format( self._dataset_name, "coco")) pred_masks = [ self._mask_preds[i]["pred_masks"] for i in self._mask_preds ] pred_labels = [ self._mask_preds[i]["pred_labels"] for i in self._mask_preds ] pred_scores = [ self._mask_preds[i]["pred_scores"] for i in self._mask_preds ] gt_masks = [self._mask_preds[i]["gt_masks"] for i in self._mask_preds] gt_labels = [ self._mask_preds[i]["gt_labels"] for i in self._mask_preds ] result = eval_instance_segmentation_coco(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels) result2 = eval_instance_segmentation_voc(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, iou_thresh=0.75) ret = OrderedDict() print(result) ret["instance_segmentation"] = { "mAP": np.mean(result['ap/iou=0.50:0.95/area=all/max_dets=100']), "AP50": np.mean(result['ap/iou=0.50/area=all/max_dets=100']), "AP75": np.mean(result['ap/iou=0.75/area=all/max_dets=100']) } return ret
def evaluate(self): target = self._targets['main'] if self.comm is not None and self.comm.rank != 0: apply_to_iterator(target.predict, None, comm=self.comm) return {} iterator = self._iterators['main'] if hasattr(iterator, 'reset'): iterator.reset() it = iterator else: it = copy.copy(iterator) in_values, out_values, rest_values = apply_to_iterator(target.predict, it, comm=self.comm) # delete unused iterators explicitly del in_values pred_masks, pred_labels, pred_scores = out_values if len(rest_values) == 2: gt_masks, gt_labels = rest_values gt_areas = None gt_crowdeds = None elif len(rest_values) == 4: gt_masks, gt_labels, gt_areas, gt_crowdeds =\ rest_values else: raise ValueError('the dataset should return ' 'sets of (img, mask, label) or sets of ' '(img, mask, label, area, crowded).') result = eval_instance_segmentation_coco(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, gt_areas, gt_crowdeds) report = {} for key in result.keys(): if key.startswith('map') or key.startswith('mar'): report[key] = result[key] if self.label_names is not None: for key in result.keys(): if key.startswith('ap') or key.startswith('ar'): for l, label_name in enumerate(self.label_names): report_key = '{}/{:s}'.format(key, label_name) try: report[report_key] = result[key][l] except IndexError: report[report_key] = np.nan observation = {} with reporter.report_scope(observation): reporter.report(report, target) return observation
def test_area_not_supplied(self): result = eval_instance_segmentation_coco(self.pred_masks, self.pred_labels, self.pred_scores, self.gt_masks, self.gt_labels) self.assertFalse('map/iou=0.50:0.95/area=small/max_dets=100' in result) self.assertFalse( 'map/iou=0.50:0.95/area=medium/max_dets=100' in result) self.assertFalse('map/iou=0.50:0.95/area=large/max_dets=100' in result)
def test(self): result = eval_instance_segmentation_coco(self.pred_masks, self.pred_labels, self.pred_scores, self.gt_masks, self.gt_labels) self.assertEqual( result['ap/iou=0.50:0.95/area=all/max_dets=100'].shape, (3, )) self.assertTrue( np.isnan(result['ap/iou=0.50:0.95/area=all/max_dets=100'][0])) self.assertEqual( np.nanmean(result['ap/iou=0.50:0.95/area=all/max_dets=100'][1:]), result['map/iou=0.50:0.95/area=all/max_dets=100'])
def eval_(out_values, rest_values): pred_masks, pred_labels, pred_scores = out_values gt_masks, gt_labels, gt_areas, gt_crowdeds = rest_values result = eval_instance_segmentation_coco( pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, gt_areas, gt_crowdeds) print() for area in ('all', 'large', 'medium', 'small'): print('mmAP ({}):'.format(area), result['map/iou=0.50:0.95/area={}/max_dets=100'.format( area)])
def test_area_specified(self): result = eval_instance_segmentation_coco(self.pred_masks, self.pred_labels, self.pred_scores, self.gt_masks, self.gt_labels, gt_areas=[[2048]]) self.assertFalse( np.isnan(result['map/iou=0.50:0.95/area=medium/max_dets=100'])) self.assertTrue( np.isnan(result['map/iou=0.50:0.95/area=small/max_dets=100'])) self.assertTrue( np.isnan(result['map/iou=0.50:0.95/area=large/max_dets=100']))
def test_crowded(self): result = eval_instance_segmentation_coco(self.pred_masks, self.pred_labels, self.pred_scores, self.gt_masks, self.gt_labels, gt_crowdeds=[[True]]) # When the only ground truth is crowded, nothing is evaluated. # In that case, all the results are nan. self.assertTrue( np.isnan(result['map/iou=0.50:0.95/area=all/max_dets=100'])) self.assertTrue(np.isnan(result['map/iou=0.50/area=all/max_dets=100'])) self.assertTrue(np.isnan(result['map/iou=0.75/area=all/max_dets=100']))
def test_eval_instance_segmentation_coco(self): pred_masks = self.result['masks'] pred_labels = self.result['labels'] pred_scores = self.result['scores'] gt_masks = self.dataset['masks'] gt_labels = self.dataset['labels'] gt_crowdeds = self.dataset['crowdeds'] gt_areas = self.dataset['areas'] result = eval_instance_segmentation_coco(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, gt_areas, gt_crowdeds) expected = { 'map/iou=0.50:0.95/area=all/max_dets=100': 0.32170935, 'map/iou=0.50/area=all/max_dets=100': 0.56469292, 'map/iou=0.75/area=all/max_dets=100': 0.30133106, 'map/iou=0.50:0.95/area=small/max_dets=100': 0.38737403, 'map/iou=0.50:0.95/area=medium/max_dets=100': 0.31018272, 'map/iou=0.50:0.95/area=large/max_dets=100': 0.32693391, 'mar/iou=0.50:0.95/area=all/max_dets=1': 0.27037258, 'mar/iou=0.50:0.95/area=all/max_dets=10': 0.41759154, 'mar/iou=0.50:0.95/area=all/max_dets=100': 0.41898236, 'mar/iou=0.50:0.95/area=small/max_dets=100': 0.46944986, 'mar/iou=0.50:0.95/area=medium/max_dets=100': 0.37675923, 'mar/iou=0.50:0.95/area=large/max_dets=100': 0.38147151 } non_existent_labels = np.setdiff1d( np.arange(max(result['existent_labels'])), result['existent_labels']) for key, item in expected.items(): non_mean_key = key[1:] self.assertIsInstance(result[non_mean_key], np.ndarray) self.assertEqual(result[non_mean_key].shape, (80, )) self.assertTrue( np.all(np.isnan(result[non_mean_key][non_existent_labels]))) np.testing.assert_almost_equal(result[key], expected[key], decimal=5)
def main(): parser = argparse.ArgumentParser() parser.add_argument('--model', choices=('fcis_resnet101', ), default='fcis_resnet101') parser.add_argument('--pretrained-model', default=None) parser.add_argument('--gpu', type=int, default=-1) args = parser.parse_args() if args.model == 'fcis_resnet101': if args.pretrained_model is None: args.pretrained_model = 'coco' proposal_creator_params = FCISResNet101.proposal_creator_params proposal_creator_params['min_size'] = 2 model = FCISResNet101( n_fg_class=len(coco_instance_segmentation_label_names), anchor_scales=(4, 8, 16, 32), pretrained_model=args.pretrained_model, proposal_creator_params=proposal_creator_params) model.use_preset('coco_evaluate') if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() dataset = COCOInstanceSegmentationDataset(split='minival', year='2014', use_crowded=True, return_crowded=True, return_area=True) iterator = iterators.SerialIterator(dataset, 1, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator(model.predict, iterator, hook=ProgressHook( len(dataset))) # delete unused iterators explicitly del in_values pred_masks, pred_labels, pred_scores = out_values gt_masks, gt_labels, gt_areas, gt_crowdeds = rest_values result = eval_instance_segmentation_coco(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, gt_areas, gt_crowdeds) keys = [ 'map/iou=0.50:0.95/area=all/max_dets=100', 'map/iou=0.50/area=all/max_dets=100', 'map/iou=0.75/area=all/max_dets=100', 'map/iou=0.50:0.95/area=small/max_dets=100', 'map/iou=0.50:0.95/area=medium/max_dets=100', 'map/iou=0.50:0.95/area=large/max_dets=100', 'mar/iou=0.50:0.95/area=all/max_dets=1', 'mar/iou=0.50:0.95/area=all/max_dets=10', 'mar/iou=0.50:0.95/area=all/max_dets=100', 'mar/iou=0.50:0.95/area=small/max_dets=100', 'mar/iou=0.50:0.95/area=medium/max_dets=100', 'mar/iou=0.50:0.95/area=large/max_dets=100', ] print('') for key in keys: print('{:s}: {:f}'.format(key, result[key]))
def main(): parser = argparse.ArgumentParser() parser.add_argument('--pretrained-model') parser.add_argument('--gpu', type=int, default=-1) args = parser.parse_args() proposal_creator_params = { 'nms_thresh': 0.7, 'n_train_pre_nms': 12000, 'n_train_post_nms': 2000, 'n_test_pre_nms': 6000, 'n_test_post_nms': 1000, 'force_cpu_nms': False, 'min_size': 0 } model = FCISPSROIAlignResNet101( n_fg_class=len(coco_instance_segmentation_label_names), min_size=800, max_size=1333, anchor_scales=(2, 4, 8, 16, 32), pretrained_model=args.pretrained_model, proposal_creator_params=proposal_creator_params) model.use_preset('coco_evaluate') if args.gpu >= 0: chainer.cuda.get_device_from_id(args.gpu).use() model.to_gpu() dataset = COCOInstanceSegmentationDataset(split='minival', use_crowded=True, return_crowded=True, return_area=True) iterator = iterators.SerialIterator(dataset, 1, repeat=False, shuffle=False) in_values, out_values, rest_values = apply_to_iterator(model.predict, iterator, hook=ProgressHook( len(dataset))) # delete unused iterators explicitly del in_values pred_masks, pred_labels, pred_scores = out_values gt_masks, gt_labels, gt_areas, gt_crowdeds = rest_values result = eval_instance_segmentation_coco(pred_masks, pred_labels, pred_scores, gt_masks, gt_labels, gt_areas, gt_crowdeds) keys = [ 'map/iou=0.50:0.95/area=all/max_dets=100', 'map/iou=0.50/area=all/max_dets=100', 'map/iou=0.75/area=all/max_dets=100', 'map/iou=0.50:0.95/area=small/max_dets=100', 'map/iou=0.50:0.95/area=medium/max_dets=100', 'map/iou=0.50:0.95/area=large/max_dets=100', 'mar/iou=0.50:0.95/area=all/max_dets=1', 'mar/iou=0.50:0.95/area=all/max_dets=10', 'mar/iou=0.50:0.95/area=all/max_dets=100', 'mar/iou=0.50:0.95/area=small/max_dets=100', 'mar/iou=0.50:0.95/area=medium/max_dets=100', 'mar/iou=0.50:0.95/area=large/max_dets=100', ] print('') for key in keys: print('{:s}: {:f}'.format(key, result[key]))