def __init__(self, num_categories, ignored_label, max_instances_per_category, offset, is_thing=None, rescale_predictions=False): """Constructs Panoptic Quality evaluation class. The class provides the interface to Panoptic Quality metrics_fn. Args: num_categories: The number of segmentation categories (or "classes" in the dataset. ignored_label: A category id that is ignored in evaluation, e.g. the void label as defined in COCO panoptic segmentation dataset. max_instances_per_category: The maximum number of instances for each category. Used in ensuring unique instance labels. offset: The maximum number of unique labels. This is used, by multiplying the ground-truth labels, to generate unique ids for individual regions of overlap between groundtruth and predicted segments. is_thing: A boolean array of length `num_categories`. The entry `is_thing[category_id]` is True iff that category is a "thing" category instead of "stuff." Default to `None`, and it means categories are not classified into these two categories. rescale_predictions: `bool`, whether to scale back prediction to original image sizes. If True, groundtruths['image_info'] is used to rescale predictions. """ self._pq_metric_module = panoptic_quality.PanopticQuality( num_categories, ignored_label, max_instances_per_category, offset) self._is_thing = is_thing self._rescale_predictions = rescale_predictions self._required_prediction_fields = ['category_mask', 'instance_mask'] self._required_groundtruth_fields = ['category_mask', 'instance_mask'] self.reset_states()
def test_instance_order_is_arbitrary(self): category_mask = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 2, 2, 1, 2, 2], [1, 2, 2, 1, 2, 2], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) groundtruth_instance_mask = np.array([ [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 1, 1, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.uint16) predicted_instance_mask = np.array([ [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.uint16) groundtruths = { 'category_mask': category_mask, 'instance_mask': groundtruth_instance_mask } predictions = { 'category_mask': category_mask, 'instance_mask': predicted_instance_mask } pq_metric = panoptic_quality.PanopticQuality( num_categories=3, ignored_label=0, max_instances_per_category=10, offset=100) pq_metric.compare_and_accumulate(groundtruths, predictions) np.testing.assert_array_equal(pq_metric.iou_per_class, [0.0, 1.0, 2.0]) np.testing.assert_array_equal(pq_metric.tp_per_class, [0, 1, 2]) np.testing.assert_array_equal(pq_metric.fn_per_class, [0, 0, 0]) np.testing.assert_array_equal(pq_metric.fp_per_class, [0, 0, 0]) results = pq_metric.result() np.testing.assert_array_equal(results['pq_per_class'], [0.0, 1.0, 1.0]) np.testing.assert_array_equal(results['rq_per_class'], [0.0, 1.0, 1.0]) np.testing.assert_array_equal(results['sq_per_class'], [0.0, 1.0, 1.0]) self.assertAlmostEqual(results['All_pq'], 1.0) self.assertAlmostEqual(results['All_rq'], 1.0) self.assertAlmostEqual(results['All_sq'], 1.0) self.assertEqual(results['All_num_categories'], 2)
def test_totally_wrong(self): category_mask = np.array([ [0, 0, 0, 0, 0, 0], [0, 1, 0, 0, 1, 0], [0, 1, 1, 1, 1, 0], [0, 1, 1, 1, 1, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.uint16) instance_mask = np.zeros([6, 6], np.uint16) groundtruths = { 'category_mask': category_mask, 'instance_mask': instance_mask } predictions = { 'category_mask': 1 - category_mask, 'instance_mask': instance_mask } pq_metric = panoptic_quality.PanopticQuality( num_categories=2, ignored_label=2, max_instances_per_category=1, offset=16) pq_metric.compare_and_accumulate(groundtruths, predictions) np.testing.assert_array_equal(pq_metric.iou_per_class, [0.0, 0.0]) np.testing.assert_array_equal(pq_metric.tp_per_class, [0, 0]) np.testing.assert_array_equal(pq_metric.fn_per_class, [1, 1]) np.testing.assert_array_equal(pq_metric.fp_per_class, [1, 1]) results = pq_metric.result() np.testing.assert_array_equal(results['pq_per_class'], [0.0, 0.0]) np.testing.assert_array_equal(results['rq_per_class'], [0.0, 0.0]) np.testing.assert_array_equal(results['sq_per_class'], [0.0, 0.0]) self.assertAlmostEqual(results['All_pq'], 0.0) self.assertAlmostEqual(results['All_rq'], 0.0) self.assertAlmostEqual(results['All_sq'], 0.0) self.assertEqual(results['All_num_categories'], 2)
def test_matches_by_iou(self): groundtruth_instance_mask = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 2, 2, 2, 1], [1, 2, 2, 2, 2, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) good_det_instance_mask = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 2, 2, 2, 2, 1], [1, 2, 2, 2, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) groundtruths = { 'category_mask': np.zeros_like(groundtruth_instance_mask), 'instance_mask': groundtruth_instance_mask } predictions = { 'category_mask': np.zeros_like(good_det_instance_mask), 'instance_mask': good_det_instance_mask } pq_metric = panoptic_quality.PanopticQuality( num_categories=1, ignored_label=2, max_instances_per_category=16, offset=16) pq_metric.compare_and_accumulate(groundtruths, predictions) # iou(1, 1) = 28/30 # iou(2, 2) = 6 / 8 np.testing.assert_array_almost_equal(pq_metric.iou_per_class, [28 / 30 + 6 / 8]) np.testing.assert_array_equal(pq_metric.tp_per_class, [2]) np.testing.assert_array_equal(pq_metric.fn_per_class, [0]) np.testing.assert_array_equal(pq_metric.fp_per_class, [0]) results = pq_metric.result() np.testing.assert_array_equal(results['pq_per_class'], [(28 / 30 + 6 / 8) / 2]) np.testing.assert_array_equal(results['rq_per_class'], [1.0]) np.testing.assert_array_equal(results['sq_per_class'], [(28 / 30 + 6 / 8) / 2]) self.assertAlmostEqual(results['All_pq'], (28 / 30 + 6 / 8) / 2) self.assertAlmostEqual(results['All_rq'], 1.0) self.assertAlmostEqual(results['All_sq'], (28 / 30 + 6 / 8) / 2) self.assertEqual(results['All_num_categories'], 1) bad_det_instance_mask = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) predictions['instance_mask'] = bad_det_instance_mask pq_metric.reset() pq_metric.compare_and_accumulate(groundtruths, predictions) # iou(1, 1) = 27/32 np.testing.assert_array_almost_equal(pq_metric.iou_per_class, [27 / 32]) np.testing.assert_array_equal(pq_metric.tp_per_class, [1]) np.testing.assert_array_equal(pq_metric.fn_per_class, [1]) np.testing.assert_array_equal(pq_metric.fp_per_class, [1]) results = pq_metric.result() np.testing.assert_array_equal(results['pq_per_class'], [27 / 32 / 2]) np.testing.assert_array_equal(results['rq_per_class'], [0.5]) np.testing.assert_array_equal(results['sq_per_class'], [27 / 32]) self.assertAlmostEqual(results['All_pq'], 27 / 32 / 2) self.assertAlmostEqual(results['All_rq'], 0.5) self.assertAlmostEqual(results['All_sq'], 27 / 32) self.assertEqual(results['All_num_categories'], 1)