def test_merge_accumulates_all_across_instances(self): categories = np.zeros([6, 6], np.uint16) good_det_labels = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 2, 2, 2, 2, 1], [1, 2, 2, 2, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) gt_labels = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 2, 2, 2, 1], [1, 2, 2, 2, 2, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) good_pq = panoptic_quality.PanopticQuality( num_categories=1, ignored_label=2, max_instances_per_category=16, offset=16) for _ in six.moves.range(2): good_pq.compare_and_accumulate(categories, gt_labels, categories, good_det_labels) bad_det_labels = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) bad_pq = panoptic_quality.PanopticQuality( num_categories=1, ignored_label=2, max_instances_per_category=16, offset=16) for _ in six.moves.range(2): bad_pq.compare_and_accumulate(categories, gt_labels, categories, bad_det_labels) good_pq.merge(bad_pq) np.testing.assert_array_almost_equal( good_pq.iou_per_class, [2 * (28 / 30 + 6 / 8) + 2 * (27 / 32)]) np.testing.assert_array_equal(good_pq.tp_per_class, [2 * 2 + 2]) np.testing.assert_array_equal(good_pq.fn_per_class, [2]) np.testing.assert_array_equal(good_pq.fp_per_class, [2]) self.assertAlmostEqual(good_pq.result(), 0.63177083)
def test_totally_wrong(self): det_categories = np.array([ [0, 0, 0, 0, 0, 0], [0, 1, 0, 0, 1, 0], [0, 1, 1, 1, 1, 0], [0, 1, 1, 1, 1, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.uint16) gt_categories = 1 - det_categories instances = np.zeros([6, 6], np.uint16) pq = panoptic_quality.PanopticQuality(num_categories=2, ignored_label=2, max_instances_per_category=1, offset=16) pq.compare_and_accumulate(gt_categories, instances, det_categories, instances) np.testing.assert_array_equal(pq.iou_per_class, [0.0, 0.0]) np.testing.assert_array_equal(pq.tp_per_class, [0, 0]) np.testing.assert_array_equal(pq.fn_per_class, [1, 1]) np.testing.assert_array_equal(pq.fp_per_class, [1, 1]) np.testing.assert_array_equal(pq.result_per_category(), [0.0, 0.0]) self.assertEqual(pq.result(), 0.0)
def test_wrong_instances(self): categories = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 2, 2, 1, 2, 2], [1, 2, 2, 1, 2, 2], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) predicted_instances = np.array([ [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 1, 1], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], ], dtype=np.uint16) groundtruth_instances = np.zeros([6, 6], dtype=np.uint16) pq = panoptic_quality.PanopticQuality(num_categories=3, ignored_label=0, max_instances_per_category=10, offset=100) pq.compare_and_accumulate(categories, groundtruth_instances, categories, predicted_instances) np.testing.assert_array_equal(pq.iou_per_class, [0.0, 1.0, 0.0]) np.testing.assert_array_equal(pq.tp_per_class, [0, 1, 0]) np.testing.assert_array_equal(pq.fn_per_class, [0, 0, 1]) np.testing.assert_array_equal(pq.fp_per_class, [0, 0, 2]) np.testing.assert_array_equal(pq.result_per_category(), [0, 1, 0]) self.assertAlmostEqual(pq.result(), 0.5)
def test_matches_by_iou(self): good_det_labels = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 2, 2, 2, 2, 1], [1, 2, 2, 2, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) gt_labels = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 2, 2, 2, 1], [1, 2, 2, 2, 2, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) pq = panoptic_quality.PanopticQuality(num_categories=1, ignored_label=2, max_instances_per_category=16, offset=16) pq.compare_and_accumulate(np.zeros_like(gt_labels), gt_labels, np.zeros_like(good_det_labels), good_det_labels) # iou(1, 1) = 28/30 # iou(2, 2) = 6/8 np.testing.assert_array_almost_equal(pq.iou_per_class, [28 / 30 + 6 / 8]) np.testing.assert_array_equal(pq.tp_per_class, [2]) np.testing.assert_array_equal(pq.fn_per_class, [0]) np.testing.assert_array_equal(pq.fp_per_class, [0]) self.assertAlmostEqual(pq.result(), (28 / 30 + 6 / 8) / 2) bad_det_labels = np.array([ [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 2, 2, 1], [1, 1, 1, 1, 1, 1], ], dtype=np.uint16) pq.reset() pq.compare_and_accumulate(np.zeros_like(gt_labels), gt_labels, np.zeros_like(bad_det_labels), bad_det_labels) # iou(1, 1) = 27/32 np.testing.assert_array_almost_equal(pq.iou_per_class, [27 / 32]) np.testing.assert_array_equal(pq.tp_per_class, [1]) np.testing.assert_array_equal(pq.fn_per_class, [1]) np.testing.assert_array_equal(pq.fp_per_class, [1]) self.assertAlmostEqual(pq.result(), (27 / 32) * (1 / 2))
def _panoptic_quality_helper( groundtruth_category_array, groundtruth_instance_array, predicted_category_array, predicted_instance_array, num_classes, max_instances_per_category, ignored_label, offset): """Helper function to compute panoptic quality.""" pq = panoptic_quality.PanopticQuality(num_classes, ignored_label, max_instances_per_category, offset) pq.compare_and_accumulate(groundtruth_category_array, groundtruth_instance_array, predicted_category_array, predicted_instance_array) return pq.iou_per_class, pq.tp_per_class, pq.fn_per_class, pq.fp_per_class
def _build_metric(metric, num_categories, ignored_label, max_instances_per_category, intersection_offset=None, normalize_by_image_size=True): """Creates a metric aggregator objet of the given name.""" if metric == 'pq': logging.warning('One should check Panoptic Quality results against the ' 'official COCO API code. Small numerical differences ' '(< 0.1%) can be magnified by rounding.') return panoptic_quality.PanopticQuality(num_categories, ignored_label, max_instances_per_category, intersection_offset) elif metric == 'pc': return parsing_covering.ParsingCovering( num_categories, ignored_label, max_instances_per_category, intersection_offset, normalize_by_image_size) else: raise ValueError('No implementation for metric "%s"' % metric)
def test_perfect_match(self): categories = np.zeros([6, 6], np.uint16) instances = np.array([ [1, 1, 1, 1, 1, 1], [1, 2, 2, 2, 2, 1], [1, 2, 2, 2, 2, 1], [1, 2, 2, 2, 2, 1], [1, 2, 2, 1, 1, 1], [1, 2, 1, 1, 1, 1], ], dtype=np.uint16) pq = panoptic_quality.PanopticQuality(num_categories=1, ignored_label=2, max_instances_per_category=16, offset=16) pq.compare_and_accumulate(categories, instances, categories, instances) np.testing.assert_array_equal(pq.iou_per_class, [2.0]) np.testing.assert_array_equal(pq.tp_per_class, [2]) np.testing.assert_array_equal(pq.fn_per_class, [0]) np.testing.assert_array_equal(pq.fp_per_class, [0]) np.testing.assert_array_equal(pq.result_per_category(), [1.0]) self.assertEqual(pq.result(), 1.0)
def test_matches_expected(self): pred_classes = test_utils.read_segmentation_with_rgb_color_map( 'team_pred_class.png', _CLASS_COLOR_MAP) pred_instances = test_utils.read_test_image('team_pred_instance.png', mode='L') instance_class_map = { 0: 0, 47: 1, 97: 1, 133: 1, 150: 1, 174: 1, 198: 2, 215: 1, 244: 1, 255: 1, } gt_instances, gt_classes = test_utils.panoptic_segmentation_with_class_map( 'team_gt_instance.png', instance_class_map) pq = panoptic_quality.PanopticQuality(num_categories=3, ignored_label=0, max_instances_per_category=256, offset=256 * 256) pq.compare_and_accumulate(gt_classes, gt_instances, pred_classes, pred_instances) np.testing.assert_array_almost_equal(pq.iou_per_class, [2.06104, 5.26827, 0.54069], decimal=4) np.testing.assert_array_equal(pq.tp_per_class, [1, 7, 1]) np.testing.assert_array_equal(pq.fn_per_class, [0, 1, 0]) np.testing.assert_array_equal(pq.fp_per_class, [0, 0, 0]) np.testing.assert_array_almost_equal(pq.result_per_category(), [2.061038, 0.702436, 0.54069]) self.assertAlmostEqual(pq.result(), 0.62156287)