コード例 #1
0
    def test_consistency(self):
        reporter = chainer.Reporter()

        if self.comm.rank == 0:
            multi_iterator = SerialIterator(self.dataset,
                                            self.batchsize,
                                            repeat=False,
                                            shuffle=False)
        else:
            multi_iterator = None
        multi_link = _SemanticSegmentationStubLink(self.labels,
                                                   self.initial_count)
        multi_evaluator = SemanticSegmentationEvaluator(multi_iterator,
                                                        multi_link,
                                                        label_names=('cls0',
                                                                     'cls1',
                                                                     'cls2'),
                                                        comm=self.comm)
        reporter.add_observer('target', multi_link)
        with reporter:
            multi_mean = multi_evaluator.evaluate()

        if self.comm.rank != 0:
            self.assertEqual(multi_mean, {})
            return

        single_iterator = SerialIterator(self.dataset,
                                         self.batchsize,
                                         repeat=False,
                                         shuffle=False)
        single_link = _SemanticSegmentationStubLink(self.labels)
        single_evaluator = SemanticSegmentationEvaluator(single_iterator,
                                                         single_link,
                                                         label_names=('cls0',
                                                                      'cls1',
                                                                      'cls2'))
        reporter.add_observer('target', single_link)
        with reporter:
            single_mean = single_evaluator.evaluate()

        self.assertEqual(set(multi_mean.keys()), set(single_mean.keys()))
        for key in multi_mean.keys():
            np.testing.assert_equal(single_mean[key], multi_mean[key])
コード例 #2
0
class TestSemanticSegmentationEvaluator(unittest.TestCase):
    def setUp(self):
        self.label_names = ('a', 'b', 'c')
        imgs = np.random.uniform(size=(1, 3, 2, 3))
        # There are labels for 'a' and 'b', but none for 'c'.
        pred_labels = np.array([[[1, 1, 1], [0, 0, 1]]])
        gt_labels = np.array([[[1, 0, 0], [0, -1, 1]]])

        self.iou_a = 1 / 3
        self.iou_b = 2 / 4
        self.pixel_accuracy = 3 / 5
        self.class_accuracy_a = 1 / 3
        self.class_accuracy_b = 2 / 2
        self.miou = np.mean((self.iou_a, self.iou_b))
        self.mean_class_accuracy = np.mean(
            (self.class_accuracy_a, self.class_accuracy_b))

        self.dataset = TupleDataset(imgs, gt_labels)
        self.link = _SemanticSegmentationStubLink(pred_labels)
        self.iterator = SerialIterator(self.dataset,
                                       5,
                                       repeat=False,
                                       shuffle=False)
        self.evaluator = SemanticSegmentationEvaluator(self.iterator,
                                                       self.link,
                                                       self.label_names)

    def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('main', self.link)
        with reporter:
            eval_ = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        np.testing.assert_equal(len(reporter.observation), 0)

        np.testing.assert_equal(eval_['main/miou'], self.miou)
        np.testing.assert_equal(eval_['main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)

    def test_call(self):
        eval_ = self.evaluator()
        # main is used as default
        np.testing.assert_equal(eval_['main/miou'], self.miou)
        np.testing.assert_equal(eval_['main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)

    def test_evaluator_name(self):
        self.evaluator.name = 'eval'
        eval_ = self.evaluator()
        # name is used as a prefix
        np.testing.assert_equal(eval_['eval/main/miou'], self.miou)
        np.testing.assert_equal(eval_['eval/main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['eval/main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['eval/main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['eval/main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['eval/main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['eval/main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['eval/main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['eval/main/class_accuracy/c'], np.nan)

    def test_current_report(self):
        reporter = chainer.Reporter()
        with reporter:
            eval_ = self.evaluator()
        # The result is reported to the current reporter.
        np.testing.assert_equal(reporter.observation, eval_)
コード例 #3
0
class TestSemanticSegmentationEvaluator(unittest.TestCase):

    def setUp(self):
        self.label_names = ('a', 'b', 'c')
        imgs = np.random.uniform(size=(1, 3, 2, 3))
        # There are labels for 'a' and 'b', but none for 'c'.
        pred_labels = np.array([[[1, 1, 1], [0, 0, 1]]])
        gt_labels = np.array([[[1, 0, 0], [0, -1, 1]]])

        self.iou_a = 1 / 3
        self.iou_b = 2 / 4
        self.pixel_accuracy = 3 / 5
        self.class_accuracy_a = 1 / 3
        self.class_accuracy_b = 2 / 2
        self.miou = np.mean((self.iou_a, self.iou_b))
        self.mean_class_accuracy = np.mean(
            (self.class_accuracy_a, self.class_accuracy_b))

        self.dataset = TupleDataset(imgs, gt_labels)
        self.link = _SemanticSegmentationStubLink(pred_labels)
        self.iterator = SerialIterator(
            self.dataset, 5, repeat=False, shuffle=False)
        self.evaluator = SemanticSegmentationEvaluator(
            self.iterator, self.link, self.label_names)

    def test_evaluate(self):
        reporter = chainer.Reporter()
        reporter.add_observer('main', self.link)
        with reporter:
            eval_ = self.evaluator.evaluate()

        # No observation is reported to the current reporter. Instead the
        # evaluator collect results in order to calculate their mean.
        np.testing.assert_equal(len(reporter.observation), 0)

        np.testing.assert_equal(eval_['main/miou'], self.miou)
        np.testing.assert_equal(eval_['main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)

    def test_call(self):
        eval_ = self.evaluator()
        # main is used as default
        np.testing.assert_equal(eval_['main/miou'], self.miou)
        np.testing.assert_equal(eval_['main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)

    def test_evaluator_name(self):
        self.evaluator.name = 'eval'
        eval_ = self.evaluator()
        # name is used as a prefix
        np.testing.assert_equal(eval_['eval/main/miou'], self.miou)
        np.testing.assert_equal(eval_['eval/main/pixel_accuracy'],
                                self.pixel_accuracy)
        np.testing.assert_equal(eval_['eval/main/mean_class_accuracy'],
                                self.mean_class_accuracy)
        np.testing.assert_equal(eval_['eval/main/iou/a'], self.iou_a)
        np.testing.assert_equal(eval_['eval/main/iou/b'], self.iou_b)
        np.testing.assert_equal(eval_['eval/main/iou/c'], np.nan)
        np.testing.assert_equal(eval_['eval/main/class_accuracy/a'],
                                self.class_accuracy_a)
        np.testing.assert_equal(eval_['eval/main/class_accuracy/b'],
                                self.class_accuracy_b)
        np.testing.assert_equal(eval_['eval/main/class_accuracy/c'], np.nan)

    def test_current_report(self):
        reporter = chainer.Reporter()
        with reporter:
            eval_ = self.evaluator()
        # The result is reported to the current reporter.
        np.testing.assert_equal(reporter.observation, eval_)