def test_consistency(self):
reporter = chainer.Reporter()
if self.comm.rank == 0:
multi_iterator = SerialIterator(self.dataset,
self.batchsize,
repeat=False,
shuffle=False)
else:
multi_iterator = None
multi_link = _SemanticSegmentationStubLink(self.labels,
self.initial_count)
multi_evaluator = SemanticSegmentationEvaluator(multi_iterator,
multi_link,
label_names=('cls0',
'cls1',
'cls2'),
comm=self.comm)
reporter.add_observer('target', multi_link)
with reporter:
multi_mean = multi_evaluator.evaluate()
if self.comm.rank != 0:
self.assertEqual(multi_mean, {})
return
single_iterator = SerialIterator(self.dataset,
self.batchsize,
repeat=False,
shuffle=False)
single_link = _SemanticSegmentationStubLink(self.labels)
single_evaluator = SemanticSegmentationEvaluator(single_iterator,
single_link,
label_names=('cls0',
'cls1',
'cls2'))
reporter.add_observer('target', single_link)
with reporter:
single_mean = single_evaluator.evaluate()
self.assertEqual(set(multi_mean.keys()), set(single_mean.keys()))
for key in multi_mean.keys():
np.testing.assert_equal(single_mean[key], multi_mean[key])
class TestSemanticSegmentationEvaluator(unittest.TestCase):
def setUp(self):
self.label_names = ('a', 'b', 'c')
imgs = np.random.uniform(size=(1, 3, 2, 3))
# There are labels for 'a' and 'b', but none for 'c'.
pred_labels = np.array([[[1, 1, 1], [0, 0, 1]]])
gt_labels = np.array([[[1, 0, 0], [0, -1, 1]]])
self.iou_a = 1 / 3
self.iou_b = 2 / 4
self.pixel_accuracy = 3 / 5
self.class_accuracy_a = 1 / 3
self.class_accuracy_b = 2 / 2
self.miou = np.mean((self.iou_a, self.iou_b))
self.mean_class_accuracy = np.mean(
(self.class_accuracy_a, self.class_accuracy_b))
self.dataset = TupleDataset(imgs, gt_labels)
self.link = _SemanticSegmentationStubLink(pred_labels)
self.iterator = SerialIterator(self.dataset,
5,
repeat=False,
shuffle=False)
self.evaluator = SemanticSegmentationEvaluator(self.iterator,
self.link,
self.label_names)
def test_evaluate(self):
reporter = chainer.Reporter()
reporter.add_observer('main', self.link)
with reporter:
eval_ = self.evaluator.evaluate()
# No observation is reported to the current reporter. Instead the
# evaluator collect results in order to calculate their mean.
np.testing.assert_equal(len(reporter.observation), 0)
np.testing.assert_equal(eval_['main/miou'], self.miou)
np.testing.assert_equal(eval_['main/pixel_accuracy'],
self.pixel_accuracy)
np.testing.assert_equal(eval_['main/mean_class_accuracy'],
self.mean_class_accuracy)
np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
np.testing.assert_equal(eval_['main/iou/c'], np.nan)
np.testing.assert_equal(eval_['main/class_accuracy/a'],
self.class_accuracy_a)
np.testing.assert_equal(eval_['main/class_accuracy/b'],
self.class_accuracy_b)
np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)
def test_call(self):
eval_ = self.evaluator()
# main is used as default
np.testing.assert_equal(eval_['main/miou'], self.miou)
np.testing.assert_equal(eval_['main/pixel_accuracy'],
self.pixel_accuracy)
np.testing.assert_equal(eval_['main/mean_class_accuracy'],
self.mean_class_accuracy)
np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
np.testing.assert_equal(eval_['main/iou/c'], np.nan)
np.testing.assert_equal(eval_['main/class_accuracy/a'],
self.class_accuracy_a)
np.testing.assert_equal(eval_['main/class_accuracy/b'],
self.class_accuracy_b)
np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)
def test_evaluator_name(self):
self.evaluator.name = 'eval'
eval_ = self.evaluator()
# name is used as a prefix
np.testing.assert_equal(eval_['eval/main/miou'], self.miou)
np.testing.assert_equal(eval_['eval/main/pixel_accuracy'],
self.pixel_accuracy)
np.testing.assert_equal(eval_['eval/main/mean_class_accuracy'],
self.mean_class_accuracy)
np.testing.assert_equal(eval_['eval/main/iou/a'], self.iou_a)
np.testing.assert_equal(eval_['eval/main/iou/b'], self.iou_b)
np.testing.assert_equal(eval_['eval/main/iou/c'], np.nan)
np.testing.assert_equal(eval_['eval/main/class_accuracy/a'],
self.class_accuracy_a)
np.testing.assert_equal(eval_['eval/main/class_accuracy/b'],
self.class_accuracy_b)
np.testing.assert_equal(eval_['eval/main/class_accuracy/c'], np.nan)
def test_current_report(self):
reporter = chainer.Reporter()
with reporter:
eval_ = self.evaluator()
# The result is reported to the current reporter.
np.testing.assert_equal(reporter.observation, eval_)
class TestSemanticSegmentationEvaluator(unittest.TestCase):
def setUp(self):
self.label_names = ('a', 'b', 'c')
imgs = np.random.uniform(size=(1, 3, 2, 3))
# There are labels for 'a' and 'b', but none for 'c'.
pred_labels = np.array([[[1, 1, 1], [0, 0, 1]]])
gt_labels = np.array([[[1, 0, 0], [0, -1, 1]]])
self.iou_a = 1 / 3
self.iou_b = 2 / 4
self.pixel_accuracy = 3 / 5
self.class_accuracy_a = 1 / 3
self.class_accuracy_b = 2 / 2
self.miou = np.mean((self.iou_a, self.iou_b))
self.mean_class_accuracy = np.mean(
(self.class_accuracy_a, self.class_accuracy_b))
self.dataset = TupleDataset(imgs, gt_labels)
self.link = _SemanticSegmentationStubLink(pred_labels)
self.iterator = SerialIterator(
self.dataset, 5, repeat=False, shuffle=False)
self.evaluator = SemanticSegmentationEvaluator(
self.iterator, self.link, self.label_names)
def test_evaluate(self):
reporter = chainer.Reporter()
reporter.add_observer('main', self.link)
with reporter:
eval_ = self.evaluator.evaluate()
# No observation is reported to the current reporter. Instead the
# evaluator collect results in order to calculate their mean.
np.testing.assert_equal(len(reporter.observation), 0)
np.testing.assert_equal(eval_['main/miou'], self.miou)
np.testing.assert_equal(eval_['main/pixel_accuracy'],
self.pixel_accuracy)
np.testing.assert_equal(eval_['main/mean_class_accuracy'],
self.mean_class_accuracy)
np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
np.testing.assert_equal(eval_['main/iou/c'], np.nan)
np.testing.assert_equal(eval_['main/class_accuracy/a'],
self.class_accuracy_a)
np.testing.assert_equal(eval_['main/class_accuracy/b'],
self.class_accuracy_b)
np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)
def test_call(self):
eval_ = self.evaluator()
# main is used as default
np.testing.assert_equal(eval_['main/miou'], self.miou)
np.testing.assert_equal(eval_['main/pixel_accuracy'],
self.pixel_accuracy)
np.testing.assert_equal(eval_['main/mean_class_accuracy'],
self.mean_class_accuracy)
np.testing.assert_equal(eval_['main/iou/a'], self.iou_a)
np.testing.assert_equal(eval_['main/iou/b'], self.iou_b)
np.testing.assert_equal(eval_['main/iou/c'], np.nan)
np.testing.assert_equal(eval_['main/class_accuracy/a'],
self.class_accuracy_a)
np.testing.assert_equal(eval_['main/class_accuracy/b'],
self.class_accuracy_b)
np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)
def test_evaluator_name(self):
self.evaluator.name = 'eval'
eval_ = self.evaluator()
# name is used as a prefix
np.testing.assert_equal(eval_['eval/main/miou'], self.miou)
np.testing.assert_equal(eval_['eval/main/pixel_accuracy'],
self.pixel_accuracy)
np.testing.assert_equal(eval_['eval/main/mean_class_accuracy'],
self.mean_class_accuracy)
np.testing.assert_equal(eval_['eval/main/iou/a'], self.iou_a)
np.testing.assert_equal(eval_['eval/main/iou/b'], self.iou_b)
np.testing.assert_equal(eval_['eval/main/iou/c'], np.nan)
np.testing.assert_equal(eval_['eval/main/class_accuracy/a'],
self.class_accuracy_a)
np.testing.assert_equal(eval_['eval/main/class_accuracy/b'],
self.class_accuracy_b)
np.testing.assert_equal(eval_['eval/main/class_accuracy/c'], np.nan)
def test_current_report(self):
reporter = chainer.Reporter()
with reporter:
eval_ = self.evaluator()
# The result is reported to the current reporter.
np.testing.assert_equal(reporter.observation, eval_)
