def test_enter_exit_threadsafe(self): # This test ensures reporter.__enter__ correctly stores the reporter # in the thread-local storage. def thread_func(reporter, record): with reporter: # Sleep for a tiny moment to cause an overlap of the context # managers. time.sleep(0.01) record.append(chainer.get_current_reporter()) record1 = [] # The current repoter in each thread is stored here. record2 = [] reporter1 = chainer.Reporter() reporter2 = chainer.Reporter() thread1 = threading.Thread( target=thread_func, args=(reporter1, record1)) thread2 = threading.Thread( target=thread_func, args=(reporter2, record2)) thread1.daemon = True thread2.daemon = True thread1.start() thread2.start() thread1.join() thread2.join() self.assertIs(record1[0], reporter1) self.assertIs(record2[0], reporter2)
def test_enter_exit(self): reporter1 = chainer.Reporter() reporter2 = chainer.Reporter() with reporter1: self.assertIs(chainer.get_current_reporter(), reporter1) with reporter2: self.assertIs(chainer.get_current_reporter(), reporter2) self.assertIs(chainer.get_current_reporter(), reporter1)
def test_scope(self): reporter1 = chainer.Reporter() reporter2 = chainer.Reporter() with reporter1: observation = {} with reporter2.scope(observation): self.assertIs(chainer.get_current_reporter(), reporter2) self.assertIs(reporter2.observation, observation) self.assertIs(chainer.get_current_reporter(), reporter1) self.assertIsNot(reporter2.observation, observation)
def test_evaluate(self, backend_config): data = backend_config.get_array(self.data) batches = [backend_config.get_array(b) for b in self.batches] device = backend_config.device iterator, converter, target, evaluator = (self.prepare( data, batches, device)) reporter = chainer.Reporter() reporter.add_observer('target', target) with reporter: mean = evaluator.evaluate() # The converter gets results of the iterator and the device number. self.assertEqual(len(converter.args), len(data)) if backend_config.use_cuda: expected_device_arg = backend_config.cuda_device else: expected_device_arg = -1 for i in range(len(data)): numpy.testing.assert_array_equal( _cpu._to_cpu(converter.args[i]['batch']), self.data[i]) self.assertEqual(converter.args[i]['device'], expected_device_arg) # The model gets results of converter. self.assertEqual(len(target.args), len(batches)) for i in range(len(batches)): numpy.testing.assert_array_equal(_cpu._to_cpu(target.args[i]), self.batches[i]) expect_mean = numpy.mean([numpy.sum(x) for x in self.batches]) self.assertAlmostEqual(_cpu._to_cpu(mean['target/loss']), expect_mean, places=4)
def _test_roc_auc_evaluator_with_labels(data1): """test `pos_labels` and `ignore_labels` behavior""" predictor = DummyPredictor() dataset = NumpyTupleDataset(*data1) iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False) evaluator = ROCAUCEvaluator( iterator, predictor, name='val', pos_labels=[1, 2], ignore_labels=-1, ) # --- test evaluate --- repo = chainer.Reporter() repo.add_observer('target', predictor) with repo: observation = evaluator.evaluate() expected_roc_auc = 0.75 # print('observation ', observation) assert observation['target/roc_auc'] == expected_roc_auc # --- test __call__ --- result = evaluator() # print('result ', result) assert result['val/main/roc_auc'] == expected_roc_auc
def test_keep_graph(self): x = chainer.Variable(numpy.array([1], numpy.float32)) y = functions.sigmoid(x) reporter = chainer.Reporter() with self._scope(True): reporter.report({'y': y}) assert reporter.observation['y'].creator is not None
def test_not_keep_graph(self): x = chainer.Variable(numpy.array([1], numpy.float32)) y = functions.Sigmoid()(x) reporter = chainer.Reporter() with self._scope(False): reporter.report({'y': y}) self.assertIsNone(reporter.observation['y'].creator)
def test_report_without_observer(self): reporter = chainer.Reporter() reporter.report({'x': 1}) observation = reporter.observation self.assertIn('x', observation) self.assertEqual(observation['x'], 1)
def test_report_key(self, metrics_fun, compute_metrics): repo = chainer.Reporter() link = Regressor(predictor=DummyPredictor(), metrics_fun=metrics_fun) link.compute_metrics = compute_metrics repo.add_observer('target', link) with repo: observation = {} with reporter.report_scope(observation): link(self.x, self.t) # print('observation ', observation) actual_keys = set(observation.keys()) if compute_metrics: if metrics_fun is None: assert set(['target/loss']) == actual_keys elif isinstance(metrics_fun, dict): assert set(['target/loss', 'target/user_key']) == actual_keys elif callable(metrics_fun): assert set(['target/loss', 'target/metrics']) == actual_keys else: raise TypeError() else: assert set(['target/loss']) == actual_keys
def test_keep_graph_default(self): x = chainer.Variable(numpy.array([1], numpy.float32)) y, = functions.Sigmoid().apply((x, )) reporter = chainer.Reporter() with self._scope(None): reporter.report({'y': y}) self.assertIsNone(reporter.observation['y'].creator)
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.target) with reporter: mean = self.evaluator.evaluate() # No observation is reported to the current reporter. Instead the # evaluator collect results in order to calculate their mean. self.assertEqual(len(reporter.observation), 0) # The converter gets results of the iterator. self.assertEqual(len(self.converter.args), len(self.data)) for i in range(len(self.data)): numpy.testing.assert_array_equal(self.converter.args[i]['batch'], self.data[i]) self.assertIsNone(self.converter.args[i]['device']) # The model gets results of converter. self.assertEqual(len(self.target.args), len(self.batches)) for i in range(len(self.batches)): numpy.testing.assert_array_equal(self.target.args[i], self.batches[i]) self.assertAlmostEqual(mean['target/loss'], self.expect_mean, places=4) self.evaluator.finalize() self.assertTrue(self.iterator.finalized)
def test_report_with_observer(self): reporter = chainer.Reporter() observer = object() reporter.add_observer('o', observer) with reporter: chainer.report({'x': 1}, observer) observation = reporter.observation self.assertIn('o/x', observation) self.assertEqual(observation['o/x'], 1)
def test_report_scope(self): reporter = chainer.Reporter() observation = {} with reporter: with chainer.report_scope(observation): chainer.report({'x': 1}) self.assertIn('x', observation) self.assertEqual(observation['x'], 1) self.assertNotIn('x', reporter.observation)
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.link) with reporter: mean = self.evaluator.evaluate() # No observation is reported to the current reporter. Instead the # evaluator collect results in order to calculate their mean. self.assertEqual(len(reporter.observation), 0) self.assertEqual(mean['target/map'], self.expect_map)
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.target) with reporter: self.evaluator.evaluate() # The model gets results of converter. self.assertEqual(len(self.target.args), len(self.batches)) for i in range(len(self.batches)): numpy.testing.assert_array_equal(self.target.args[i], self.batches[i])
def test_image(self): img = np.zeros(750).reshape((10, 5, 5, 3)) with self.assertRaises(IndexError) as e: summary.image(img, 'test', ch_axis=-1) assert 'out of range' in str(e.exception) reporter = chainer.Reporter() observer = summary.chainerui_image_observer reporter.add_observer(summary.CHAINERUI_IMAGE_PREFIX, observer) with reporter.scope(observer.observation): summary.image(img, 'test', ch_axis=-1) assert len(observer.observation) == 1 key = summary.CHAINERUI_IMAGE_PREFIX+'/test' assert key in observer.observation img2 = np.zeros(750).reshape((10, 5, 5, 3)) img2[0, 0, 0, 1] = 1 with reporter.scope(observer.observation): summary.image(img2, 'test', ch_axis=-1) assert len(observer.observation) == 1 assert key in observer.observation assert np.allclose(observer.observation[key]['array'], img2) assert 'row' not in observer.observation[key] assert 'mode' not in observer.observation[key] img3 = np.zeros(750).reshape((10, 5, 5, 3)) img3[0, 0, 0, 2] = 1 img3 = chainer.Variable(img3) with reporter.scope(observer.observation): summary.image(img3, row=5, ch_axis=-1, mode='hsv') assert len(observer.observation) == 2 none_key = summary.CHAINERUI_IMAGE_PREFIX+'/0' assert none_key in observer.observation assert np.allclose(observer.observation[none_key]['array'], img3.data) assert observer.observation[none_key]['row'] == 5 assert observer.observation[none_key]['mode'] == 'hsv' img4 = np.zeros(750).reshape((10, 3, 5, 5)) img4[0, 0, 1, 0] = 1 with reporter.scope(observer.observation): summary.image(img4, 'test') assert len(observer.observation) == 2 expected_img4 = img4.transpose(0, 2, 3, 1) assert np.allclose(observer.observation[key]['array'], expected_img4) assert 'row' not in observer.observation[key] assert 'mode' not in observer.observation[key]
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.target) with reporter: mean = self.evaluator.evaluate() # The model gets results of converter. self.assertEqual(len(self.target.args), len(self.batches)) for i in range(len(self.batches)): numpy.testing.assert_array_equal(self.target.args[i][0], self.batches[i]['x']) numpy.testing.assert_array_equal(self.target.args[i][1], self.batches[i]['y']) expect_mean = numpy.mean( [numpy.sum(x['x']) + numpy.sum(x['y']) for x in self.batches]) self.assertAlmostEqual(mean['target/loss'], expect_mean, places=4)
def _test_prc_auc_evaluator_raise_error(data, raise_value_error=True): predictor = DummyPredictor() dataset = NumpyTupleDataset(*data) iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False) evaluator = PRCAUCEvaluator( iterator, predictor, name='train', pos_labels=1, ignore_labels=None, raise_value_error=raise_value_error ) repo = chainer.Reporter() repo.add_observer('target', predictor) with repo: observation = evaluator.evaluate() return observation['target/prc_auc']
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.link) with reporter: mean = self.evaluator.evaluate() # No observation is reported to the current reporter. Instead the # evaluator collect results in order to calculate their mean. self.assertEqual(len(reporter.observation), 0) key = 'ap/iou=0.50:0.95/area=all/max_dets=100' np.testing.assert_equal(mean['target/m{}'.format(key)], self.expected_ap) np.testing.assert_equal(mean['target/{}/cls0'.format(key)], np.nan) np.testing.assert_equal(mean['target/{}/cls1'.format(key)], np.nan) np.testing.assert_equal(mean['target/{}/cls2'.format(key)], self.expected_ap)
def _test_r2_score_evaluator(inputs): predictor = DummyPredictor() x0, x1, _ = inputs dataset = NumpyTupleDataset(x0, x1) iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False) evaluator = R2ScoreEvaluator(iterator, predictor, name='train') repo = chainer.Reporter() repo.add_observer('target', predictor) with repo: observation = evaluator.evaluate() expected = r2_score(x0, x1) pytest.approx(observation['target/r2_score'][0], expected) # --- test __call__ --- result = evaluator() pytest.approx(result['train/main/r2_score'][0], expected)
def __init__(self, model, iterator, optimizers, converter, device=None, loss_func=None, iterative=False): super(Updater, self).__init__(iterator, optimizers, converter, device, loss_func) self.model = model self.optimizers = optimizers self.iterative = iterative reporter = chainer.Reporter() reporter.add_observer('main', model) reporter.add_observers('main', model.namedlinks(skipself=True)) self.reporter = reporter
def run(): # arguments parser = argparse.ArgumentParser() parser.add_argument('-eid', '--experiment_id', type=str) parser.add_argument('-d', '--model_directory', type=str, default=MODEL_DIRECTORY) parser.add_argument('-dd', '--dataset_directory', type=str, default=DATASET_DIRECTORY) parser.add_argument('-cls', '--class_ids', type=str, default=CLASS_IDS_ALL) parser.add_argument('-s', '--seed', type=int, default=RANDOM_SEED) parser.add_argument('-g', '--gpu', type=int, default=GPU) args = parser.parse_args() directory_output = os.path.join(args.model_directory, args.experiment_id) # set random seed, gpu random.seed(args.seed) np.random.seed(args.seed) cp.random.seed(args.seed) chainer.cuda.get_device(args.gpu).use() # load dataset dataset_test = datasets.ShapeNet(args.dataset_directory, args.class_ids.split(','), 'test') # setup model & optimizer model = models.Model() model.to_gpu() chainer.serializers.load_npz(os.path.join(directory_output, 'model.npz'), model) # evaluate reporter = chainer.Reporter() observation = {} with reporter.scope(observation): training.validation(None, model, dataset_test) for key in sorted(observation.keys()): key_display = key for class_id in CLASS_NAMES.keys(): key_display = key_display.replace(class_id, CLASS_NAMES[class_id]) print '%s: %.4f' % (key_display, observation[key])
def test_add_observers(self): reporter = chainer.Reporter() observer1 = object() reporter.add_observer('o1', observer1) observer2 = object() reporter.add_observer('o2', observer2) reporter.report({'x': 1}, observer1) reporter.report({'y': 2}, observer2) observation = reporter.observation self.assertIn('o1/x', observation) self.assertEqual(observation['o1/x'], 1) self.assertIn('o2/y', observation) self.assertEqual(observation['o2/y'], 2) self.assertNotIn('x', observation) self.assertNotIn('y', observation) self.assertNotIn('o1/y', observation) self.assertNotIn('o2/x', observation)
def test_consistency(self): reporter = chainer.Reporter() if self.comm.rank == 0: multi_iterator = SerialIterator(self.dataset, self.batchsize, repeat=False, shuffle=False) else: multi_iterator = None multi_link = _SemanticSegmentationStubLink(self.labels, self.initial_count) multi_evaluator = SemanticSegmentationEvaluator(multi_iterator, multi_link, label_names=('cls0', 'cls1', 'cls2'), comm=self.comm) reporter.add_observer('target', multi_link) with reporter: multi_mean = multi_evaluator.evaluate() if self.comm.rank != 0: self.assertEqual(multi_mean, {}) return single_iterator = SerialIterator(self.dataset, self.batchsize, repeat=False, shuffle=False) single_link = _SemanticSegmentationStubLink(self.labels) single_evaluator = SemanticSegmentationEvaluator(single_iterator, single_link, label_names=('cls0', 'cls1', 'cls2')) reporter.add_observer('target', single_link) with reporter: single_mean = single_evaluator.evaluate() self.assertEqual(set(multi_mean.keys()), set(single_mean.keys())) for key in multi_mean.keys(): np.testing.assert_equal(single_mean[key], multi_mean[key])
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('target', self.target) with reporter: mean = self.evaluator.evaluate() # The converter gets results of the iterator and the device number. self.assertEqual(len(self.converter.args), len(self.data)) for i in range(len(self.data)): numpy.testing.assert_array_equal(self.converter.args[i]['batch'], self.data[i]) self.assertEqual(self.converter.args[i]['device'], 1) # The model gets results of converter. self.assertEqual(len(self.target.args), len(self.batches)) for i in range(len(self.batches)): numpy.testing.assert_array_equal(self.target.args[i], self.batches[i]) expect_mean = numpy.mean([numpy.sum(x) for x in self.batches]) self.assertAlmostEqual(mean['target/loss'], expect_mean, places=4)
def _test_prc_auc_evaluator_default_args(data0): predictor = DummyPredictor() dataset = NumpyTupleDataset(*data0) iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False) evaluator = PRCAUCEvaluator( iterator, predictor, name='train', pos_labels=1, ignore_labels=None ) repo = chainer.Reporter() repo.add_observer('target', predictor) with repo: observation = evaluator.evaluate() expected_prc_auc = 0.7916 pytest.approx(observation['target/prc_auc'], expected_prc_auc) # --- test __call__ --- result = evaluator() pytest.approx(result['train/main/prc_auc'], expected_prc_auc)
def setup_support( batch_size: int, gpu: Optional[int], model: Model, dataset: chainer.dataset.DatasetMixin, ): optimizer = _create_optimizer(model) train_iter = MultiprocessIterator(dataset, batch_size) if gpu is not None: model.to_gpu(gpu) updater = StandardUpdater( device=gpu, iterator=train_iter, optimizer=optimizer, converter=concat_optional, ) reporter = chainer.Reporter() reporter.add_observer('main', model) return updater, reporter
def _test_roc_auc_evaluator_default_args(data0): predictor = DummyPredictor() dataset = NumpyTupleDataset(*data0) iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False) evaluator = ROCAUCEvaluator( iterator, predictor, name='train', pos_labels=1, ignore_labels=None ) repo = chainer.Reporter() repo.add_observer('target', predictor) with repo: observation = evaluator.evaluate() expected_roc_auc = 0.75 # print('observation ', observation) assert observation['target/roc_auc'] == expected_roc_auc # --- test __call__ --- result = evaluator() # print('result ', result) assert result['train/main/roc_auc'] == expected_roc_auc
def test_evaluate(self): reporter = chainer.Reporter() reporter.add_observer('main', self.link) with reporter: eval_ = self.evaluator.evaluate() # No observation is reported to the current reporter. Instead the # evaluator collect results in order to calculate their mean. np.testing.assert_equal(len(reporter.observation), 0) np.testing.assert_equal(eval_['main/miou'], self.miou) np.testing.assert_equal(eval_['main/pixel_accuracy'], self.pixel_accuracy) np.testing.assert_equal(eval_['main/mean_class_accuracy'], self.mean_class_accuracy) np.testing.assert_equal(eval_['main/iou/a'], self.iou_a) np.testing.assert_equal(eval_['main/iou/b'], self.iou_b) np.testing.assert_equal(eval_['main/iou/c'], np.nan) np.testing.assert_equal(eval_['main/class_accuracy/a'], self.class_accuracy_a) np.testing.assert_equal(eval_['main/class_accuracy/b'], self.class_accuracy_b) np.testing.assert_equal(eval_['main/class_accuracy/c'], np.nan)
def _test_prc_auc_evaluator_with_labels(data1): """test `pos_labels` and `ignore_labels` behavior""" predictor = DummyPredictor() dataset = NumpyTupleDataset(*data1) iterator = SerialIterator(dataset, 2, repeat=False, shuffle=False) evaluator = PRCAUCEvaluator( iterator, predictor, name='val', pos_labels=[1, 2], ignore_labels=-1, ) # --- test evaluate --- repo = chainer.Reporter() repo.add_observer('target', predictor) with repo: observation = evaluator.evaluate() expected_prc_auc = 0.7916 pytest.approx(observation['target/prc_auc'], expected_prc_auc) # --- test __call__ --- result = evaluator() pytest.approx(result['val/main/prc_auc'], expected_prc_auc)