def test_train_with_validation_on_odd_gap_check_recorded(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (0.66, 80.3) trainable.evaluate_validation_set.return_value = (1.25, 50.1) train_watcher = _TrainWatcherRecorder() (epochs, validation_epochs) = trainer.train( trainable, np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), train_watcher, epochs=7, validation_gap=5, ) self.assertEqual(epochs, 7) self.assertEqual(validation_epochs, 2) self.assertEqual(train_watcher.epochs, list(range(7))) self.assertEqual(train_watcher.costs, [0.66] * 7) self.assertEqual(train_watcher.accuracies, [80.3] * 7) self.assertEqual(train_watcher.validation_epochs, [0, 5]) self.assertEqual(train_watcher.validation_costs, [1.25] * 2) self.assertEqual(train_watcher.validation_accuracies, [50.1] * 2)
def test_train_with_validation_assert_data_is_passed(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (0.09, 100) trainable.evaluate_validation_set.return_value = (0.10, 60.0) trainer.train( trainable, np.array([0, 9, 8, 7, 6]), np.array([1, 2, 3, 4, 5]), np.array([1, 2, 3]), np.array([0, 0.5, 0.7]), None, epochs=1, validation_gap=1, minibatch_size=5, seed=0, ) self.assertEqual(trainable.update_params.call_count, 1) self.assertEqual( trainable.evaluate_validation_set.call_count, 1, ) ((validation_dataset, validation_labels), _) = \ trainable.evaluate_validation_set.call_args self.assertTrue(np.array_equal( validation_dataset, np.array([1, 2, 3]), )) self.assertTrue(np.array_equal( validation_labels, np.array([0, 0.5, 0.7]), ))
def test_train_multiple_epoch_with_minibatches_check_recorded(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (1.2, 70.5) train_watcher = _TrainWatcherRecorder() (epochs, validation_epochs) = trainer.train( trainable, np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), None, None, train_watcher, epochs=5, minibatch_size=1, ) self.assertEqual(epochs, 5) self.assertEqual(validation_epochs, 0) self.assertEqual(train_watcher.epochs, list(range(5))) self.assertEqual(train_watcher.costs, [1.2] * 5) self.assertEqual(train_watcher.accuracies, [70.5] * 5) self.assertEqual(train_watcher.validation_epochs, []) self.assertEqual(train_watcher.validation_costs, []) self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_assert_returned_epochs_is_zero(self): trainer = SgdTrainer() trainable = MagicMock() epochs, validation_epochs = trainer.train( trainable, np.array([None]), np.array([None]), None, None, None, ) self.assertEqual(epochs, 0) self.assertEqual(validation_epochs, 0)
def test_train_check_recorded(self): arch = MagicMock() arch.update_params.return_value = (0.4, None) model = Model(arch) trainer = SgdTrainer() measurer = MagicMock() measurer.measure.return_value = 60.5 train_watcher = _TrainWatcherRecorder() model.train( np.array([None]), np.array([None]), None, None, trainer, measurer, train_watcher, epochs=8, ) self.assertEqual(train_watcher.epochs, list(range(8))) self.assertEqual(train_watcher.costs, [0.4] * 8) self.assertEqual(train_watcher.accuracies, [60.5] * 8) self.assertEqual(train_watcher.validation_epochs, []) self.assertEqual(train_watcher.validation_costs, []) self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_check_returned(self): arch = MagicMock() arch.update_params.return_value = (0.4, None) model = Model(arch) trainer = SgdTrainer() measurer = MagicMock() measurer.measure.return_value = 60.5 ( epochs, costs, accuracies, validation_epochs, validation_costs, validation_accuracies, ) = model.train( np.array([None]), np.array([None]), None, None, trainer, measurer, None, epochs=8, ) self.assertEqual(epochs, 8) self.assertEqual(costs, [0.4] * 8) self.assertEqual(accuracies, [60.5] * 8) self.assertEqual(validation_epochs, 0) self.assertEqual(validation_costs, []) self.assertEqual(validation_accuracies, [])
def test_train_with_minibatches_check_recorded(self): arch = MagicMock() arch.update_params.return_value = (0.1, None) arch.check_cost.return_value = (0.66, None) model = Model(arch) trainer = SgdTrainer() measurer = MagicMock() measurer.measure.return_value = 90.5 train_watcher = _TrainWatcherRecorder() model.train( np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2]), np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2]), np.array([None]), np.array([None]), trainer, measurer, train_watcher, epochs=9, validation_gap=3, minibatch_size=3, ) self.assertEqual(train_watcher.epochs, list(range(9))) self.assertEqual(train_watcher.costs, [0.1] * 9) self.assertEqual(train_watcher.accuracies, [90.5] * 9) self.assertEqual(train_watcher.validation_epochs, [0, 3, 6]) self.assertEqual(train_watcher.validation_costs, [0.66] * 3) self.assertEqual(train_watcher.validation_accuracies, [90.5] * 3)
def test_train_with_minibatches_check_returned(self): arch = MagicMock() arch.update_params.return_value = (16.7, None) arch.check_cost.return_value = (24.6, None) model = Model(arch) trainer = SgdTrainer() measurer = MagicMock() measurer.measure.return_value = 33.8 ( epochs, costs, accuracies, validation_epochs, validation_costs, validation_accuracies, ) = model.train( np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3]), np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3]), np.array([None]), np.array([None]), trainer, measurer, None, epochs=13, validation_gap=4, minibatch_size=6, ) self.assertEqual(epochs, 13) self.assertEqual(costs, [16.7] * 13) self.assertEqual(accuracies, [33.8] * 13) self.assertEqual(validation_epochs, 4) self.assertEqual(validation_costs, [24.6] * 4) self.assertEqual(validation_accuracies, [33.8] * 4)
def test_train_single_epoch_check_trainable(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (1.0, 0.5) trainer.train( trainable, np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), None, None, None, epochs=1, ) self.assertEqual(trainable.update_params.call_count, 1) self.assertEqual(trainable.evaluate_validation_set.call_count, 0)
def test_train_with_multiple_minibatches_check_trainable(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (1.2, 70.5) trainer.train( trainable, np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), None, None, None, epochs=5, minibatch_size=1, ) self.assertEqual(trainable.update_params.call_count, 15) self.assertEqual(trainable.evaluate_validation_set.call_count, 0)
def get_trainer(trainer, patience=5): if trainer == 'simple': return SimpleTrainer() elif trainer == 'sgd': return SgdTrainer() elif trainer == 'early_stop': return StaticPatienceDecorator(EarlyStopTrainer(), patience) elif trainer == 'sgd_early_stop': return StaticPatienceDecorator(SgdEarlyStopTrainer(), patience)
def test_train_with_validation_check_trainable(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (0.09, 100) trainable.evaluate_validation_set.return_value = (0.10, 60.0) trainer.train( trainable, np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), None, epochs=11, validation_gap=5, ) self.assertEqual(trainable.update_params.call_count, 11) self.assertEqual(trainable.evaluate_validation_set.call_count, 3)
def test_train_assert_watcher_records_are_empty(self): trainer = SgdTrainer() trainable = MagicMock() train_watcher = _TrainWatcherRecorder() trainer.train( trainable, np.array([None]), np.array([None]), None, None, train_watcher, ) self.assertEqual(train_watcher.epochs, []) self.assertEqual(train_watcher.costs, []) self.assertEqual(train_watcher.accuracies, []) self.assertEqual(train_watcher.validation_epochs, []) self.assertEqual(train_watcher.validation_costs, []) self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_with_minibatches_and_measurer_check_accuracies(self): arch = MagicMock() arch.update_params.return_value = ( 0.0, np.array([ [11, 0.8, 33.1], [20, 21.4, 7.6], ]), ) arch.check_cost.return_value = (0.0, np.array([ [0.2, 0.1, 0], [0.8, 0.9, 0.2], [0.0, 2.1, 0.1], [20, 21.4, 7.6], ])) model = Model(arch) trainer = SgdTrainer() _, _, accuracies, _, _, validation_accuracies = model.train( np.array([None, None, None, None]), np.array([ [16, 0.67, 9], [0.3, 0.7, 0.1], [0.3, -34, 1], [20, 21.4, 7.6], ]), np.array([]), np.array([ [20, 21.4, 7.6], [56, 3, 0], [22, 111, 0.2], [0.4, 0.1, 0.6], ]), trainer, ProbsMeasurer(), None, epochs=4, validation_gap=1, minibatch_size=2, seed=0, ) # Remember: Dataset and labels get shuffled, permutation indices: # epoch 0: [2, 3, 1, 0] # epoch 1: [0, 2, 1, 3] # epoch 2: [3, 0, 2, 1] # epoch 2: [1, 0, 2, 3] self.assertEqual(accuracies, [1 / 2, 1 / 4, 0.5, 0.5]) self.assertEqual(validation_accuracies, [1 / 4] * 4)
def test_train_single_epoch_check_recorded(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (0.1, 90.5) train_watcher = _TrainWatcherRecorder() (epochs, validation_epochs) = trainer.train( trainable, np.array([1, 2, 3]), np.array([0.1, 0.2, 0.3]), None, None, train_watcher, epochs=1, ) self.assertEqual(epochs, 1) self.assertEqual(validation_epochs, 0) self.assertEqual(train_watcher.epochs, [0]) self.assertEqual(train_watcher.costs, [0.1]) self.assertEqual(train_watcher.accuracies, [90.5]) self.assertEqual(train_watcher.validation_epochs, []) self.assertEqual(train_watcher.validation_costs, []) self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_assert_data_is_passed_and_shuffled(self): trainer = SgdTrainer() trainable = MagicMock() trainable.update_params.return_value = (0.09, 100) trainer.train( trainable, np.array([[1, 2], [2, 3], [3, 4]]), np.array([0, 2, 4]), None, None, None, epochs=1, minibatch_size=2, seed=0, ) call_args_list = trainable.update_params.call_args_list (first_call, _), (second_call, _) = call_args_list self.assertEqual(trainable.update_params.call_count, 2) self.assertTrue(np.array_equal( first_call[0], np.array([[3, 4], [2, 3]]), )) self.assertTrue(np.array_equal( first_call[1], np.array([4, 2]), )) self.assertTrue(np.array_equal( second_call[0], np.array([[1, 2]]), )) self.assertTrue(np.array_equal( second_call[1], np.array([0]), ))
def test_train_with_minibathes_assert_recorded_and_returned_are_identical( self): arch = MagicMock() arch.update_params.return_value = (0.1, None) arch.check_cost.return_value = (0.66, None) model = Model(arch) trainer = SgdTrainer() measurer = MagicMock() measurer.measure.return_value = 90.5 train_watcher = _TrainWatcherRecorder() ( epochs, costs, accuracies, validation_epochs, validation_costs, validation_accuracies, ) = model.train( np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]), np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]), np.array([None]), np.array([None]), trainer, measurer, train_watcher, epochs=11, validation_gap=1, minibatch_size=4, ) self.assertEqual(train_watcher.epochs, list(range(epochs))) self.assertEqual(train_watcher.costs, costs) self.assertEqual(train_watcher.accuracies, accuracies) self.assertEqual(len(train_watcher.validation_epochs), validation_epochs) self.assertEqual(train_watcher.validation_costs, validation_costs) self.assertEqual(train_watcher.validation_accuracies, validation_accuracies)
def test_train_with_measurer_check_accuracies(self): arch = MagicMock() arch.update_params.return_value = ( 0.0, np.array([ [10, 7.8, -3.1], [2.4, 60, 0.6], [11, 9.2, 45.3], ]), ) model = Model(arch) trainer = SgdTrainer() _, _, accuracies, *_ = model.train( np.array([None, None, None]), np.array([ [11, 6.4, 0.77], [99.9, 99.3, 44.5], [10, 1.2, 19.5], ]), None, None, trainer, ProbsMeasurer(), None, epochs=5, seed=0, ) # Remember: Dataset and labels get shuffled, permutation indices: # epoch 0: [2, 1, 0] # epoch 1: [2, 0, 1] # epoch 2: [0, 2, 1] # epoch 3: [2, 0, 1] # epoch 4: [2, 1, 0] self.assertEqual(accuracies, [0, 0, 1 / 3, 0, 0])