def test_train_with_validation_on_odd_gap_check_recorded(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (0.66, 80.3)
trainable.evaluate_validation_set.return_value = (1.25, 50.1)
train_watcher = _TrainWatcherRecorder()
(epochs, validation_epochs) = trainer.train(
trainable,
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
train_watcher,
epochs=7,
validation_gap=5,
)
self.assertEqual(epochs, 7)
self.assertEqual(validation_epochs, 2)
self.assertEqual(train_watcher.epochs, list(range(7)))
self.assertEqual(train_watcher.costs, [0.66] * 7)
self.assertEqual(train_watcher.accuracies, [80.3] * 7)
self.assertEqual(train_watcher.validation_epochs, [0, 5])
self.assertEqual(train_watcher.validation_costs, [1.25] * 2)
self.assertEqual(train_watcher.validation_accuracies, [50.1] * 2)
def test_train_with_validation_assert_data_is_passed(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (0.09, 100)
trainable.evaluate_validation_set.return_value = (0.10, 60.0)
trainer.train(
trainable,
np.array([0, 9, 8, 7, 6]),
np.array([1, 2, 3, 4, 5]),
np.array([1, 2, 3]),
np.array([0, 0.5, 0.7]),
None,
epochs=1,
validation_gap=1,
minibatch_size=5,
seed=0,
)
self.assertEqual(trainable.update_params.call_count, 1)
self.assertEqual(
trainable.evaluate_validation_set.call_count,
1,
)
((validation_dataset, validation_labels), _) = \
trainable.evaluate_validation_set.call_args
self.assertTrue(np.array_equal(
validation_dataset,
np.array([1, 2, 3]),
))
self.assertTrue(np.array_equal(
validation_labels,
np.array([0, 0.5, 0.7]),
))
def test_train_multiple_epoch_with_minibatches_check_recorded(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (1.2, 70.5)
train_watcher = _TrainWatcherRecorder()
(epochs, validation_epochs) = trainer.train(
trainable,
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
None,
None,
train_watcher,
epochs=5,
minibatch_size=1,
)
self.assertEqual(epochs, 5)
self.assertEqual(validation_epochs, 0)
self.assertEqual(train_watcher.epochs, list(range(5)))
self.assertEqual(train_watcher.costs, [1.2] * 5)
self.assertEqual(train_watcher.accuracies, [70.5] * 5)
self.assertEqual(train_watcher.validation_epochs, [])
self.assertEqual(train_watcher.validation_costs, [])
self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_assert_returned_epochs_is_zero(self):
trainer = SgdTrainer()
trainable = MagicMock()
epochs, validation_epochs = trainer.train(
trainable,
np.array([None]),
np.array([None]),
None,
None,
None,
)
self.assertEqual(epochs, 0)
self.assertEqual(validation_epochs, 0)
def test_train_check_recorded(self):
arch = MagicMock()
arch.update_params.return_value = (0.4, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 60.5
train_watcher = _TrainWatcherRecorder()
model.train(
np.array([None]),
np.array([None]),
None,
None,
trainer,
measurer,
train_watcher,
epochs=8,
)
self.assertEqual(train_watcher.epochs, list(range(8)))
self.assertEqual(train_watcher.costs, [0.4] * 8)
self.assertEqual(train_watcher.accuracies, [60.5] * 8)
self.assertEqual(train_watcher.validation_epochs, [])
self.assertEqual(train_watcher.validation_costs, [])
self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_check_returned(self):
arch = MagicMock()
arch.update_params.return_value = (0.4, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 60.5
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([None]),
np.array([None]),
None,
None,
trainer,
measurer,
None,
epochs=8,
)
self.assertEqual(epochs, 8)
self.assertEqual(costs, [0.4] * 8)
self.assertEqual(accuracies, [60.5] * 8)
self.assertEqual(validation_epochs, 0)
self.assertEqual(validation_costs, [])
self.assertEqual(validation_accuracies, [])
def test_train_with_minibatches_check_recorded(self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
model.train(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2]),
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2]),
np.array([None]),
np.array([None]),
trainer,
measurer,
train_watcher,
epochs=9,
validation_gap=3,
minibatch_size=3,
)
self.assertEqual(train_watcher.epochs, list(range(9)))
self.assertEqual(train_watcher.costs, [0.1] * 9)
self.assertEqual(train_watcher.accuracies, [90.5] * 9)
self.assertEqual(train_watcher.validation_epochs, [0, 3, 6])
self.assertEqual(train_watcher.validation_costs, [0.66] * 3)
self.assertEqual(train_watcher.validation_accuracies, [90.5] * 3)
def test_train_with_minibatches_check_returned(self):
arch = MagicMock()
arch.update_params.return_value = (16.7, None)
arch.check_cost.return_value = (24.6, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 33.8
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3]),
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3]),
np.array([None]),
np.array([None]),
trainer,
measurer,
None,
epochs=13,
validation_gap=4,
minibatch_size=6,
)
self.assertEqual(epochs, 13)
self.assertEqual(costs, [16.7] * 13)
self.assertEqual(accuracies, [33.8] * 13)
self.assertEqual(validation_epochs, 4)
self.assertEqual(validation_costs, [24.6] * 4)
self.assertEqual(validation_accuracies, [33.8] * 4)
def test_train_single_epoch_check_trainable(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (1.0, 0.5)
trainer.train(
trainable,
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
None,
None,
None,
epochs=1,
)
self.assertEqual(trainable.update_params.call_count, 1)
self.assertEqual(trainable.evaluate_validation_set.call_count, 0)
def test_train_with_multiple_minibatches_check_trainable(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (1.2, 70.5)
trainer.train(
trainable,
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
None,
None,
None,
epochs=5,
minibatch_size=1,
)
self.assertEqual(trainable.update_params.call_count, 15)
self.assertEqual(trainable.evaluate_validation_set.call_count, 0)
def get_trainer(trainer, patience=5):
if trainer == 'simple':
return SimpleTrainer()
elif trainer == 'sgd':
return SgdTrainer()
elif trainer == 'early_stop':
return StaticPatienceDecorator(EarlyStopTrainer(), patience)
elif trainer == 'sgd_early_stop':
return StaticPatienceDecorator(SgdEarlyStopTrainer(), patience)
def test_train_with_validation_check_trainable(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (0.09, 100)
trainable.evaluate_validation_set.return_value = (0.10, 60.0)
trainer.train(
trainable,
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
None,
epochs=11,
validation_gap=5,
)
self.assertEqual(trainable.update_params.call_count, 11)
self.assertEqual(trainable.evaluate_validation_set.call_count, 3)
def test_train_assert_watcher_records_are_empty(self):
trainer = SgdTrainer()
trainable = MagicMock()
train_watcher = _TrainWatcherRecorder()
trainer.train(
trainable,
np.array([None]),
np.array([None]),
None,
None,
train_watcher,
)
self.assertEqual(train_watcher.epochs, [])
self.assertEqual(train_watcher.costs, [])
self.assertEqual(train_watcher.accuracies, [])
self.assertEqual(train_watcher.validation_epochs, [])
self.assertEqual(train_watcher.validation_costs, [])
self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_with_minibatches_and_measurer_check_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
0.0,
np.array([
[11, 0.8, 33.1],
[20, 21.4, 7.6],
]),
)
arch.check_cost.return_value = (0.0,
np.array([
[0.2, 0.1, 0],
[0.8, 0.9, 0.2],
[0.0, 2.1, 0.1],
[20, 21.4, 7.6],
]))
model = Model(arch)
trainer = SgdTrainer()
_, _, accuracies, _, _, validation_accuracies = model.train(
np.array([None, None, None, None]),
np.array([
[16, 0.67, 9],
[0.3, 0.7, 0.1],
[0.3, -34, 1],
[20, 21.4, 7.6],
]),
np.array([]),
np.array([
[20, 21.4, 7.6],
[56, 3, 0],
[22, 111, 0.2],
[0.4, 0.1, 0.6],
]),
trainer,
ProbsMeasurer(),
None,
epochs=4,
validation_gap=1,
minibatch_size=2,
seed=0,
)
# Remember: Dataset and labels get shuffled, permutation indices:
# epoch 0: [2, 3, 1, 0]
# epoch 1: [0, 2, 1, 3]
# epoch 2: [3, 0, 2, 1]
# epoch 2: [1, 0, 2, 3]
self.assertEqual(accuracies, [1 / 2, 1 / 4, 0.5, 0.5])
self.assertEqual(validation_accuracies, [1 / 4] * 4)
def test_train_single_epoch_check_recorded(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (0.1, 90.5)
train_watcher = _TrainWatcherRecorder()
(epochs, validation_epochs) = trainer.train(
trainable,
np.array([1, 2, 3]),
np.array([0.1, 0.2, 0.3]),
None,
None,
train_watcher,
epochs=1,
)
self.assertEqual(epochs, 1)
self.assertEqual(validation_epochs, 0)
self.assertEqual(train_watcher.epochs, [0])
self.assertEqual(train_watcher.costs, [0.1])
self.assertEqual(train_watcher.accuracies, [90.5])
self.assertEqual(train_watcher.validation_epochs, [])
self.assertEqual(train_watcher.validation_costs, [])
self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_assert_data_is_passed_and_shuffled(self):
trainer = SgdTrainer()
trainable = MagicMock()
trainable.update_params.return_value = (0.09, 100)
trainer.train(
trainable,
np.array([[1, 2], [2, 3], [3, 4]]),
np.array([0, 2, 4]),
None,
None,
None,
epochs=1,
minibatch_size=2,
seed=0,
)
call_args_list = trainable.update_params.call_args_list
(first_call, _), (second_call, _) = call_args_list
self.assertEqual(trainable.update_params.call_count, 2)
self.assertTrue(np.array_equal(
first_call[0],
np.array([[3, 4], [2, 3]]),
))
self.assertTrue(np.array_equal(
first_call[1],
np.array([4, 2]),
))
self.assertTrue(np.array_equal(
second_call[0],
np.array([[1, 2]]),
))
self.assertTrue(np.array_equal(
second_call[1],
np.array([0]),
))
def test_train_with_minibathes_assert_recorded_and_returned_are_identical(
self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]),
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]),
np.array([None]),
np.array([None]),
trainer,
measurer,
train_watcher,
epochs=11,
validation_gap=1,
minibatch_size=4,
)
self.assertEqual(train_watcher.epochs, list(range(epochs)))
self.assertEqual(train_watcher.costs, costs)
self.assertEqual(train_watcher.accuracies, accuracies)
self.assertEqual(len(train_watcher.validation_epochs),
validation_epochs)
self.assertEqual(train_watcher.validation_costs, validation_costs)
self.assertEqual(train_watcher.validation_accuracies,
validation_accuracies)
def test_train_with_measurer_check_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
0.0,
np.array([
[10, 7.8, -3.1],
[2.4, 60, 0.6],
[11, 9.2, 45.3],
]),
)
model = Model(arch)
trainer = SgdTrainer()
_, _, accuracies, *_ = model.train(
np.array([None, None, None]),
np.array([
[11, 6.4, 0.77],
[99.9, 99.3, 44.5],
[10, 1.2, 19.5],
]),
None,
None,
trainer,
ProbsMeasurer(),
None,
epochs=5,
seed=0,
)
# Remember: Dataset and labels get shuffled, permutation indices:
# epoch 0: [2, 1, 0]
# epoch 1: [2, 0, 1]
# epoch 2: [0, 2, 1]
# epoch 3: [2, 0, 1]
# epoch 4: [2, 1, 0]
self.assertEqual(accuracies, [0, 0, 1 / 3, 0, 0])
