def test_train_with_validation_check_recorded(self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SimpleTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
model.train(
None,
None,
None,
None,
trainer,
measurer,
train_watcher,
epochs=9,
validation_gap=3,
)
self.assertEqual(train_watcher.epochs, list(range(9)))
self.assertEqual(train_watcher.costs, [0.1] * 9)
self.assertEqual(train_watcher.accuracies, [90.5] * 9)
self.assertEqual(train_watcher.validation_epochs, [0, 3, 6])
self.assertEqual(train_watcher.validation_costs, [0.66] * 3)
self.assertEqual(train_watcher.validation_accuracies, [90.5] * 3)
def test_train_having_decreasing_cost_instance_check_it_does_not_stops_early(
self):
arch = _PlaybackArch(
[0.05] * 3,
[None] * 3,
[0.1, 0.01, 0.001],
[None] * 3,
)
model = Model(arch)
measurer = MagicMock()
measurer.measure.return_value = 30.1
train_watcher = _TrainWatcherRecorder()
model.train(
None,
None,
None,
None,
EarlyStopTrainer(),
measurer,
train_watcher,
epochs=3,
validation_gap=1,
patience=1,
)
self.assertEqual(train_watcher.epochs, list(range(3)))
self.assertEqual(train_watcher.costs, [0.05] * 3)
self.assertEqual(train_watcher.accuracies, [30.1] * 3)
self.assertEqual(train_watcher.validation_epochs, list(range(3)))
self.assertEqual(train_watcher.validation_costs, [0.1, 0.01, 0.001])
self.assertEqual(train_watcher.validation_accuracies, [30.1] * 3)
self.assertEqual(arch.update_params_call_count, 3)
self.assertEqual(arch.check_cost_call_count, 3)
def test_train_with_measurer_check_accuracies(self):
arch = _PlaybackArch(
[0.05] * 9,
[np.array([[0, 1], [1, 0], [0, 1]])] * 9,
[0.1, 0.01, 0.001],
[np.array([[1, 0], [1, 0]])] * 3,
)
model = Model(arch)
train_watcher = _TrainWatcherRecorder()
model.train(
np.array([[1], [2], [3]]),
np.array([[0, 1], [0, 1], [0, 1]]),
np.array([[1], [2]]),
np.array([[0, 1], [0, 1]]),
SgdEarlyStopTrainer(),
ProbsMeasurer(),
train_watcher,
epochs=3,
validation_gap=1,
minibatch_size=1,
patience=1,
seed=0,
)
self.assertEqual(train_watcher.epochs, list(range(3)))
self.assertEqual(train_watcher.costs, [0.05] * 3)
self.assertEqual(train_watcher.accuracies, [2 / 3] * 3)
self.assertEqual(train_watcher.validation_epochs, list(range(3)))
self.assertEqual(train_watcher.validation_costs, [0.1, 0.01, 0.001])
self.assertEqual(train_watcher.validation_accuracies, [0] * 3)
def test_train_with_minibatches_check_recorded(self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
model.train(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2]),
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2]),
np.array([None]),
np.array([None]),
trainer,
measurer,
train_watcher,
epochs=9,
validation_gap=3,
minibatch_size=3,
)
self.assertEqual(train_watcher.epochs, list(range(9)))
self.assertEqual(train_watcher.costs, [0.1] * 9)
self.assertEqual(train_watcher.accuracies, [90.5] * 9)
self.assertEqual(train_watcher.validation_epochs, [0, 3, 6])
self.assertEqual(train_watcher.validation_costs, [0.66] * 3)
self.assertEqual(train_watcher.validation_accuracies, [90.5] * 3)
def test_train_check_recorded(self):
arch = MagicMock()
arch.update_params.return_value = (0.4, None)
model = Model(arch)
trainer = SimpleTrainer()
measurer = MagicMock()
measurer.measure.return_value = 60.5
train_watcher = _TrainWatcherRecorder()
model.train(
None,
None,
None,
None,
trainer,
measurer,
train_watcher,
epochs=8,
)
self.assertEqual(train_watcher.epochs, list(range(8)))
self.assertEqual(train_watcher.costs, [0.4] * 8)
self.assertEqual(train_watcher.accuracies, [60.5] * 8)
self.assertEqual(train_watcher.validation_epochs, [])
self.assertEqual(train_watcher.validation_costs, [])
self.assertEqual(train_watcher.validation_accuracies, [])
def test_train_check_returned(self):
arch = MagicMock()
arch.update_params.return_value = (0.4, None)
model = Model(arch)
trainer = SimpleTrainer()
measurer = MagicMock()
measurer.measure.return_value = 60.5
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
None,
None,
None,
None,
trainer,
measurer,
None,
epochs=8,
)
self.assertEqual(epochs, 8)
self.assertEqual(costs, [0.4] * 8)
self.assertEqual(accuracies, [60.5] * 8)
self.assertEqual(validation_epochs, 0)
self.assertEqual(validation_costs, [])
self.assertEqual(validation_accuracies, [])
def test_train_with_measurer_check_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
None,
np.array([
[1, 0, -3],
[3, 22, 0],
[1, 2, 0.5],
]),
)
model = Model(arch)
trainer = SimpleTrainer()
_, _, accuracies, *_ = model.train(
None,
np.array([
[6, 3, 0],
[1, 0.5, 8],
[10, -100, 9.5],
]),
None,
None,
trainer,
ProbsMeasurer(),
None,
epochs=7,
)
self.assertEqual(accuracies, [1 / 3] * 7)
def test_train_with_validation_check_returned(self):
arch = MagicMock()
arch.update_params.return_value = (16.7, None)
arch.check_cost.return_value = (24.6, None)
model = Model(arch)
trainer = SimpleTrainer()
measurer = MagicMock()
measurer.measure.return_value = 33.8
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
None,
None,
None,
None,
trainer,
measurer,
None,
epochs=13,
validation_gap=4,
)
self.assertEqual(epochs, 13)
self.assertEqual(costs, [16.7] * 13)
self.assertEqual(accuracies, [33.8] * 13)
self.assertEqual(validation_epochs, 4)
self.assertEqual(validation_costs, [24.6] * 4)
self.assertEqual(validation_accuracies, [33.8] * 4)
def test_train_with_minibatches_check_returned(self):
arch = MagicMock()
arch.update_params.return_value = (16.7, None)
arch.check_cost.return_value = (24.6, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 33.8
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3]),
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3]),
np.array([None]),
np.array([None]),
trainer,
measurer,
None,
epochs=13,
validation_gap=4,
minibatch_size=6,
)
self.assertEqual(epochs, 13)
self.assertEqual(costs, [16.7] * 13)
self.assertEqual(accuracies, [33.8] * 13)
self.assertEqual(validation_epochs, 4)
self.assertEqual(validation_costs, [24.6] * 4)
self.assertEqual(validation_accuracies, [33.8] * 4)
def test_train_check_returned(self):
arch = MagicMock()
arch.update_params.return_value = (0.4, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = EarlyStopTrainer()
measurer = MagicMock()
measurer.measure.return_value = 60.5
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
None,
None,
None,
None,
trainer,
measurer,
None,
epochs=8,
validation_gap=1,
patience=2,
)
self.assertEqual(epochs, 3)
self.assertEqual(costs, [0.4] * 3)
self.assertEqual(accuracies, [60.5] * 3)
self.assertEqual(validation_epochs, 3)
self.assertEqual(validation_costs, [0.66] * 3)
self.assertEqual(validation_accuracies, [60.5] * 3)
def test_train_with_simple_trainer_check_recorded_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
None,
np.array([
[1, 0, -3],
[3, 22, 0],
[1, 2, 0.5],
]),
)
arch.check_cost.return_value = (
None,
np.array([
[10, 1, -9],
[-0.01, -0.8, -0.5],
[11, 20, 0.5],
]),
)
model = Model(arch)
trainer = SimpleTrainer()
recorder = _TrainWatcherRecorder()
model.train(
None,
np.array([
[6, 3, 0],
[1, 0.5, 8],
[10, -100, 9.5],
]),
None,
np.array([
[11, 1, -9],
[-0.02, -0.8, -0.5],
[11, 21, 0.5],
]),
trainer,
ProbsMeasurer(),
recorder,
epochs=7,
validation_gap=1,
)
self.assertEqual(recorder.accuracies, [1 / 3] * 7)
self.assertEqual(recorder.validation_accuracies, [1] * 7)
def test_train_assert_hooks_are_called(self):
arch = MagicMock()
model = Model(arch)
trainer = MagicMock()
trainer.train.return_value = (None, None)
model.train(
None,
None,
None,
None,
trainer,
None,
None,
epochs=0,
)
self.assertEqual(arch.train_initialize.call_count, 1)
self.assertEqual(arch.train_finalize.call_count, 1)
def test_train_with_minibatches_and_measurer_check_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
0.0,
np.array([
[11, 0.8, 33.1],
[20, 21.4, 7.6],
]),
)
arch.check_cost.return_value = (0.0,
np.array([
[0.2, 0.1, 0],
[0.8, 0.9, 0.2],
[0.0, 2.1, 0.1],
[20, 21.4, 7.6],
]))
model = Model(arch)
trainer = SgdTrainer()
_, _, accuracies, _, _, validation_accuracies = model.train(
np.array([None, None, None, None]),
np.array([
[16, 0.67, 9],
[0.3, 0.7, 0.1],
[0.3, -34, 1],
[20, 21.4, 7.6],
]),
np.array([]),
np.array([
[20, 21.4, 7.6],
[56, 3, 0],
[22, 111, 0.2],
[0.4, 0.1, 0.6],
]),
trainer,
ProbsMeasurer(),
None,
epochs=4,
validation_gap=1,
minibatch_size=2,
seed=0,
)
# Remember: Dataset and labels get shuffled, permutation indices:
# epoch 0: [2, 3, 1, 0]
# epoch 1: [0, 2, 1, 3]
# epoch 2: [3, 0, 2, 1]
# epoch 2: [1, 0, 2, 3]
self.assertEqual(accuracies, [1 / 2, 1 / 4, 0.5, 0.5])
self.assertEqual(validation_accuracies, [1 / 4] * 4)
def test_train_assert_recorded_and_returned_are_identical(self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SgdEarlyStopTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([1, 2, 3]),
np.array([1, 0, 0]),
None,
None,
trainer,
measurer,
train_watcher,
epochs=9,
validation_gap=2,
minibatch_size=2,
patience=1,
)
self.assertEqual(epochs, 3)
self.assertEqual(train_watcher.epochs, list(range(epochs)))
self.assertEqual(train_watcher.costs, costs)
self.assertEqual(train_watcher.accuracies, accuracies)
self.assertEqual(len(train_watcher.validation_epochs),
validation_epochs)
self.assertEqual(train_watcher.validation_costs, validation_costs)
self.assertEqual(train_watcher.validation_accuracies,
validation_accuracies)
self.assertEqual(arch.update_params.call_count, 6)
self.assertEqual(arch.check_cost.call_count, 2)
def test_train_check_returned_and_call_count(self):
arch = MagicMock()
arch.update_params.return_value = (0.4, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SgdEarlyStopTrainer()
measurer = MagicMock()
measurer.measure.return_value = 60.5
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([1, 2, 3]),
np.array([0, 1, 0]),
None,
None,
trainer,
measurer,
None,
epochs=8,
validation_gap=1,
minibatch_size=1,
patience=2,
)
self.assertEqual(epochs, 3)
self.assertEqual(costs, [0.4] * 3)
self.assertEqual(accuracies, [60.5] * 3)
self.assertEqual(validation_epochs, 3)
self.assertEqual(validation_costs, [0.66] * 3)
self.assertEqual(validation_accuracies, [60.5] * 3)
self.assertEqual(arch.update_params.call_count, 9)
self.assertEqual(arch.check_cost.call_count, 3)
def test_train_with_validation_and_measurer_check_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
None,
np.array([
[-1, 8, 7],
[-1, 1, 0],
]),
)
arch.check_cost.return_value = (
None,
np.array([
[8, 0, 7],
[-6, 1, 0],
]),
)
model = Model(arch)
trainer = SimpleTrainer()
_, _, accuracies, _, _, validation_accuracies = model.train(
None,
np.array([
[62.9, 63, 0.0],
[11, 0.5, 0.8],
]),
None,
np.array([
[0, 1, 11.0],
[-8, 0, 6.66],
]),
trainer,
ProbsMeasurer(),
None,
epochs=8,
validation_gap=4,
)
self.assertEqual(accuracies, [1 / 2] * 8)
self.assertEqual(validation_accuracies, [0] * 2)
def test_train_with_minibathes_assert_recorded_and_returned_are_identical(
self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SgdTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]),
np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]),
np.array([None]),
np.array([None]),
trainer,
measurer,
train_watcher,
epochs=11,
validation_gap=1,
minibatch_size=4,
)
self.assertEqual(train_watcher.epochs, list(range(epochs)))
self.assertEqual(train_watcher.costs, costs)
self.assertEqual(train_watcher.accuracies, accuracies)
self.assertEqual(len(train_watcher.validation_epochs),
validation_epochs)
self.assertEqual(train_watcher.validation_costs, validation_costs)
self.assertEqual(train_watcher.validation_accuracies,
validation_accuracies)
def test_train_assert_recorded_and_returned_are_identical(self):
arch = MagicMock()
arch.update_params.return_value = (0.1, None)
arch.check_cost.return_value = (0.66, None)
model = Model(arch)
trainer = SimpleTrainer()
measurer = MagicMock()
measurer.measure.return_value = 90.5
train_watcher = _TrainWatcherRecorder()
(
epochs,
costs,
accuracies,
validation_epochs,
validation_costs,
validation_accuracies,
) = model.train(
None,
None,
None,
None,
trainer,
measurer,
train_watcher,
epochs=9,
validation_gap=3,
)
self.assertEqual(train_watcher.epochs, list(range(epochs)))
self.assertEqual(train_watcher.costs, costs)
self.assertEqual(train_watcher.accuracies, accuracies)
self.assertEqual(len(train_watcher.validation_epochs),
validation_epochs)
self.assertEqual(train_watcher.validation_costs, validation_costs)
self.assertEqual(train_watcher.validation_accuracies,
validation_accuracies)
def test_train_with_measurer_check_accuracies(self):
arch = MagicMock()
arch.update_params.return_value = (
0.0,
np.array([
[10, 7.8, -3.1],
[2.4, 60, 0.6],
[11, 9.2, 45.3],
]),
)
model = Model(arch)
trainer = SgdTrainer()
_, _, accuracies, *_ = model.train(
np.array([None, None, None]),
np.array([
[11, 6.4, 0.77],
[99.9, 99.3, 44.5],
[10, 1.2, 19.5],
]),
None,
None,
trainer,
ProbsMeasurer(),
None,
epochs=5,
seed=0,
)
# Remember: Dataset and labels get shuffled, permutation indices:
# epoch 0: [2, 1, 0]
# epoch 1: [2, 0, 1]
# epoch 2: [0, 2, 1]
# epoch 3: [2, 0, 1]
# epoch 4: [2, 1, 0]
self.assertEqual(accuracies, [0, 0, 1 / 3, 0, 0])
error = labels - evaluations
delta = error * (evaluations * (1 - evaluations))
self._weights += np.sum(self._learning_rate * delta *
np.transpose(dataset),
axis=1)
self._bias += np.sum(self._learning_rate * delta)
return (np.average(error), evaluations)
def evaluate(self, dataset):
return sigmoid(np.sum(dataset * self._weights, axis=1) + self._bias)
model = Model(SigmoidPerceptron(2))
epochs, costs, accuracies, *_ = model.train(
np.array([[0, 0], [0, 1], [1, 0], [1, 1]]),
np.array([0, 0, 0, 1]),
None,
None,
SimpleTrainer(),
ThresholdMeasurer(0.5),
None,
epochs=35,
)
print('Epochs: ', epochs)
print('Costs: ', costs)
print('Accuracies: ', accuracies)
print('Predictions:', model.evaluate(np.array([[0, 0], [0, 1], [1, 0], [1,
1]])))
evaluations = self.evaluate(dataset)
error = labels - evaluations
self._weights += np.sum(self._learning_rate * error *
np.transpose(dataset),
axis=1)
return (np.average(error), evaluations)
def evaluate(self, dataset):
sum_ = np.sum(dataset * self._weights, axis=1)
return (sum_ > self._threshold).astype(int)
model = Model(Perceptron(2))
epochs, costs, accuracies, *_ = model.train(
np.array([[0, 0], [0, 1], [1, 0], [1, 1]]),
np.array([0, 0, 0, 1]),
None,
None,
SimpleTrainer(),
PairwiseMeasurer(),
None,
epochs=10,
)
print('Epochs: ', epochs)
print('Costs: ', costs)
print('Accuracies: ', accuracies)
print('Predictions:', model.evaluate(np.array([[0, 0], [0, 1], [1, 0], [1,
1]])))