def test_execute_training_with_early_stopping(self):
""" Should execute training without errors and save results with correct shapes.
It should also return a valid checkpoint and early-stopping index.
Use a simple net with one linear layer and fake-data_loaders with samples of shape (1,4). """
# create net and setup trainer with fake-DataLoader (use the same loader for training, validation and test)
net = generate_single_layer_net()
fake_loader = generate_fake_data_loader()
trainer = TrainerAdam(0., fake_loader, fake_loader, fake_loader, save_early_stop=True)
# perform training with mocked validation-loss
with mock.patch('training.trainer.TrainerAdam.compute_val_loss',
side_effect=[2.0, 1.0, 0.5, 1.0]) as mocked_val_loss:
net, train_loss_hist, val_loss_hist, val_acc_hist, test_acc_hist, early_stop_index, early_stop_cp = \
trainer.train_net(net, epoch_count=2, plot_step=2) # 8 batches (iterations), 4 early-stop evaluations
self.assertEqual(4, mocked_val_loss.call_count)
# early-stopping criterion is True for first three calls (last one counts), thus 5 is the 'early_stop_index'
self.assertEqual(5, early_stop_index)
net.load_state_dict(early_stop_cp) # check if the checkpoint can be loaded without errors
self.assertIs(net is not None, True)
np.testing.assert_array_less(np.zeros(4, dtype=float), train_loss_hist) # check for positive entries
np.testing.assert_array_less(np.zeros(4, dtype=float), val_loss_hist)
np.testing.assert_array_less(np.zeros(4, dtype=float), val_acc_hist)
np.testing.assert_array_less(np.zeros(4, dtype=float), test_acc_hist)
def test_compute_val_loss(self):
""" Should calculate a positive validation loss. """
# create net and setup trainer with fake-DataLoader (use the same loader for training, validation and test)
net = generate_single_layer_net()
fake_loader = generate_fake_data_loader()
trainer = TrainerAdam(0., fake_loader, fake_loader, fake_loader)
self.assertLessEqual(0.0, trainer.compute_val_loss(net))
def test_compute_val_acc(self):
""" Should calculate the correct validation-accuracy.
The fake-net with one linear layer classifies all fake-samples correctly.
Use a fake-val_loader with one batch to validate the result. """
# create net and setup trainer and fake-DataLoader with one batch
net = generate_single_layer_net()
val_loader = [[torch.tensor([[2., 2., 2., 2.], [2., 2., 0., 0.], [0., 0., 2., 2.]]), torch.tensor([0, 0, 1])]]
trainer = TrainerAdam(0., [], val_loader, test_loader=[])
self.assertEqual(1., trainer.compute_acc(net, test=False))
def test_compute_test_acc(self):
""" Should calculate the correct test-accuracy.
The fake-net with one linear layer classifies half of the fake-samples correctly.
Use a fake-val_loader with one batch to validate the result. """
# create net and setup trainer and fake-DataLoader with two batches (use the same samples for both batches)
net = generate_single_layer_net()
samples = torch.tensor([[2., 2., 2., 2.], [2., 2., 0., 0.], [0., 0., 2., 2.]])
test_loader = [[samples, torch.tensor([0, 0, 1])], [samples, torch.tensor([1, 1, 0])]]
trainer = TrainerAdam(0., [], [], test_loader=test_loader)
self.assertEqual(0.5, trainer.compute_acc(net, test=True))
def test_execute_training(self):
""" The training should be executed without errors and results should have correct shapes.
Use a simple net with one linear layer and fake-data_loaders with samples of shape (1,4). """
# create net and setup trainer with fake-DataLoader (use the same loader for training, validation and test)
net = generate_single_layer_net()
fake_loader = generate_fake_data_loader()
trainer = TrainerAdam(0., fake_loader, fake_loader, fake_loader)
net, train_loss_hist, val_loss_hist, val_acc_hist, test_acc_hist, _, _ = \
trainer.train_net(net, epoch_count=2, plot_step=4)
self.assertIs(net is not None, True)
np.testing.assert_array_less(np.zeros(2, dtype=float), train_loss_hist) # check for positive entries
np.testing.assert_array_less(np.zeros(2, dtype=float), val_loss_hist)
np.testing.assert_array_less(np.zeros(2, dtype=float), val_acc_hist)
np.testing.assert_array_less(np.zeros(2, dtype=float), test_acc_hist)
def load_data_and_setup_trainer(self):
""" Load dataset and initialize trainer.
Store the length of the training-loader into 'self.epoch_length' to initialize histories. """
# load dataset
if self.specs.dataset == DatasetNames.MNIST:
train_ld, val_ld, test_ld = get_mnist_data_loaders(device=self.specs.device, verb=self.specs.verbosity)
elif self.specs.dataset == DatasetNames.CIFAR10:
train_ld, val_ld, test_ld = get_cifar10_data_loaders(device=self.specs.device, verb=self.specs.verbosity)
else:
raise AssertionError(f"Could not load datasets, because the given name {self.specs.dataset} is invalid.")
self.epoch_length = len(train_ld)
self.trainer = TrainerAdam(self.specs.learning_rate, train_ld, val_ld, test_ld, self.device,
self.specs.save_early_stop, self.specs.verbosity)
def test_should_save_early_stop_checkpoint_new_checkpoint(self):
""" Should return True, because the validation-accuracy reached a new minimum. """
trainer = TrainerAdam(0., [], [], [], save_early_stop=True)
self.assertIs(trainer.should_save_early_stop_checkpoint(0.1, 0.2), True)
def test_should_save_early_stop_checkpoint_no_new_minimum_equal(self):
""" Should return False, because the current validation-loss is equal to the the minimum. """
trainer = TrainerAdam(0., [], [], [], save_early_stop=True)
self.assertIs(trainer.should_save_early_stop_checkpoint(0.2, 0.2), False)
def test_should_save_early_stop_checkpoint_no_evaluation(self):
""" Should return False, because the early-stopping criterion should not be evaluated. """
trainer = TrainerAdam(0., [], [], [], save_early_stop=False)
self.assertIs(trainer.should_save_early_stop_checkpoint(0.5, 0.2), False)