def test_random_histories_file_exists(self):
""" Should return True, because a random-histories-file exists for 42. """
histories = ExperimentHistories()
histories.setup(2, 2, 2, 2, 2)
with TemporaryDirectory() as tmp_dir_name:
result_saver.save_experiment_histories_random_retrain(
tmp_dir_name, 'prefix', 42, histories)
self.assertIs(
result_loader.random_histories_file_exists(
f"{tmp_dir_name}/prefix", 42), True)
def test_setup_experiment_histories(self):
""" Should setup all np.arrays correctly. """
expected_history = np.zeros(
(3, 2, 24),
dtype=float) # expected history for accuracies and losses
histories = ExperimentHistories()
histories.setup(3, 1, 42, 4, 7)
np.testing.assert_array_equal(expected_history, histories.train_loss)
np.testing.assert_array_equal(expected_history, histories.val_loss)
np.testing.assert_array_equal(expected_history, histories.val_acc)
np.testing.assert_array_equal(expected_history, histories.test_acc)
np.testing.assert_array_equal(np.ones(2, dtype=float),
histories.sparsity)
def test_get_random_experiment_histories_from_file(self):
""" Should load fake random_histories from npz file. """
histories = ExperimentHistories()
histories.setup(1, 1, 1, 1, 1)
with TemporaryDirectory() as tmp_dir_name:
result_saver.save_experiment_histories_random_retrain(
tmp_dir_name, 'prefix', 42, histories)
# load and validate histories from file
experiment_path_prefix = f"{tmp_dir_name}/prefix"
loaded_histories = result_loader.get_random_experiment_histories_from_file(
experiment_path_prefix, 42)
self.assertEqual(histories, loaded_histories)
def test_save_experiment_histories_random_retrain(self):
""" Should save fake histories into npz file. """
histories = ExperimentHistories()
histories.setup(2, 1, 3, 2, 3)
with TemporaryDirectory() as tmp_dir_name:
result_saver.save_experiment_histories_random_retrain(
tmp_dir_name, 'prefix', 42, histories)
# load and validate histories from file
result_file_path = os.path.join(tmp_dir_name,
'prefix-random-histories42.npz')
with np.load(result_file_path) as result_file:
reconstructed_histories = ExperimentHistories(**result_file)
self.assertEqual(histories, reconstructed_histories)
def get_experiment_histories_from_file(experiment_path_prefix):
""" Read history-arrays from the specified npz-file and return them as ExperimentHistories object. """
histories_file_path = generate_experiment_histories_file_path(
experiment_path_prefix)
with np.load(histories_file_path) as histories_file:
return ExperimentHistories(
**histories_file) # unpack dict-like histories-file
def test_stack_histories_fail_unequal_sparsity(self):
""" Should raise an error, as the sparsity-histories do not match. """
histories0, histories1 = ExperimentHistories(), ExperimentHistories()
histories0.setup(1, 1, 2, 3, 2)
histories1.setup(1, 1, 2, 3, 2)
histories0.sparsity, histories1.sparsity = np.array(
[1., 0.5]), np.array([1., 0.7])
with self.assertRaises(AssertionError):
histories0.stack_histories(histories1)
def test_stack_histories_fail_wrong_type(self):
""" Should raise an error, as other is no ExperimentHistory but a list. """
histories0 = ExperimentHistories()
histories0.setup(1, 1, 1, 1, 1)
with self.assertRaises(AssertionError):
histories0.stack_histories([])
def test_experiment_histories_are_unequal(self):
""" Should return False, because both ExperimentHistories contain unequal arrays. """
histories1, histories2 = ExperimentHistories(), ExperimentHistories()
histories1.setup(1, 1, 1, 1, 1)
histories2.setup(1, 1, 2, 1, 1)
self.assertIs(ExperimentHistories.__eq__(histories1, histories2),
False)
def __init__(self, specs):
super(Experiment, self).__init__()
self.specs = specs
log_from_medium(self.specs.verbosity, specs)
self.device = torch.device(specs.device)
# setup epoch_length and trainer in load_data_and_setup_trainer()
self.trainer = None
self.epoch_length = 0
# setup history-arrays in setup_experiment()
self.hists = ExperimentHistories()
self.stop_hists = EarlyStopHistoryList()
def test_stack_histories(self):
""" Should stack two histories, i.e. append their arrays at the net-dimension.
The sparsity histories should not be appended, as they need to be equal. """
histories0 = ExperimentHistories()
histories1 = ExperimentHistories()
histories0.setup(
2, 1, 2, 3, 2
) # arrays for accuracies and losses have shape (2, 2, 3), all filled with zeroes
histories1.setup(2, 1, 2, 3, 2)
# fill arrays for accuracies and losses with ones
histories1.train_loss = np.ones_like(histories1.train_loss)
histories1.val_loss = np.ones_like(histories1.val_loss)
histories1.val_acc = np.ones_like(histories1.val_acc)
histories1.test_acc = np.ones_like(histories1.test_acc)
# expected history for accuracies and losses
expected_history = np.array(
[[[0., 0., 0.], [0., 0., 0.]], [[0., 0., 0.], [0., 0., 0.]],
[[1., 1., 1.], [1., 1., 1.]], [[1., 1., 1.], [1., 1., 1.]]],
dtype=float)
result_history = histories0.stack_histories(histories1)
np.testing.assert_array_equal(expected_history,
result_history.train_loss)
np.testing.assert_array_equal(expected_history,
result_history.val_loss)
np.testing.assert_array_equal(expected_history, result_history.val_acc)
np.testing.assert_array_equal(expected_history,
result_history.test_acc)
np.testing.assert_array_equal(histories0.sparsity,
result_history.sparsity)
def test_experiment_histories_error_on_invalid_type(self):
""" Should return False, because both ExperimentHistories contain unequal arrays. """
with self.assertRaises(AssertionError):
ExperimentHistories.__eq__(ExperimentHistories(), dict())
def test_experiment_histories_are_equal(self):
""" Should return True, because both ExperimentHistories contain equal arrays. """
histories = ExperimentHistories()
histories.setup(1, 1, 1, 1, 1)
self.assertIs(ExperimentHistories.__eq__(histories, histories), True)