def test_get_early_stop_history_list_from_files_no_save_early_stop(self):
""" Should raise assertion error as 'save_early_stop' flag is not set in specs. """
experiment_specs = get_specs_lenet_toy()
experiment_specs.save_early_stop = False
with self.assertRaises(AssertionError):
result_loader.get_early_stop_history_list_from_files(
"some_path", experiment_specs)
def test_generate_model_from_state_dict_invalid_model_name(self):
""" Should raise assertion error if specs contain an invalid entry for 'net'. """
experiment_specs = get_specs_lenet_toy()
experiment_specs.net = "Invalid name"
with self.assertRaises(AssertionError):
result_loader.generate_model_from_state_dict(
dict(), experiment_specs)
def test_perform_toy_lenet_experiment(self):
""" Should run IMP-Experiment with small Lenet and toy-dataset without errors. """
specs = get_specs_lenet_toy()
specs.prune_count = 1
specs.save_early_stop = True
early_stop_history = EarlyStopHistory()
early_stop_history.setup(specs.prune_count)
net = Net(specs.net, specs.dataset, specs.plan_conv, specs.plan_fc)
early_stop_history.state_dicts[0] = net.state_dict()
early_stop_history.state_dicts[1] = net.state_dict()
early_stop_history_list = EarlyStopHistoryList()
early_stop_history_list.setup(1, 0)
early_stop_history_list.histories[0] = early_stop_history
fake_mnist_data_loaders = generate_fake_mnist_data_loaders()
with mock.patch('experiments.experiment.get_mnist_data_loaders',
return_value=fake_mnist_data_loaders):
with TemporaryDirectory(
) as tmp_dir_name: # save results into a temporary folder
result_saver.save_specs(tmp_dir_name, 'prefix', specs)
result_saver.save_early_stop_history_list(
tmp_dir_name, 'prefix', early_stop_history_list)
path_to_specs = os.path.join(tmp_dir_name, 'prefix-specs.json')
experiment = ExperimentRandomRetrain(path_to_specs, 0, 1)
experiment.run_experiment()
self.assertEqual(
1,
len(
glob.glob(
os.path.join(tmp_dir_name,
'prefix-random-histories0.npz'))))
def test_get_early_stop_history_list_from_files(self):
""" Should load fake EarlyStopHistoryList from pth files. """
plan_fc = [2]
net0 = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
net1 = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
history_list = EarlyStopHistoryList()
history_list.setup(2, 0)
history_list.histories[0].state_dicts[0] = deepcopy(net0.state_dict())
history_list.histories[1].state_dicts[0] = deepcopy(net1.state_dict())
history_list.histories[0].indices[0] = 3
history_list.histories[1].indices[0] = 42
specs = get_specs_lenet_toy()
specs.save_early_stop = True
specs.net_count = 2
specs.prune_count = 0
with TemporaryDirectory() as tmp_dir_name:
# save checkpoints
result_saver.save_early_stop_history_list(tmp_dir_name, 'prefix',
history_list)
# load and validate histories from file
experiment_path_prefix = f"{tmp_dir_name}/prefix"
loaded_history_list = result_loader.get_early_stop_history_list_from_files(
experiment_path_prefix, specs)
self.assertEqual(loaded_history_list, history_list)
net0.load_state_dict(history_list.histories[0].state_dicts[0])
net1.load_state_dict(history_list.histories[1].state_dicts[0])
def test_get_early_stop_history_from_file_invalid_number_tall(self):
""" Should raise assertion error as 'net_number' is too tall. """
experiment_specs = get_specs_lenet_toy()
experiment_specs.save_early_stop = True
experiment_specs.net_count = 3
with self.assertRaises(AssertionError):
result_loader.get_early_stop_history_from_file("some_path",
experiment_specs,
net_number=3)
def test_perform_toy_lenet_osp_experiment(self):
""" Should run OSP-Experiment with small Lenet and toy-dataset without errors. """
specs = get_specs_lenet_toy()
specs.experiment_name = experiment_specs.ExperimentNames.OSP
with mock.patch('experiments.experiment.get_mnist_data_loaders', return_value=self.fake_mnist_data_loaders):
with TemporaryDirectory() as tmp_dir_name: # save results into a temporary folder
experiment = ExperimentOSP(specs, tmp_dir_name)
experiment.run_experiment()
self.assertEqual(1, len(glob.glob(os.path.join(tmp_dir_name, '*-specs.json'))))
def test_save_specs(self):
""" Should save toy_specs into json file. """
specs = get_specs_lenet_toy()
with TemporaryDirectory() as tmp_dir_name:
result_saver.save_specs(tmp_dir_name, 'prefix', specs)
result_file_path = os.path.join(tmp_dir_name, 'prefix-specs.json')
with open(result_file_path, 'r') as result_file:
self.assertEqual(asdict(specs), j_load(result_file))
def test_get_specs_from_file_as_experiment_specs(self):
""" Should load toy_specs from json file as ExperimentSpecs. """
experiment_specs = get_specs_lenet_toy()
with TemporaryDirectory() as tmp_dir_name:
result_saver.save_specs(tmp_dir_name, 'prefix', experiment_specs)
result_file_path = f"{tmp_dir_name}/prefix-specs.json"
loaded_experiment_specs = result_loader.get_specs_from_file(
result_file_path, as_dict=False)
self.assertEqual(experiment_specs, loaded_experiment_specs)
def test_do_not_save_early_stop_checkpoints(self):
""" Should not save checkpoints from early-stopping. """
specs = get_specs_lenet_toy()
specs.save_early_stop = False
with mock.patch('experiments.experiment.get_mnist_data_loaders',
return_value=self.fake_mnist_data_loaders):
with TemporaryDirectory(
) as tmp_dir_name: # save results into a temporary folder
experiment = ExperimentIMP(specs, tmp_dir_name)
experiment.run_experiment()
self.assertEqual([],
glob.glob(
os.path.join(tmp_dir_name,
'*-early-stop*.pth')))
def test_subnetwork_from_toy_lenet_osp_experiment_has_equal_init_weight(self):
""" The subnetwork should have the same 'weight_init' buffer as the original network. """
specs = get_specs_lenet_toy()
specs.experiment_name = experiment_specs.ExperimentNames.OSP
specs.net_count = 1
with mock.patch('experiments.experiment.get_mnist_data_loaders', return_value=self.fake_mnist_data_loaders):
with TemporaryDirectory() as tmp_dir_name: # temporary folder to check if experiment generates files
experiment = ExperimentOSP(specs, tmp_dir_name)
experiment.setup_experiment()
initial_net = experiment.nets[0].get_new_instance(reset_weight=False)
experiment.execute_experiment() # execute does not save files
self.assertEqual([], os.listdir(tmp_dir_name)) # experiment should not generate files
# check if subnetwork has correct sparsity
np.testing.assert_allclose(experiment.hists.sparsity, [1., 0.801], atol=0.001)
# check if subnetwork has been generated from original net
subnet = experiment.nets[0].get_new_instance(reset_weight=False)
self.assertIs(torch.equal(subnet.fc[0].weight_init, initial_net.fc[0].weight_init), True)
self.assertIs(torch.equal(subnet.out.weight_init, initial_net.out.weight_init), True)
def test_generate_file_prefix_for_toy_experiment(self):
""" Should generate the correct file_prefix for toy specs and a fake time-string. """
specs = get_specs_lenet_toy()
self.assertEqual(
'Time-Lenet-MNIST',
result_saver.generate_file_prefix(specs, save_time='Time'))
