def test_get_early_stop_history_list_from_files_no_save_early_stop(self): """ Should raise assertion error as 'save_early_stop' flag is not set in specs. """ experiment_specs = get_specs_lenet_toy() experiment_specs.save_early_stop = False with self.assertRaises(AssertionError): result_loader.get_early_stop_history_list_from_files( "some_path", experiment_specs)
def test_generate_model_from_state_dict_invalid_model_name(self): """ Should raise assertion error if specs contain an invalid entry for 'net'. """ experiment_specs = get_specs_lenet_toy() experiment_specs.net = "Invalid name" with self.assertRaises(AssertionError): result_loader.generate_model_from_state_dict( dict(), experiment_specs)
def test_perform_toy_lenet_experiment(self): """ Should run IMP-Experiment with small Lenet and toy-dataset without errors. """ specs = get_specs_lenet_toy() specs.prune_count = 1 specs.save_early_stop = True early_stop_history = EarlyStopHistory() early_stop_history.setup(specs.prune_count) net = Net(specs.net, specs.dataset, specs.plan_conv, specs.plan_fc) early_stop_history.state_dicts[0] = net.state_dict() early_stop_history.state_dicts[1] = net.state_dict() early_stop_history_list = EarlyStopHistoryList() early_stop_history_list.setup(1, 0) early_stop_history_list.histories[0] = early_stop_history fake_mnist_data_loaders = generate_fake_mnist_data_loaders() with mock.patch('experiments.experiment.get_mnist_data_loaders', return_value=fake_mnist_data_loaders): with TemporaryDirectory( ) as tmp_dir_name: # save results into a temporary folder result_saver.save_specs(tmp_dir_name, 'prefix', specs) result_saver.save_early_stop_history_list( tmp_dir_name, 'prefix', early_stop_history_list) path_to_specs = os.path.join(tmp_dir_name, 'prefix-specs.json') experiment = ExperimentRandomRetrain(path_to_specs, 0, 1) experiment.run_experiment() self.assertEqual( 1, len( glob.glob( os.path.join(tmp_dir_name, 'prefix-random-histories0.npz'))))
def test_get_early_stop_history_list_from_files(self): """ Should load fake EarlyStopHistoryList from pth files. """ plan_fc = [2] net0 = Net(NetNames.LENET, DatasetNames.MNIST, plan_conv=[], plan_fc=plan_fc) net1 = Net(NetNames.LENET, DatasetNames.MNIST, plan_conv=[], plan_fc=plan_fc) history_list = EarlyStopHistoryList() history_list.setup(2, 0) history_list.histories[0].state_dicts[0] = deepcopy(net0.state_dict()) history_list.histories[1].state_dicts[0] = deepcopy(net1.state_dict()) history_list.histories[0].indices[0] = 3 history_list.histories[1].indices[0] = 42 specs = get_specs_lenet_toy() specs.save_early_stop = True specs.net_count = 2 specs.prune_count = 0 with TemporaryDirectory() as tmp_dir_name: # save checkpoints result_saver.save_early_stop_history_list(tmp_dir_name, 'prefix', history_list) # load and validate histories from file experiment_path_prefix = f"{tmp_dir_name}/prefix" loaded_history_list = result_loader.get_early_stop_history_list_from_files( experiment_path_prefix, specs) self.assertEqual(loaded_history_list, history_list) net0.load_state_dict(history_list.histories[0].state_dicts[0]) net1.load_state_dict(history_list.histories[1].state_dicts[0])
def test_get_early_stop_history_from_file_invalid_number_tall(self): """ Should raise assertion error as 'net_number' is too tall. """ experiment_specs = get_specs_lenet_toy() experiment_specs.save_early_stop = True experiment_specs.net_count = 3 with self.assertRaises(AssertionError): result_loader.get_early_stop_history_from_file("some_path", experiment_specs, net_number=3)
def test_perform_toy_lenet_osp_experiment(self): """ Should run OSP-Experiment with small Lenet and toy-dataset without errors. """ specs = get_specs_lenet_toy() specs.experiment_name = experiment_specs.ExperimentNames.OSP with mock.patch('experiments.experiment.get_mnist_data_loaders', return_value=self.fake_mnist_data_loaders): with TemporaryDirectory() as tmp_dir_name: # save results into a temporary folder experiment = ExperimentOSP(specs, tmp_dir_name) experiment.run_experiment() self.assertEqual(1, len(glob.glob(os.path.join(tmp_dir_name, '*-specs.json'))))
def test_save_specs(self): """ Should save toy_specs into json file. """ specs = get_specs_lenet_toy() with TemporaryDirectory() as tmp_dir_name: result_saver.save_specs(tmp_dir_name, 'prefix', specs) result_file_path = os.path.join(tmp_dir_name, 'prefix-specs.json') with open(result_file_path, 'r') as result_file: self.assertEqual(asdict(specs), j_load(result_file))
def test_get_specs_from_file_as_experiment_specs(self): """ Should load toy_specs from json file as ExperimentSpecs. """ experiment_specs = get_specs_lenet_toy() with TemporaryDirectory() as tmp_dir_name: result_saver.save_specs(tmp_dir_name, 'prefix', experiment_specs) result_file_path = f"{tmp_dir_name}/prefix-specs.json" loaded_experiment_specs = result_loader.get_specs_from_file( result_file_path, as_dict=False) self.assertEqual(experiment_specs, loaded_experiment_specs)
def test_do_not_save_early_stop_checkpoints(self): """ Should not save checkpoints from early-stopping. """ specs = get_specs_lenet_toy() specs.save_early_stop = False with mock.patch('experiments.experiment.get_mnist_data_loaders', return_value=self.fake_mnist_data_loaders): with TemporaryDirectory( ) as tmp_dir_name: # save results into a temporary folder experiment = ExperimentIMP(specs, tmp_dir_name) experiment.run_experiment() self.assertEqual([], glob.glob( os.path.join(tmp_dir_name, '*-early-stop*.pth')))
def test_subnetwork_from_toy_lenet_osp_experiment_has_equal_init_weight(self): """ The subnetwork should have the same 'weight_init' buffer as the original network. """ specs = get_specs_lenet_toy() specs.experiment_name = experiment_specs.ExperimentNames.OSP specs.net_count = 1 with mock.patch('experiments.experiment.get_mnist_data_loaders', return_value=self.fake_mnist_data_loaders): with TemporaryDirectory() as tmp_dir_name: # temporary folder to check if experiment generates files experiment = ExperimentOSP(specs, tmp_dir_name) experiment.setup_experiment() initial_net = experiment.nets[0].get_new_instance(reset_weight=False) experiment.execute_experiment() # execute does not save files self.assertEqual([], os.listdir(tmp_dir_name)) # experiment should not generate files # check if subnetwork has correct sparsity np.testing.assert_allclose(experiment.hists.sparsity, [1., 0.801], atol=0.001) # check if subnetwork has been generated from original net subnet = experiment.nets[0].get_new_instance(reset_weight=False) self.assertIs(torch.equal(subnet.fc[0].weight_init, initial_net.fc[0].weight_init), True) self.assertIs(torch.equal(subnet.out.weight_init, initial_net.out.weight_init), True)
def test_generate_file_prefix_for_toy_experiment(self): """ Should generate the correct file_prefix for toy specs and a fake time-string. """ specs = get_specs_lenet_toy() self.assertEqual( 'Time-Lenet-MNIST', result_saver.generate_file_prefix(specs, save_time='Time'))