def test_get_early_stop_history_list_from_files(self):
""" Should load fake EarlyStopHistoryList from pth files. """
plan_fc = [2]
net0 = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
net1 = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
history_list = EarlyStopHistoryList()
history_list.setup(2, 0)
history_list.histories[0].state_dicts[0] = deepcopy(net0.state_dict())
history_list.histories[1].state_dicts[0] = deepcopy(net1.state_dict())
history_list.histories[0].indices[0] = 3
history_list.histories[1].indices[0] = 42
specs = get_specs_lenet_toy()
specs.save_early_stop = True
specs.net_count = 2
specs.prune_count = 0
with TemporaryDirectory() as tmp_dir_name:
# save checkpoints
result_saver.save_early_stop_history_list(tmp_dir_name, 'prefix',
history_list)
# load and validate histories from file
experiment_path_prefix = f"{tmp_dir_name}/prefix"
loaded_history_list = result_loader.get_early_stop_history_list_from_files(
experiment_path_prefix, specs)
self.assertEqual(loaded_history_list, history_list)
net0.load_state_dict(history_list.histories[0].state_dicts[0])
net1.load_state_dict(history_list.histories[1].state_dicts[0])
def test_perform_toy_lenet_experiment(self):
""" Should run IMP-Experiment with small Lenet and toy-dataset without errors. """
specs = get_specs_lenet_toy()
specs.prune_count = 1
specs.save_early_stop = True
early_stop_history = EarlyStopHistory()
early_stop_history.setup(specs.prune_count)
net = Net(specs.net, specs.dataset, specs.plan_conv, specs.plan_fc)
early_stop_history.state_dicts[0] = net.state_dict()
early_stop_history.state_dicts[1] = net.state_dict()
early_stop_history_list = EarlyStopHistoryList()
early_stop_history_list.setup(1, 0)
early_stop_history_list.histories[0] = early_stop_history
fake_mnist_data_loaders = generate_fake_mnist_data_loaders()
with mock.patch('experiments.experiment.get_mnist_data_loaders',
return_value=fake_mnist_data_loaders):
with TemporaryDirectory(
) as tmp_dir_name: # save results into a temporary folder
result_saver.save_specs(tmp_dir_name, 'prefix', specs)
result_saver.save_early_stop_history_list(
tmp_dir_name, 'prefix', early_stop_history_list)
path_to_specs = os.path.join(tmp_dir_name, 'prefix-specs.json')
experiment = ExperimentRandomRetrain(path_to_specs, 0, 1)
experiment.run_experiment()
self.assertEqual(
1,
len(
glob.glob(
os.path.join(tmp_dir_name,
'prefix-random-histories0.npz'))))
def test_save_early_stop_history_list(self):
""" Should save two fake EarlyStopHistories into two pth files. """
plan_fc = [2]
net0 = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
net1 = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
history_list = EarlyStopHistoryList()
history_list.setup(2, 0)
history_list.histories[0].state_dicts[0] = deepcopy(net0.state_dict())
history_list.histories[1].state_dicts[0] = deepcopy(net1.state_dict())
history_list.histories[0].indices[0] = 3
history_list.histories[1].indices[0] = 42
with TemporaryDirectory() as tmp_dir_name:
result_saver.save_early_stop_history_list(
tmp_dir_name, 'prefix', history_list) # save checkpoints
# load and validate histories from file
result_file_path0 = os.path.join(tmp_dir_name,
'prefix-early-stop0.pth')
result_file_path1 = os.path.join(tmp_dir_name,
'prefix-early-stop1.pth')
for net_num, result_file_path in enumerate(
[result_file_path0, result_file_path1]):
with open(result_file_path, 'rb') as result_file:
reconstructed_hist = t_load(result_file)
net = Net(NetNames.LENET,
DatasetNames.MNIST,
plan_conv=[],
plan_fc=plan_fc)
np.testing.assert_array_equal(
reconstructed_hist.indices,
history_list.histories[net_num].indices)
net.load_state_dict(reconstructed_hist.state_dicts[0])
def test_generate_randomly_reinitialized_net(self):
""" Should generate a network with equal masks but different weights. """
specs = experiment_specs.get_specs_lenet_mnist()
specs.save_early_stop = True
torch.manual_seed(0)
net = Net(specs.net, specs.dataset, specs.plan_conv, specs.plan_fc)
torch.manual_seed(1)
new_net = ExperimentRandomRetrain.generate_randomly_reinitialized_net(
specs, net.state_dict())
self.assertIs(net.fc[0].weight.eq(new_net.fc[0].weight).all().item(),
False)
self.assertIs(
net.fc[0].weight_mask.eq(new_net.fc[0].weight_mask).all().item(),
True)
self.assertIs(
net.out.weight.eq(new_net.out.weight).all().item(), False)
self.assertIs(
net.out.weight_mask.eq(new_net.out.weight_mask).all().item(), True)
