batch_size_test=1000, batch_size_train=(first_batch_size, 64), log_verbose_structure=False, ) class FollowupExperiment(mixins.TestNoise, ExploratoryExperiment): def __init__(self, **config): super().__init__(noise_test_at_end=True, noise_test_freq=0, noise_levels=list(np.arange(0.0, 0.51, 0.05)), **config) if __name__ == "__main__": experiment_name = os.path.basename(__file__).replace(".py", "") alphas = (0.7, 0.8, 0.9) samples_per_frontier_config = 5 ax_optimize_accuracy_weightsparsity( experiments.as_ray_trainable(ExploratoryExperiment), experiments.as_ray_trainable(FollowupExperiment), experiment_name, os.path.dirname(os.path.realpath(__file__)), alphas, PARAMETERS, NUM_TRAINING_ITERATIONS, samples_per_frontier_config, )
NUM_TRAINING_ITERATIONS = 15 class SupervisedNoiseBoostingRezeroCovariance(mixins.UpdateBoostStrength, mixins.RezeroWeights, mixins.TestNoise, mixins.LogCovariance, experiments.Supervised): pass if __name__ == "__main__": ray.init() tune.run( experiments.as_ray_trainable(SupervisedNoiseBoostingRezeroCovariance), name=os.path.basename(__file__).replace(".py", ""), config=dict( network_name="mnist_lesparsenet", network_params=dict(), dataset_name="MNIST", dataset_params={}, optim_alg="SGD", optim_params=dict(lr=0.02, ), lr_scheduler_alg="StepLR", lr_scheduler_params=dict( step_size=1, gamma=0.8, ), training_iterations=NUM_TRAINING_ITERATIONS, use_tqdm=False,
DEFAULT_LOGGERS, ) class SupervisedNoiseBoostingRezero(mixins.UpdateBoostStrength, mixins.RezeroWeights, mixins.TestNoise, experiments.Supervised): pass if __name__ == "__main__": ray.init() tune.run( experiments.as_ray_trainable(SupervisedNoiseBoostingRezero), name=os.path.basename(__file__).replace(".py", ""), config=dict( model_alg="gsc_lesparsenet", model_params=dict(), dataset_name="PreprocessedGSC", dataset_params={}, optim_alg="SGD", optim_params=dict( lr=0.01, weight_decay=0.01, ), lr_scheduler_alg="StepLR",
) NUM_TRAINING_ITERATIONS = 30 class SupervisedNoiseRezero(mixins.RezeroWeights, mixins.TestNoise, experiments.Supervised): pass if __name__ == "__main__": ray.init() tune.run( experiments.as_ray_trainable(SupervisedNoiseRezero), name=os.path.basename(__file__).replace(".py", ""), config=dict( network_class=networks.gsc_lesparsenet, network_args=dict( cnn_activity_percent_on=(1.0, 1.0), linear_activity_percent_on=(1.0,), ), dataset_class=datasets.PreprocessedGSC, dataset_args={}, optim_class=torch.optim.SGD, optim_args=dict( lr=0.01, weight_decay=0.01,
if __name__ == "__main__": ray.init() regimes = [ dict(optim_alg="Adam", optim_params=dict(lr=0.01, ), lr_scheduler_alg="StepLR", lr_scheduler_params=dict( step_size=5, gamma=0.5, ), training_iterations=50), ] tune.run( experiments.as_ray_trainable( SupervisedNoiseConstrainedLoggedRegularized), name=os.path.basename(__file__).replace(".py", ""), config=tune.grid_search([ dict(model_alg="mnist_lenet_backpropstructure", model_params=dict( l0_strength=7e-6, droprate_init=0.2, ), dataset_name="MNIST", dataset_params={}, use_tqdm=False, batch_size_train=(64, 64), batch_size_test=1000, noise_test_freq=0, noise_test_at_end=True, noise_levels=list(np.arange(0.0, 1.0, 0.05)),
if iteration == 0 or 5 <= iteration <= 19: # Either the optimizer doesn't exist yet, or we're changing the loss # function and the adaptive state is invalidated. I don't think the # original paper contained this logic, never resetting the Adam # optimizer while performing "warmup" on the regularization, but it # seems like the right thing to do. self.optimizer = torch.optim.Adam(self._get_parameters(), lr=lr) else: for param_group in self.optimizer.param_groups: param_group["lr"] = lr return super().run_epoch(iteration) if __name__ == "__main__": ray.init() tune.run( experiments.as_ray_trainable(VDropExperiment), name=os.path.basename(__file__).replace(".py", ""), num_samples=1, checkpoint_freq=0, checkpoint_at_end=False, resources_per_trial={ "cpu": 1, "gpu": (1 if torch.cuda.is_available() else 0) }, loggers=DEFAULT_LOGGERS, verbose=1, )