batch_size_test=1000,
batch_size_train=(first_batch_size, 64),
log_verbose_structure=False,
)
class FollowupExperiment(mixins.TestNoise, ExploratoryExperiment):
def __init__(self, **config):
super().__init__(noise_test_at_end=True,
noise_test_freq=0,
noise_levels=list(np.arange(0.0, 0.51, 0.05)),
**config)
if __name__ == "__main__":
experiment_name = os.path.basename(__file__).replace(".py", "")
alphas = (0.7, 0.8, 0.9)
samples_per_frontier_config = 5
ax_optimize_accuracy_weightsparsity(
experiments.as_ray_trainable(ExploratoryExperiment),
experiments.as_ray_trainable(FollowupExperiment),
experiment_name,
os.path.dirname(os.path.realpath(__file__)),
alphas,
PARAMETERS,
NUM_TRAINING_ITERATIONS,
samples_per_frontier_config,
)
NUM_TRAINING_ITERATIONS = 15
class SupervisedNoiseBoostingRezeroCovariance(mixins.UpdateBoostStrength,
mixins.RezeroWeights,
mixins.TestNoise,
mixins.LogCovariance,
experiments.Supervised):
pass
if __name__ == "__main__":
ray.init()
tune.run(
experiments.as_ray_trainable(SupervisedNoiseBoostingRezeroCovariance),
name=os.path.basename(__file__).replace(".py", ""),
config=dict(
network_name="mnist_lesparsenet",
network_params=dict(),
dataset_name="MNIST",
dataset_params={},
optim_alg="SGD",
optim_params=dict(lr=0.02, ),
lr_scheduler_alg="StepLR",
lr_scheduler_params=dict(
step_size=1,
gamma=0.8,
),
training_iterations=NUM_TRAINING_ITERATIONS,
use_tqdm=False,
DEFAULT_LOGGERS,
)
class SupervisedNoiseBoostingRezero(mixins.UpdateBoostStrength,
mixins.RezeroWeights,
mixins.TestNoise,
experiments.Supervised):
pass
if __name__ == "__main__":
ray.init()
tune.run(
experiments.as_ray_trainable(SupervisedNoiseBoostingRezero),
name=os.path.basename(__file__).replace(".py", ""),
config=dict(
model_alg="gsc_lesparsenet",
model_params=dict(),
dataset_name="PreprocessedGSC",
dataset_params={},
optim_alg="SGD",
optim_params=dict(
lr=0.01,
weight_decay=0.01,
),
lr_scheduler_alg="StepLR",
)
NUM_TRAINING_ITERATIONS = 30
class SupervisedNoiseRezero(mixins.RezeroWeights,
mixins.TestNoise,
experiments.Supervised):
pass
if __name__ == "__main__":
ray.init()
tune.run(
experiments.as_ray_trainable(SupervisedNoiseRezero),
name=os.path.basename(__file__).replace(".py", ""),
config=dict(
network_class=networks.gsc_lesparsenet,
network_args=dict(
cnn_activity_percent_on=(1.0, 1.0),
linear_activity_percent_on=(1.0,),
),
dataset_class=datasets.PreprocessedGSC,
dataset_args={},
optim_class=torch.optim.SGD,
optim_args=dict(
lr=0.01,
weight_decay=0.01,
if __name__ == "__main__":
ray.init()
regimes = [
dict(optim_alg="Adam",
optim_params=dict(lr=0.01, ),
lr_scheduler_alg="StepLR",
lr_scheduler_params=dict(
step_size=5,
gamma=0.5,
),
training_iterations=50),
]
tune.run(
experiments.as_ray_trainable(
SupervisedNoiseConstrainedLoggedRegularized),
name=os.path.basename(__file__).replace(".py", ""),
config=tune.grid_search([
dict(model_alg="mnist_lenet_backpropstructure",
model_params=dict(
l0_strength=7e-6,
droprate_init=0.2,
),
dataset_name="MNIST",
dataset_params={},
use_tqdm=False,
batch_size_train=(64, 64),
batch_size_test=1000,
noise_test_freq=0,
noise_test_at_end=True,
noise_levels=list(np.arange(0.0, 1.0, 0.05)),
if iteration == 0 or 5 <= iteration <= 19:
# Either the optimizer doesn't exist yet, or we're changing the loss
# function and the adaptive state is invalidated. I don't think the
# original paper contained this logic, never resetting the Adam
# optimizer while performing "warmup" on the regularization, but it
# seems like the right thing to do.
self.optimizer = torch.optim.Adam(self._get_parameters(), lr=lr)
else:
for param_group in self.optimizer.param_groups:
param_group["lr"] = lr
return super().run_epoch(iteration)
if __name__ == "__main__":
ray.init()
tune.run(
experiments.as_ray_trainable(VDropExperiment),
name=os.path.basename(__file__).replace(".py", ""),
num_samples=1,
checkpoint_freq=0,
checkpoint_at_end=False,
resources_per_trial={
"cpu": 1,
"gpu": (1 if torch.cuda.is_available() else 0)
},
loggers=DEFAULT_LOGGERS,
verbose=1,
)
