def generate_data_SFO_N(num_passes=51, base_fname='num_minibatches', store_x=True):
"""
Same as generate_data(), but compares SFO with different numbers of minibatches
rather than SFO to other optimizers.
"""
#models_to_train = ( models.logistic, models.Hopfield )
#models_to_train = ( models.Hopfield, ) # DEBUG
#models_to_train = ( models.logistic, ) # DEBUG
models_to_train = ( models.MLP_soft, ) # DEBUG
# the different numbers of history terms to experiment with
L_set = [5,10,20]
for model_class in models_to_train:
# # first do LBFGS
# np.random.seed(0) # make experiments repeatable
# model = model_class(scale_by_N=False)
# trainer = optimization_wrapper.train(model, full_objective_per_pass=1)
# optimizer = trainer.LBFGS
# print("\n\n\n" + model.name + "\n" + str(optimizer))
# optimizer(num_passes=num_passes)
# save_results(trainer, base_fname=base_fname, store_x=store_x)
# then do SFO with different minibatch sizes
for L in L_set:
np.random.seed(0) # make experiments repeatable
model = model_class()
trainer = optimization_wrapper.train(model)
optimizer = trainer.SFO
np.random.seed(0) # make experiments exactly repeatable
print("\n\n\n" + model.name + "\n" + str(optimizer))
optimizer(num_passes=num_passes, max_history_terms=L, learner_name='SFO $L=%d$'%(L))
convergence_utils.save_results(trainer, base_fname=(base_fname+'_N=%d'%(L)), store_x=store_x)
def generate_data_SFO_N(num_passes=51, base_fname='num_minibatches', store_x=True):
"""
Same as generate_data(), but compares SFO with different numbers of minibatches
rather than SFO to other optimizers.
"""
models_to_train = ( models.logistic, models.Hopfield )
models_to_train = ( models.logistic, ) # DEBUG
# the different numbers of minibatches to experiment with
N_set = np.round(np.logspace(0, np.log10(200), 6)).astype(int)
#N_set = np.round(np.logspace(0, 2, 3)).astype(int)
for model_class in models_to_train:
# # first do LBFGS
# np.random.seed(0) # make experiments repeatable
# model = model_class(scale_by_N=False)
# trainer = optimization_wrapper.train(model, full_objective_per_pass=1)
# optimizer = trainer.LBFGS
# print("\n\n\n" + model.name + "\n" + str(optimizer))
# optimizer(num_passes=num_passes)
# save_results(trainer, base_fname=base_fname, store_x=store_x)
# then do SFO with different minibatch sizes
for N in N_set:
np.random.seed(0) # make experiments repeatable
model = model_class(num_subfunctions=N, scale_by_N=False)
trainer = optimization_wrapper.train(model, full_objective_per_pass=1)
optimizer = trainer.SFO
np.random.seed(0) # make experiments exactly repeatable
print("\n\n\n" + model.name + "\n" + str(optimizer))
optimizer(num_passes=num_passes, learner_name='SFO $N=%d$'%(N))
convergence_utils.save_results(trainer, base_fname=(base_fname+'_N=%d'%(N)), store_x=store_x)
def generate_data_SFO_variations(num_passes=51,
base_fname='convergence_variations',
store_x=True):
"""
Same as generate_data(), but compares different variations of SFO to each
other, rather than SFO to other optimizers.
"""
models_to_train = (models.logistic, models.Hopfield)
models_to_train = (models.logistic, ) #DEBUG
for model_class in models_to_train:
np.random.seed(0) # make experiments repeatable
model = model_class()
trainer = optimization_wrapper.train(model)
optimizers_to_use = [
trainer.SFO_variations,
]
for optimizer in optimizers_to_use:
np.random.seed(0) # make experiments exactly repeatable
print("\n\n\n" + model.name + "\n" + str(optimizer))
optimizer(num_passes=num_passes)
# save_results doesn't need to be called until outside this loop,
# but this way we can peak at partial results
# also, note that saved files for the same model but different optimizers
# can be combined in plots, just by calling load_results with all the saved files
convergence_utils.save_results(trainer,
base_fname=base_fname,
store_x=store_x)
def generate_data(num_passes=51, base_fname='figure_data_', store_x=True):
"""
train all the models in models_to_train using all the
optimizers, and save the resulting function value traces
in figure_data_*.npz files.
"""
models_to_train = (
#models.DeepAE,
models.CIFARConvNet,
models.ICA,
models.toy,
models.logistic,
models.MLP_soft,
# models.MLP_hard,
models.ContractiveAutoencoder,
models.Hopfield,
)
models_to_train = (models.logistic, ) # DEBUG
for model_class in models_to_train:
np.random.seed(0) # make experiments repeatable
model = model_class()
trainer = optimization_wrapper.train(model)
optimizers_to_use = (
trainer.SFO,
trainer.LBFGS,
trainer.LBFGS_minibatch,
trainer.ADA,
trainer.SGD,
trainer.SAG,
trainer.SGD_momentum,
)
for optimizer in optimizers_to_use:
np.random.seed(0) # make experiments exactly repeatable
print("\n\n\n" + model.name + "\n" + str(optimizer))
optimizer(num_passes=num_passes)
# save_results doesn't need to be called until outside this loop,
# but this way we can peak at partial results
# also, note that saved files for the same model but different optimizers
# can be combined in plots, just by calling load_results with all the saved files
convergence_utils.save_results(trainer,
base_fname=base_fname,
store_x=store_x)
def generate_data(num_passes=51, base_fname='figure_data_', store_x=True):
"""
train all the models in models_to_train using all the
optimizers, and save the resulting function value traces
in figure_data_*.npz files.
"""
models_to_train = (
#models.DeepAE,
models.CIFARConvNet,
models.ICA,
models.toy,
models.logistic,
models.MLP_soft,
# models.MLP_hard,
models.ContractiveAutoencoder,
models.Hopfield,
)
models_to_train = ( models.logistic, ) # DEBUG
for model_class in models_to_train:
np.random.seed(0) # make experiments repeatable
model = model_class()
trainer = optimization_wrapper.train(model)
optimizers_to_use = (
trainer.SFO,
trainer.LBFGS,
trainer.LBFGS_minibatch,
trainer.ADA,
trainer.SGD,
trainer.SAG,
trainer.SGD_momentum,
)
for optimizer in optimizers_to_use:
np.random.seed(0) # make experiments exactly repeatable
print("\n\n\n" + model.name + "\n" + str(optimizer))
optimizer(num_passes=num_passes)
# save_results doesn't need to be called until outside this loop,
# but this way we can peak at partial results
# also, note that saved files for the same model but different optimizers
# can be combined in plots, just by calling load_results with all the saved files
convergence_utils.save_results(trainer, base_fname=base_fname, store_x=store_x)
def generate_data_SFO_variations(num_passes=51, base_fname='convergence_variations', store_x=True):
"""
Same as generate_data(), but compares different variations of SFO to each
other, rather than SFO to other optimizers.
"""
models_to_train = ( models.logistic, models.Hopfield )
models_to_train = ( models.logistic, ) #DEBUG
for model_class in models_to_train:
np.random.seed(0) # make experiments repeatable
model = model_class()
trainer = optimization_wrapper.train(model)
optimizers_to_use = [trainer.SFO_variations,]
for optimizer in optimizers_to_use:
np.random.seed(0) # make experiments exactly repeatable
print("\n\n\n" + model.name + "\n" + str(optimizer))
optimizer(num_passes=num_passes)
# save_results doesn't need to be called until outside this loop,
# but this way we can peak at partial results
# also, note that saved files for the same model but different optimizers
# can be combined in plots, just by calling load_results with all the saved files
convergence_utils.save_results(trainer, base_fname=base_fname, store_x=store_x)
