ap.add_argument("--random_glimpse", default=False)
args = ap.parse_args()
mnist = MiniBatches((MnistDataset()), batch_size=1)
model_path = args.model
network = get_network(model_path, std=args.variance,
disable_reinforce=args.disable_reinforce, random_glimpse=args.random_glimpse)
trainer_conf = TrainerConfig()
trainer_conf.learning_rate = args.learning_rate
trainer_conf.weight_l2 = 0.0001
trainer_conf.hidden_l2 = 0.0001
trainer_conf.method = args.method
trainer_conf.disable_reinforce=args.disable_reinforce
trainer_conf.disable_backprop=args.disable_backprop
trainer = AttentionTrainer(network, network.layers[0], config=trainer_conf)
trainer_conf.report()
timer = Timer()
for _ in list(trainer.train(mnist.train_set(), mnist.valid_set(), mnist.test_set())):
pass
timer.end()
network.save_params(model_path)
timer.report()
model.stack(HighwayLayerLRDiagDropoutBatchNorm(activation=activation, gate_bias=gate_bias, projection_dim=d, d_p_0 = dropout_p_h_0, d_p_1 = dropout_p_h_1, init=init, quasi_ortho_init=True))
#model.stack(BatchNormalization(),Dropout(p=dropout_p_2), Dense(10, init=init))
model.stack(Dropout(p=dropout_p_2), Dense(10, init=init))
learning_rate_start = 3e-3
#learning_rate_target = 3e-7
#learning_rate_epochs = 100
#learning_rate_decay = (learning_rate_target / learning_rate_start) ** (1.0 / learning_rate_epochs)
conf = TrainerConfig()
conf.learning_rate = LearningRateAnnealer.learning_rate(learning_rate_start)
#conf.gradient_clipping = 1
conf.patience = 20
#conf.gradient_tolerance = 5
conf.avoid_nan = True
conf.min_improvement = 1e-10
#trainer = MomentumTrainer(model)
trainer = AdamTrainer(model, conf)
mnist = MiniBatches(MnistDataset(), batch_size=100)
#mnist = MiniBatches(MnistDatasetSmallValid(), batch_size=100)
#trainer.run(mnist, controllers=[IncrementalLearningRateAnnealer(trainer, 0, learning_rate_decay)])
trainer.run(mnist, controllers=[LearningRateAnnealer(trainer, 3, 14)])
logging.info('Setting best parameters for testing.')
trainer.set_params(*trainer.best_params)
trainer._run_test(-1, mnist.test_set())
model.save_params(model_path)
model_path = args.model
network = get_network(model_path,
std=args.variance,
disable_reinforce=args.disable_reinforce,
random_glimpse=args.random_glimpse)
trainer_conf = TrainerConfig()
trainer_conf.learning_rate = LearningRateAnnealer.learning_rate(
args.learning_rate)
trainer_conf.weight_l2 = 0.0001
trainer_conf.hidden_l2 = 0.0001
trainer_conf.method = args.method
trainer = FirstGlimpseTrainer(network,
network.layers[0],
config=trainer_conf)
annealer = LearningRateAnnealer(trainer, patience=5)
timer = Timer()
for _ in trainer.train(mnist.train_set(), mnist.valid_set(),
mnist.test_set()):
if annealer.invoke():
break
timer.end()
network.save_params(model_path)
timer.report()
quasi_ortho_init=True))
#model.stack(BatchNormalization(),Dropout(p=dropout_p_2), Dense(10, init=init))
model.stack(Dropout(p=dropout_p_2), Dense(10, init=init))
learning_rate_start = 3e-3
#learning_rate_target = 3e-7
#learning_rate_epochs = 100
#learning_rate_decay = (learning_rate_target / learning_rate_start) ** (1.0 / learning_rate_epochs)
conf = TrainerConfig()
conf.learning_rate = LearningRateAnnealer.learning_rate(
learning_rate_start)
#conf.gradient_clipping = 1
conf.patience = 20
#conf.gradient_tolerance = 5
conf.avoid_nan = True
conf.min_improvement = 1e-10
#trainer = MomentumTrainer(model)
trainer = AdamTrainer(model, conf)
mnist = MiniBatches(MnistDataset(), batch_size=100)
#mnist = MiniBatches(MnistDatasetSmallValid(), batch_size=100)
#trainer.run(mnist, controllers=[IncrementalLearningRateAnnealer(trainer, 0, learning_rate_decay)])
trainer.run(mnist, controllers=[LearningRateAnnealer(trainer, 3, 14)])
logging.info('Setting best parameters for testing.')
trainer.set_params(*trainer.best_params)
trainer._run_test(-1, mnist.test_set())
model.save_params(model_path)