layer_dropout.varin = model.models_stack[i].varin
if (i + 2) % 4 == 0:
model.models_stack[i - 2].threshold = 0.
pretrain_lr = pretrain_lr_lin
layer_cost = layer_dropout.cost() + layer_dropout.weightdecay(
weightdecay)
else:
pretrain_lr = pretrain_lr_zae
layer_cost = layer_dropout.cost()
trainer = GraddescentMinibatch(varin=model.varin,
data=train_x,
cost=layer_cost,
params=layer_dropout.params_private,
supervised=False,
batchsize=batchsize,
learningrate=pretrain_lr,
momentum=momentum,
rng=npy_rng)
prev_cost = numpy.inf
patience = 0
for epoch in xrange(pretrain_epc):
cost = trainer.epoch()
if prev_cost <= cost:
patience += 1
if patience > 10:
patience = 0
trainer.set_learningrate(0.9 * trainer.learningrate)
if trainer.learningrate < 1e-10:
[],
T.mean(T.neq(model.models_stack[-1].predict(), test_set_y)),
givens={model.models_stack[0].varin: test_set_x},
)
#############
# PRE-TRAIN #
#############
for i in range(len(model.models_stack) - 1):
print "\n\nPre-training layer %d:" % i
trainer = GraddescentMinibatch(varin=model.varin,
data=train_set_x,
cost=model.models_stack[i].cost(),
params=model.models_stack[i].params_private,
supervised=False,
batchsize=1,
learningrate=0.001,
momentum=0.,
rng=npy_rng)
layer_analyse = model.models_stack[i].encoder()
layer_analyse.draw_weight(patch_shape=(28, 28, 1), npatch=100)
layer_analyse.hist_weight()
for epoch in xrange(15):
trainer.step()
layer_analyse.draw_weight(patch_shape=(28, 28, 1), npatch=100)
layer_analyse.hist_weight()
save_params(model=model, filename="mnist_sae_784_784_784_10.npy")
##| Instantiating model
##`--------------------
model = cAE(n_visible=n_visible, n_hiddens=n_hiddens,
contraction=contraction,
activation=activation, n_samples=n_samples,
BINARY=BINARY, MINIBATCH_SIZE=MINIBATCH_SIZE, corruptF=corruptF)
##,----------------------
##| Instantiating trainer
##`----------------------
gd = GraddescentMinibatch(model=model, data=data_s,
batchsize=MINIBATCH_SIZE,
learningrate=learningrate,
momentum=momentum,
normalizefilters=normalizefilters,
rng=rng,
verbose=verbose)
##,---------
##| Training
##`---------
#for epoch in np.arange(epochs):
# gd.step()
while (model.cost(data_s.get_value()) > eps) and gd.epochcount < epochs:
gd.step()
##################################
## Stage 4: Visualizing results ##
)
print "Done."
#############
# FINE-TUNE #
#############
pdb.set_trace()
print "\n\n... fine-tuning the whole network"
trainer = GraddescentMinibatch(varin=model.varin,
data=train_x,
truth=model.models_stack[-1].vartruth,
truth_data=train_y,
supervised=True,
cost=model.models_stack[-1].cost(),
params=model.params,
batchsize=batchsize,
learningrate=lr,
momentum=momentum,
rng=npy_rng)
trainer_adam = Adam(varin=model_adam.varin,
data=train_x,
truth=model_adam.models_stack[-1].vartruth,
truth_data=train_y,
supervised=True,
cost=model_adam.models_stack[-1].cost(),
params=model_adam.params,
batchsize=batchsize,
learningrate=lr / 5.,
