def test_SdA(datasets,
output_folder, base_folder,
window_size,
pretraining_epochs=15,
training_epochs=1000):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# split the datasets
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of examples given in training set
n_in = window_size*3 # number of input units
n_out = 7 # number of output units
x = T.matrix('x')
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print '... building the model'
# construct the stacked denoising autoencoder class
sda = SdA(
numpy_rng=numpy_rng,
n_ins=n_in,
hidden_layers_sizes=[window_size*2, window_size*2],
n_outs=n_out
)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
'''
print '... getting the pretraining functions'
pretraining_fns, pretraining_updates = sda.pretraining_functions(train_set_x=train_set_x,
window_size=window_size)
'''
print '... pre-training the model'
# using scipy conjugate gradient optimizer
import scipy.optimize
print ("Optimizing using scipy.optimize.fmin_cg...")
start_time = timeit.default_timer()
index = T.lscalar('index')
n_train_samples = train_set_x.get_value(borrow=True).shape[0] - window_size + 1
## Pre-train layer-wise
corruption_levels = [.1, .2]
for da_index in xrange(sda.n_layers):
cur_dA=sda.dA_layers[da_index]
# get the cost and the updates list
cost = cur_dA.get_cost(corruption_levels[da_index])
# compile a theano function that returns the cost
sample_cost = theano.function(
inputs=[index],
outputs=cost,
givens={
sda.x: train_set_x[index: index + window_size]
},
on_unused_input='warn'
)
# compile a theano function that returns the gradient with respect to theta
sample_grad = theano.function(
inputs=[index],
outputs=T.grad(cost, cur_dA.theta),
givens={
sda.x: train_set_x[index: index + window_size]
},
on_unused_input='warn'
)
def train_fn(theta_value):
sda.dA_layers[da_index].theta.set_value(theta_value, borrow=True)
train_losses = [sample_cost(i)
for i in xrange(n_train_samples)]
this_train_loss = float(numpy.mean(train_losses))
sda.dA_layers[da_index].train_cost_array.append([])
sda.dA_layers[da_index].train_cost_array[-1].append(sda.dA_layers[da_index].epoch)
sda.dA_layers[da_index].train_cost_array[-1].append(this_train_loss)
sda.dA_layers[da_index].epoch += 1
return numpy.mean(train_losses)
def train_fn_grad(theta_value):
sda.dA_layers[da_index].theta.set_value(theta_value, borrow=True)
grad = sample_grad(0)
for i in xrange(1, n_train_samples):
grad += sample_grad(i)
return grad / n_train_samples
best_w_b = scipy.optimize.fmin_cg(
f=train_fn,
x0=numpy.zeros((sda.dA_layers[da_index].n_visible + 1) * sda.dA_layers[da_index].n_hidden,
dtype=x.dtype),
fprime=train_fn_grad,
disp=0,
maxiter=pretraining_epochs
)
visualize_pretraining(train_cost=sda.dA_layers[da_index].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[da_index],
n_hidden=sda.dA_layers[da_index].n_hidden,
da_layer=da_index,
datasets_folder=output_folder,
base_folder=base_folder)
end_time = timeit.default_timer()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing functions for the model
print '... getting the finetuning functions'
train_fn, train_fn_grad, callback = sda.build_finetune_functions(
datasets=datasets,
window_size=window_size
)
print '... finetunning the model'
import scipy.optimize
print ("Optimizing using scipy.optimize.fmin_cg...")
start_time = timeit.default_timer()
best_w_b = scipy.optimize.fmin_cg(
f=train_fn,
x0=numpy.zeros((n_in + 1) * n_out, dtype=sda.x.dtype),
fprime=train_fn_grad,
callback=callback,
disp=0,
maxiter=training_epochs
)
print(
(
'Optimization complete with best validation score of %f %%, with '
'test performance %f %%'
)
% (sda.validation_scores[0] * 100., sda.validation_scores[1] * 100.)
)
visualize_finetuning(train_cost=sda.logLayer.train_cost_array,
train_error=sda.logLayer.train_error_array,
valid_error=sda.logLayer.valid_error_array,
test_error=sda.logLayer.test_error_array,
window_size=window_size,
learning_rate=0,
datasets_folder=output_folder,
base_folder=base_folder)
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.1fs' % ((end_time - start_time)))
def test_SdA(
datasets,
output_folder,
base_folder,
window_size,
corruption_levels,
pretraining_epochs,
training_epochs,
pretrain_lr=0,
finetune_lr=0,
):
"""
Demonstrates how to train and test a stochastic denoising autoencoder.
This is demonstrated on ICHI.
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:type n_iter: int
:param n_iter: maximal number of iterations ot run the optimizer
:type datasets: array
:param datasets: [train_set, valid_set, test_set]
:type output_folder: string
:param output_folder: folder for costand error graphics with results
"""
# split the datasets
(train_set_x, train_set_y) = datasets[0]
(valid_set_x, valid_set_y) = datasets[1]
(test_set_x, test_set_y) = datasets[2]
# compute number of examples given in training set
n_in = window_size * 3 # number of input units
n_out = 7 # number of output units
# numpy random generator
# start-snippet-3
numpy_rng = numpy.random.RandomState(89677)
print "... building the model"
# construct the stacked denoising autoencoder class
sda = SdA(numpy_rng=numpy_rng, n_ins=n_in, hidden_layers_sizes=[window_size * 2, window_size], n_outs=n_out)
# end-snippet-3 start-snippet-4
#########################
# PRETRAINING THE MODEL #
#########################
start_time = timeit.default_timer()
pretrained_sda = pretrain_sda_sgd(
sda=sda,
train_set_x=train_set_x,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
pretrain_lr=pretrain_lr,
corruption_levels=corruption_levels,
)
"""
pretrained_sda = pretrain_sda_cg(sda=sda,
train_set_x=train_set_x,
window_size=window_size,
pretraining_epochs=pretraining_epochs,
corruption_levels=corruption_levels)
"""
end_time = timeit.default_timer()
for i in xrange(sda.n_layers):
print (i, "i pretrained")
visualize_pretraining(
train_cost=pretrained_sda.dA_layers[i].train_cost_array,
window_size=window_size,
learning_rate=0,
corruption_level=corruption_levels[i],
n_hidden=sda.dA_layers[i].n_hidden,
da_layer=i,
datasets_folder=output_folder,
base_folder=base_folder,
)
print >> sys.stderr, (
"The pretraining code for file "
+ os.path.split(__file__)[1]
+ " ran for %.2fm" % ((end_time - start_time) / 60.0)
)
# end-snippet-4
########################
# FINETUNING THE MODEL #
########################
start_time = timeit.default_timer()
"""
finetuned_sda = finetune_sda_sgd(sda=pretrained_sda,
datasets=datasets,
window_size=window_size,
finetune_lr=finetune_lr,
training_epochs=training_epochs)
"""
finetuned_sda = finetune_sda_cg(
sda=pretrained_sda, datasets=datasets, window_size=window_size, training_epochs=training_epochs
)
end_time = timeit.default_timer()
visualize_finetuning(
train_cost=finetuned_sda.logLayer.train_cost_array,
train_error=finetuned_sda.logLayer.train_error_array,
valid_error=finetuned_sda.logLayer.valid_error_array,
test_error=finetuned_sda.logLayer.test_error_array,
window_size=window_size,
learning_rate=0,
datasets_folder=output_folder,
base_folder=base_folder,
)
