class SdA(object) :
""" Stacked denoising auto-encoder class (SdA)
A stacked denoising autoencoder model is obtained by stacking serveral
dAs. The hidden layer of the dA at layer 'i' becomes that input of the
dA at layer 'i+1'. The first layer dA gets as input the input of the SdA,
and the hidden layer of the last dA represents the output. Note that after
pretraining, the SdA is dealt with as a normal MLP, the dAs are only used to
initialize the weights.
"""
def __init__(
self,
numpy_rng,
theano_rng=None,
n_ins=784,
hidden_layers_sizes=[500,500],
n_outs=10,
corruption_levels=[0.1,0.1]
):
"""This class is made to support a variable number of layers
:param numpy_rng: numpy random number generator used to draw initial weights
:param theano_rng: Theano random generator; if None is given one is generated based
on a seed drawn from 'rng'
:param n_ins: dimension of the input to the sdA
:param hidden_layers_sizes: intermediate layers size, must contain at least one value
:param n_outs: dimension of the output of the network
:param corruption_levels: amount of corruption to use for each layer
"""
self.sigmoid_layer = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
assert self.n_layers > 0
if not theano_rng :
theano_rng = Randomstreams(numpy_rng.randint(2 ** 30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
# the Sda is an MLP, for which all wieghts of intermediate layers are shared with
# a different denoising autoencoders we will first construct the SdA as a deep
# multilayer perceptron, and when constructing each sigmoidal layer we also construct
# a denoising autoencoder that shares weights with that layer During pretraining we
# will train these autoencoders (which will lead to chainging the weights of the MLP
# as well) During finetuning we will finish training the SdA by doing stochastich gradient
# descent on MLP
# """
for i in xrange(self.n_layers) :
# construct the sigmoid layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0 :
input_size = n_ins
else :
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first layer
if i == 0:
layer_input = self.x
else :
layer_input = self.sigmoid_layer[i-1].output
sigmoid_layer = HiddenLayer(
rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layer.append(sigmoid_layer)
# its arguably a philisophical question ...
# but we are going to only declare that hte parameters of the
# sigmoid_layers are parameters of the StackedDAA the visible biases
# in the dA are parameters of those dA , but not the SdA
self.params.extend(sigmoid_layer.params)
# construct a denoising autoencoder that shared weights with this layer
dA_layer = dA(
numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=sigmoid_layer.W,
bhid=sigmoid_layer.b)
self.dA_layers.append(dA_layer)
# we now need to add a logistic layer on top of the MLP
self.logLaer = LogisticRegression(
input=self.sigmoid_layer[-1].output,
n_in = hidden_layers_sizes[-1],
n_out = n_outs)
self.params.extend(self.logLaer.params)
# construct a function that implements one step of finetunining
# compute the cost for second phase of traning, defined as the
# negative log likelihood
self.finetune_cost = self.logLaer.negative_log_likehood(self.y)
def evaluate_lenet5(learning_rate=0.1, n_epochs=200,
dataset='data/mnist.pkl.gz', nkearns=[20, 50], batch_size=500):
"""
Demonstrates lenet on MNIST dataset
:param learning_rate: learnning rate used (factor for the stochastic gradient)
:param n_epochs: maximal number of epochs to run the optimizer
:param dataset: path to the dataset used for training /testing (MNIST here)
:param nkearns: number of kernels on each layer
"""
rng = numpy.random.RandomState(23455)
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
#######################
## BUILD ACTUAL MODEL #
#######################
print '... building the model '
# Reshape matrix of rasterized image of shape(batch_size, 28 * 28)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (28,28) is the size of MNIST images.
layer0_input = x.reshape((batch_size, 1, 28, 28))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to(28-5+1, 28-5+1) =(24, 24)
# maxpooling reduces this further to (24/2, 24/2) = (12, 12)
# 4D output tensor is thus of shape(batch_size, nkerns[0], 12, 12)
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkearns[0], 1, 5, 5),
poolsize=(2,2)
)
# Construct the second convolutional pooling layer
# filtering reduces the image size to (12 - 5 + 1, 12 - 5 + 1) = (8,8)
# maxpooling reduces this further to(8/2, 8/2) = (4,4)
# 4D output tensor is thus of shape( batch_size, nkers[1], 4, 4)
layer1 = LeNetConvPoolLayer(
rng,
input=layer0.output,
image_shape=(batch_size, nkearns[0], 12, 12),
filter_shape=(nkearns[1], nkearns[0], 5, 5),
poolsize=(2,2)
)
# the HiddenLayer beging fulling-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# this will generate a matrix of shape(batch_size, nkerns[1]*4*4),
# or (500, 50 * 4 * 4) = (500, 800) with the default values.
layer2_input = layer1.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer2 = HiddenLayer(
rng,
input=layer2_input,
n_in=nkearns[1] * 4 * 4,
n_out = 500,
activation=T.tanh
)
# classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likehood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer3.errors(y),
givens={
x: test_set_x[index * batch_size : (index + 1) * batch_size],
y: test_set_y[index * batch_size : (index + 1) * batch_size]
}
)
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size]
}
)
# create a list of all model parameters to be fit by gradient descent
params = layer3.params + layer2.params + layer1.params + layer0.params
# create a list of gradient for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGG Since this model has many parameters. it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [
(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size : (index + 1) * batch_size],
y: train_set_y[index * batch_size : (index + 1) * batch_size]
}
)
#################
## TRAIN MODEL ##
#################
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience/2)
# go throught this many
# minibache before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping) :
epoch += 1
for minibatch_index in xrange(n_train_batches) :
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print 'training @ iter = ', iter
cost_ij = train_model(minibatch_index)
if ( iter + 1) % validation_frequency == 0 :
# compute zero-one loss on validation set
validation_loss = [validate_model(i) for i
in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_loss)
print ('epoch %i, minbatch %i / %i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss :
# improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * this_validation_loss:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [
test_model(i)
for i in xrange(n_test_batches)
]
test_score = numpy.mean(test_losses)
print (' epoch %i, minibatch %i/%i, test error of '
'best model %f %%' %
(epoch, minibatch_index + 1, n_test_batches,
test_score * 100.))
if patience <= iter :
done_looping = True
break
end_time = timeit.default_timer()
print('Optimization complete. ')
print ('Best validation score of %f %% obtained at iteration %i, '
'with best performance %f %%' %
(best_validation_loss*100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file' +
os.path.split(__file__)[0] +
'ran for %.2fm' % ((end_time - start_time) / 60.))