def evaluate_lenet5(datasets,
learning_seed=0.01, n_epochs=500,
batch_size=250,
save_folder='./cache',
channel_count=1):
""" Evaluate a convnet for three dimensional image inputs.
:type learning_seed: float
:param learning_seed: learning rate used (factor for the stochastic
gradient) during initialization.
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: path to the dataset used for training /testing (MNIST here)
:type batch_size: integer
:param batch_size: size for batched testing
:type channel_count: integer
:param channel_count: number of channels per image
"""
rng = numpy.random.RandomState(23455)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
new_rate = T.lscalar() # The learning rate.
# start-snippet-1
r = T.dscalar('r') # the learning rate as a variable.
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# Reshape matrix of rasterized images of shape
# (batch_size, channel_count, 32 * 32)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (32, 32) is the size of CIFAR images.
layer0_input = x.reshape((batch_size, channel_count, 32, 32))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (32+2-3+1 , 32+2-3+1) = (32, 32)
# maxpooling reduces this further to (32/2, 32/2) = (16, 16)
# 4D output tensor is thus of shape (batch_size, nkerns[0], 16, 16)
layer0 = LeNetConvPoolLayer(
rng,
input=layer0_input,
image_shape=(batch_size, channel_count, 32, 32),
filter_shape=(128, channel_count, 3, 3),
poolsize=(1, 1)
)
layer1 = LeNetConvPoolLayer(
rng,
input=layer0.output,
image_shape=(batch_size, 128, 32, 32),
filter_shape=(128, 128, 3, 3),
poolsize=(2, 2)
)
# Construct the second convolutional pooling layer
# filtering reduces the image size to (16+2-3+1, 16+2-3+1) = (16, 16)
# maxpooling reduces this further to (16/2, 16/2) = (8, 8)
# 4D output tensor is thus of shape (batch_size, nkerns[1], 8, 8)
layer2 = LeNetConvPoolLayer(
rng,
input=layer1.output,
image_shape=(batch_size, 128, 16, 16),
filter_shape=(256, 128, 3, 3),
poolsize=(1, 1)
)
layer3 = LeNetConvPoolLayer(
rng,
input=layer2.output,
image_shape=(batch_size, 256, 16, 16),
filter_shape=(256, 256, 3, 3),
poolsize=(1, 1)
)
layer4 = LeNetConvPoolLayer(
rng,
input=layer3.output,
image_shape=(batch_size, 256, 16, 16),
filter_shape=(256, 256, 3, 3),
poolsize=(1, 1)
)
layer5 = LeNetConvPoolLayer(
rng,
input=layer4.output,
image_shape=(batch_size, 256, 16, 16),
filter_shape=(256, 256, 3, 3),
poolsize=(2, 2)
)
# Construct the third convolutional pooling layer
# filtering reduces the image size to (8+2-3+1, 8+2-3+1) = (8, 8)
# No maxpooling (aka maxpooling (1, 1))
# 4D output tensor is thus of shape (batch_size, nkerns[1], 8, 8)
layer6 = LeNetConvPoolLayer(
rng,
input=layer5.output,
image_shape=(batch_size, 256, 8, 8),
filter_shape=(512, 256, 3, 3),
poolsize=(1, 1)
)
layer7 = LeNetConvPoolLayer(
rng,
input=layer6.output,
image_shape=(batch_size, 512, 8, 8),
filter_shape=(512, 512, 3, 3),
poolsize=(1, 1)
)
layer8 = LeNetConvPoolLayer(
rng,
input=layer7.output,
image_shape=(batch_size, 512, 8, 8),
filter_shape=(512, 512, 3, 3),
poolsize=(1, 1)
)
layer9 = LeNetConvPoolLayer(
rng,
input=layer8.output,
image_shape=(batch_size, 512, 8, 8),
filter_shape=(512, 512, 3, 3),
poolsize=(2, 2)
)
# Construct the third convolutional pooling layer
# filtering reduces the image size to (8+2-3+1, 8+2-3+1) = (8, 8)
# maxpooling reduces this further to (8/2, 8/2) = (4, 4)
# 4D output tensor is thus of shape (batch_size, nkerns[1], 4, 4)
layer10 = LeNetConvPoolLayer(
rng,
input=layer9.output,
image_shape=(batch_size, 512, 4, 4),
filter_shape=(512, 512, 3, 3),
poolsize=(1, 1)
)
layer11 = LeNetConvPoolLayer(
rng,
input=layer10.output,
image_shape=(batch_size, 512, 4, 4),
filter_shape=(512, 512, 3, 3),
poolsize=(1, 1)
)
layer12 = LeNetConvPoolLayer(
rng,
input=layer11.output,
image_shape=(batch_size, 512, 4, 4),
filter_shape=(512, 512, 3, 3),
poolsize=(1, 1)
)
layer13 = LeNetConvPoolLayer(
rng,
input=layer12.output,
image_shape=(batch_size, 512, 4, 4),
filter_shape=(512, 512, 3, 3),
poolsize=(1, 1)
)
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (500, 512 * 4 * 4) = (500, 8192) with the default values.
layer14_input = layer13.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer14 = HiddenLayer(
rng,
input=layer14_input,
n_in=512 * 4 * 4,
n_out=2048,
activation=relu
)
layer15 = HiddenLayer(
rng,
input=layer14.output,
n_in=2048,
n_out=1024,
activation=relu
)
# classify the values of the fully-connected sigmoidal layer
# there are 10 labels in total.
layer16 = HiddenLayer(
rng,
input=layer15.output,
n_in=1024,
n_out=10,
activation=relu
)
# the cost we minimize during training is the NLL of the model
cost = layer16.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer16.errors(y),
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size]
}
)
validate_model = theano.function(
[index],
layer16.errors(y),
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size]
}
)
# create a list of all model parameters to be fit by gradient descent
params = layer16.params + \
layer15.params + \
layer14.params + \
layer13.params + \
layer12.params + \
layer11.params + \
layer10.params + \
layer9.params + \
layer8.params + \
layer7.params + \
layer6.params + \
layer6.params + \
layer5.params + \
layer4.params + \
layer3.params + \
layer2.params + \
layer1.params + \
layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [
(param_i, param_i - r * grad_i)
for param_i, grad_i in zip(params, grads)
]
train_model = theano.function(
[index, new_rate],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size],
r: new_rate
}
)
# end-snippet-1
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
cur_learning_rate = learning_seed
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print 'training @ iter = ', iter
cost_ij = train_model(minibatch_index, cur_learning_rate)
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [
test_model(i)
for i in xrange(n_test_batches)
]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
else: # Did not get a new best validation score.
cur_learning_rate /= 10
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print('Optimization complete.')
print('Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
