def test_mlp(learning_rate=0.01,
L1_reg=0.00,
L2_reg=0.0001,
n_epochs=1000,
batch_size=20,
n_hidden=500,
verbose=True,
fileName='predictionsMLP'):
"""
Wrapper function for testing LeNet on SVHN dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
:type batch_size: int
:param batch_szie: number of examples in minibatch.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
"""
rng = numpy.random.RandomState(23455)
datasets = load_data()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
learning_rate = theano.shared(learning_rate)
testing = T.lscalar('testing')
testValue = testing
getTestValue = theano.function([testing], testValue)
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 3 * 32 * 32)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
layer0_input = x.reshape((batch_size, 3, 32, 32))
layer0_input = layer0_input.flatten(2)
# TODO: Construct the first convolutional pooling layer
layer0 = HiddenLayer(rng,
input=layer0_input,
n_in=32 * 32 * 3,
n_out=n_hidden,
activation=T.tanh)
layer1 = HiddenLayer(rng,
input=layer0.output,
n_in=n_hidden,
n_out=n_hidden,
activation=T.tanh)
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# TODO: construct a fully-connected sigmoidal layer
layer2 = DropConnect(rng,
input=layer1.output,
n_in=n_hidden,
n_out=batch_size,
testing=testing)
# TODO: classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=batch_size, n_out=10)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likelihood(y)
print("Model building complete")
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer3.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(1)
},
on_unused_input='ignore')
getPredictedValue = theano.function(
[index],
layer3.predictedValue(),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(1)
},
on_unused_input='ignore')
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(1)
},
on_unused_input='ignore')
# TODO: create a list of all model parameters to be fit by gradient descent
params = layer3.params + layer2.params + layer1.params + layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
#updates = [
# (param_i, param_i - learning_rate * layer2.maskW.get_value() * grad_i) if (param_i.name == 'WDrop') else (param_i, param_i - learning_rate * layer2.maskb.get_value() * grad_i) if(param_i.name == 'bDrop') else (param_i, param_i - learning_rate * grad_i)
# for param_i, grad_i in zip(params, grads)
#]
updates = []
momentum = 0.9
for param in params:
param_update = theano.shared(param.get_value() * 0.,
broadcastable=param.broadcastable)
if (param.name == 'WDrop'):
updates.append((param, param - learning_rate.get_value().item() *
layer2.maskW.get_value() * param_update))
elif (param.name == 'bDrop'):
updates.append((param, param - learning_rate.get_value().item() *
layer2.maskb.get_value() * param_update))
else:
updates.append(
(param,
param - learning_rate.get_value().item() * param_update))
updates.append(
(param_update,
momentum * param_update + (1. - momentum) * T.grad(cost, param)))
'''
updates = [
(param_i, param_i - learning_rate * grad_i) if ((param_i.name == 'WDrop') or (param_i.name == 'bDrop')) else (param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
'''
print("Commpiling the train model function")
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(0)
},
on_unused_input='ignore',
allow_input_downcast=True)
###############
# TRAIN MODEL #
###############
print('... training')
predictions = train_nn(train_model, validate_model, test_model,
getPredictedValue, n_train_batches, n_valid_batches,
n_test_batches, n_epochs, learning_rate, verbose)
f = open(fileName, 'wb')
cPickle.dump(predictions, f, protocol=cPickle.HIGHEST_PROTOCOL)
f.close()
def test_dropout(learning_rate=0.1,
n_epochs=1000,
nkerns=[64, 128],
batch_size=120,
verbose=False):
"""
Wrapper function for testing LeNet on SVHN dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
:type batch_size: int
:param batch_szie: number of examples in minibatch.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
"""
rng = numpy.random.RandomState(23455)
datasets = load_data()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
testing = T.iscalar('testing')
testValue = testing
getTestValue = theano.function([testing], testValue)
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 3 * 32 * 32)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
layer0_input = x.reshape((batch_size, 3, 32, 32))
# TODO: Construct the first convolutional pooling layer
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 3, 32, 32),
filter_shape=(nkerns[0], 3, 5, 5),
poolsize=(2, 2))
# TODO: Construct the second convolutional pooling layer
layer1 = LeNetConvPoolLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], 14, 14),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
layer2_input = layer1.output.flatten(2)
# TODO: construct a fully-connected sigmoidal layer
layer2 = DropOut(rng,
input=layer2_input,
n_in=nkerns[1] * 5 * 5,
n_out=batch_size,
testing=testing)
# TODO: classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=batch_size, n_out=10)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer3.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(1)
},
on_unused_input='ignore')
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(1)
},
on_unused_input='ignore')
# TODO: create a list of all model parameters to be fit by gradient descent
params = layer3.params + layer2.params + layer1.params + layer0.params
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size],
testing: getTestValue(1)
},
on_unused_input='ignore',
allow_input_downcast=True)
###############
# TRAIN MODEL #
###############
print('... training')
train_nn(train_model, validate_model, test_model, n_train_batches,
n_valid_batches, n_test_batches, n_epochs, verbose)
def test_CDNN(learning_rate=0.1,
n_epochs=1000,
nkerns=[16, 512],
batch_size=200,
verbose=False,
filter_size=5):
"""
Wrapper function for testing CNN in cascade with DNN
"""
rng = numpy.random.RandomState(23455)
datasets = load_data()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 3 * 32 * 32)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
layer0_input = x.reshape((batch_size, 3, 32, 32))
# TODO: Construct the first convolutional pooling layer
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 3, 32, 32),
filter_shape=(nkerns[0], 3, filter_size,
filter_size),
poolsize=(2, 2))
# TODO: Construct the second convolutional pooling layer
new_shape = (32 - filter_size + 1) // 2
layer1 = LeNetConvPoolLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], new_shape,
new_shape),
filter_shape=(nkerns[1], nkerns[0],
filter_size, filter_size),
poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
layer2_input = layer1.output.flatten(2)
# TODO: construct a fully-connected sigmoidal layer
new_factors = (new_shape - filter_size + 1) // 2
layer2 = HiddenLayer(rng,
input=layer2_input,
n_in=nkerns[1] * new_factors * new_factors,
n_out=500,
activation=T.tanh)
layer3 = HiddenLayer(rng,
input=layer2.output,
n_in=500,
n_out=500,
activation=T.tanh)
# TODO: classify the values of the fully-connected sigmoidal layer
layer4 = LogisticRegression(input=layer3.output, n_in=500, n_out=10)
# the cost we minimize during training is the NLL of the model
cost = layer4.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer4.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer4.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# TODO: create a list of all model parameters to be fit by gradient descent
params = layer4.params + layer3.params + layer2.params + layer1.params + layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
###############
# TRAIN MODEL #
###############
print('... training')
return train_nn(train_model, validate_model, test_model, n_train_batches,
n_valid_batches, n_test_batches, n_epochs, verbose)
def test_convnet(learning_rate=0.1,
n_epochs=1000,
nkerns=[16, 512, 20],
batch_size=200,
verbose=False,
filter_size=2):
"""
Wrapper function for testing Multi-Stage ConvNet on SVHN dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
:type batch_size: int
:param batch_szie: number of examples in minibatch.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
"""
rng = numpy.random.RandomState(23455)
datasets = load_data()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 3 * 32 * 32)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
layer0_input = x.reshape((batch_size, 3, 32, 32))
# TODO: Construct the first convolutional pooling layer:
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 3, 32, 32),
filter_shape=(nkerns[0], 3, filter_size,
filter_size),
poolsize=(2, 2))
# TODO: Construct the second convolutional pooling layer
new_shape = (32 - filter_size + 1) // 2
layer1 = LeNetConvPoolLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], new_shape,
new_shape),
filter_shape=(nkerns[1], nkerns[0],
filter_size, filter_size),
poolsize=(2, 2))
# Combine Layer 0 output and Layer 1 output
# TODO: downsample the first layer output to match the size of the second
# layer output.
# TDOD: change ds
layer0_output_ds = downsample.max_pool_2d(input=layer0.output,
ds=(2, 2),
ignore_border=True)
# concatenate layer
layer2_input = T.concatenate([layer1.output, layer0_output_ds], axis=1)
# TODO: Construct the third convolutional pooling layer
new_shape = (new_shape - filter_size + 1) // 2
layer2 = LeNetConvPoolLayer(rng,
input=layer2_input,
image_shape=(batch_size, nkerns[0] + nkerns[1],
new_shape, new_shape),
filter_shape=(nkerns[2], nkerns[0] + nkerns[1],
filter_size, filter_size),
poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (batch_size, nkerns[2] * 1 * 1).
layer3_input = layer2.output.flatten(2)
# construct a fully-connected sigmoidal layer
new_shape = (new_shape - filter_size + 1) // 2
layer3 = HiddenLayer(rng,
input=layer3_input,
n_in=nkerns[2] * new_shape * new_shape,
n_out=500,
activation=T.tanh)
# classify the values of the fully-connected sigmoidal layer
layer4 = LogisticRegression(input=layer3.output, n_in=500, n_out=10)
# the cost we minimize during training is the NLL of the model
cost = layer4.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer4.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
layer4.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# TODO: create a list of all model parameters to be fit by gradient descent
params = layer4.params + layer3.params + layer2.params + layer1.params + layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
###############
# TRAIN MODEL #
###############
print('... training')
return train_nn(train_model, validate_model, test_model, n_train_batches,
n_valid_batches, n_test_batches, n_epochs, verbose)
def test_lenet(learning_rate=0.1,
n_epochs=1000,
nkerns=[16, 512],
batch_size=200,
filter_size=5,
dnn_layers=1,
n_hidden=500,
gabor=False,
lmbda=None,
verbose=False):
"""
Wrapper function for testing LeNet on SVHN dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
:type batch_size: int
:param batch_szie: number of examples in minibatch.
:type verbose: boolean
:param verbose: to print out epoch summary or not to.
"""
print test_lenet.__name__, nkerns, filter_size, gabor, lmbda
rng = numpy.random.RandomState(23455)
datasets = load_data()
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 3 * 32 * 32)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
layer0_input = x.reshape((batch_size, 3, 32, 32))
if gabor is True:
# Generate Gabor filters
filters = build_gabor(filter_size, nkerns[0], lmbda)
# filters = numpy.array([filters[i][0] for i in range(len(filters))])
filters = numpy.array([filters[i] for i in range(len(filters))])
# print filters.shape
filter_weights = numpy.tile(filters,
(1, 3, 1)).reshape(nkerns[0], 3,
filter_size,
filter_size)
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 3, 32, 32),
filter_shape=(nkerns[0], 3, filter_size,
filter_size),
poolsize=(2, 2),
weights=filter_weights)
print 'gabor filter weights are working'
else:
# TODO: Construct the first convolutional pooling layer
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 3, 32, 32),
filter_shape=(nkerns[0], 3, filter_size,
filter_size),
poolsize=(2, 2))
# TODO: Construct the second convolutional pooling layer
i_s_1 = (32 - filter_size + 1) / 2
layer1 = LeNetConvPoolLayer(rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], i_s_1,
i_s_1),
filter_shape=(nkerns[1], nkerns[0],
filter_size, filter_size),
poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
layer2_input = layer1.output.flatten(2)
# TODO: construct a fully-connected sigmoidal layer
i_s_2 = (i_s_1 - filter_size + 1) / 2
if hasattr(n_hidden, '__iter__'):
assert (len(n_hidden) == dnn_layers)
else:
n_hidden = (n_hidden, ) * dnn_layers
DNN_Layers = []
for i in xrange(dnn_layers):
h_input = layer2_input if i == 0 else DNN_Layers[i - 1].output
h_in = nkerns[1] * i_s_2 * i_s_2 if i == 0 else n_hidden[i - 1]
DNN_Layers.append(
HiddenLayer(rng=rng,
input=h_input,
n_in=h_in,
n_out=n_hidden[i],
activation=T.tanh))
# layer2 = HiddenLayer(
# rng,
# input=layer2_input,
# n_in=nkerns[1] * i_s_2 * i_s_2,
# n_out=500,
# activation=T.tanh
# )
# TODO: classify the values of the fully-connected sigmoidal layer
LR_Layer = LogisticRegression(input=DNN_Layers[-1].output,
n_in=n_hidden[i],
n_out=10)
# the cost we minimize during training is the NLL of the model
cost = LR_Layer.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
LR_Layer.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
[index],
LR_Layer.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# TODO: create a list of all model parameters to be fit by gradient descent
params = LR_Layer.params
for layer in DNN_Layers:
params += layer.params
if gabor is True:
print 'gabor params is workings'
params += layer1.params
else:
params += layer1.params + layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
###############
# TRAIN MODEL #
###############
print('... training')
train_nn(train_model, validate_model, test_model, n_train_batches,
n_valid_batches, n_test_batches, n_epochs, verbose)
def MY_lenet(learning_rate=0.1,
n_epochs=200,
nkerns=[20, 50],
batch_size=500,
L1_reg=0.00,
L2_reg=0.0001):
rng = numpy.random.RandomState(23455)
ds_rate = None
datasets = load_data(ds_rate=ds_rate, theano_shared=False)
train_set_x, train_set_y = datasets[0]
train_size = train_set_x.shape
n_train = train_size[0]
'''
print '... Translating images'
train_set_x_tran = np.empty(train_size)
for i in range(n_train):
img = (np.reshape(train_set_x[i],(3,32,32))).transpose(1,2,0)
img_tran = translate_image(img)
train_set_x_tran[i] = np.reshape(img_tran.transpose(2,0,1),(3*32*32))
print '... Rotating images'
train_set_x_rota = np.empty(train_size)
for i in range(n_train):
img = (np.reshape(train_set_x[i],(3,32,32))).transpose(1,2,0)
img_tran = rotate_image(img)
train_set_x_rota[i] = np.reshape(img_tran.transpose(2,0,1),(3*32*32))
'''
print '... Fliping images'
train_set_x_flip = np.empty(train_size)
for i in range(n_train):
img = (np.reshape(train_set_x[i], (3, 32, 32))).transpose(1, 2, 0)
img_tran = flip_image(img)
train_set_x_flip[i] = np.reshape(img_tran.transpose(2, 0, 1),
(3 * 32 * 32))
'''
print '... Ennoising images'
train_set_x_nois = np.empty(train_size)
for i in range(n_train):
img = (np.reshape(train_set_x[i],(3,32,32))).transpose(1,2,0)
img_tran = noise_injection(img)
train_set_x_aug[i] = np.reshape(img_tran.transpose(2,0,1),(3*32*32))
'''
train_set_x = np.concatenate(
(
train_set_x,
#train_set_x_tran,
#train_set_x_rota,
train_set_x_flip),
axis=0)
train_set_y = np.concatenate(
(
train_set_y,
#train_set_y,
#train_set_y,
train_set_y),
axis=0)
datasets[0] = [train_set_x, train_set_y]
train_set_x, train_set_y = shared_dataset(datasets[0])
valid_set_x, valid_set_y = shared_dataset(datasets[1])
test_set_x, test_set_y = shared_dataset(datasets[2])
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches //= batch_size
n_valid_batches //= batch_size
n_test_batches //= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
# start-snippet-1
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
training_enabled = T.iscalar(
'training_enabled'
) # pseudo boolean for switching between training and prediction
######################
# BUILD ACTUAL MODEL #
######################
print('... building the model')
# Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (28, 28) is the size of MNIST images.
layer0_input = x.reshape((batch_size, 3, 32, 32))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24)
# maxpooling reduces this further to (24/2, 24/2) = (12, 12)
# 4D output tensor is thus of shape (batch_size, nkerns[0], 12, 12)
layer0 = LeNetConvPoolLayer(rng,
input=layer0_input,
image_shape=(batch_size, 3, 32, 32),
filter_shape=(nkerns[0], 3, 3, 3),
poolsize=(2, 2))
#print 'layer0.output.shape ='
#print layer0.output.shape.eval({x: np.random.rand(2,2).astype(dtype=theano.config.floatX)})
layerbn = BatchNormalization(input_shape=(batch_size, nkerns[0], 15, 15),
mode=1,
momentum=0.9)
layerbn_output = layerbn.get_result(layer0.output)
#print 'layerbn_output.shape ='
#print layerbn_output.shape.eval({x: np.random.rand(2,2).astype(dtype=theano.config.floatX)})
# Construct the second convolutional pooling layer
# filtering reduces the image size to (12-5+1, 12-5+1) = (8, 8)
# maxpooling reduces this further to (8/2, 8/2) = (4, 4)
# 4D output tensor is thus of shape (batch_size, nkerns[1], 4, 4)
layer1 = LeNetConvPoolLayer(rng,
input=layerbn_output,
image_shape=(batch_size, nkerns[0], 15, 15),
filter_shape=(nkerns[1], nkerns[0], 3, 3),
poolsize=(2, 2))
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (batch_size, nkerns[1] * 4 * 4),
# or (500, 50 * 4 * 4) = (500, 800) with the default values.
layer2_input = layer1.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer2 = DropoutHiddenLayer(rng,
is_train=training_enabled,
input=layer2_input,
n_in=nkerns[1] * 6 * 6,
n_out=4096,
activation=T.nnet.relu)
# construct a fully-connected sigmoidal layer
layer3 = DropoutHiddenLayer(rng,
is_train=training_enabled,
input=layer2.output,
n_in=4096,
n_out=2048,
activation=T.nnet.relu)
# construct a fully-connected sigmoidal layer
layer4 = DropoutHiddenLayer(rng,
is_train=training_enabled,
input=layer3.output,
n_in=2048,
n_out=1024,
activation=T.nnet.relu)
# construct a fully-connected sigmoidal layer
layer5 = DropoutHiddenLayer(rng,
is_train=training_enabled,
input=layer4.output,
n_in=1024,
n_out=512,
activation=T.nnet.relu)
# classify the values of the fully-connected sigmoidal layer
layer6 = LogisticRegression(input=layer5.output, n_in=512, n_out=10)
# L1 norm ; one regularization option is to enforce L1 norm to
# be small
L1 = (abs(layer2.W).sum() + abs(layer3.W).sum() + abs(layer4.W).sum() +
abs(layer5.W).sum() + abs(layer6.W).sum())
# square of L2 norm ; one regularization option is to enforce
# square of L2 norm to be small
L2_sqr = ((layer2.W**2).sum() + (layer3.W**2).sum() + (layer4.W**2).sum() +
(layer5.W**2).sum() + (layer6.W**2).sum())
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost = (layer6.negative_log_likelihood(y) + L1_reg * L1 + L2_reg * L2_sqr)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer6.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size],
training_enabled: numpy.cast['int32'](0)
})
validate_model = theano.function(
[index],
layer6.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size],
training_enabled: numpy.cast['int32'](0)
})
# create a list of all model parameters to be fit by gradient descent
params = layer6.params + layer5.params + layer4.params + layer3.params + layer2.params + layer1.params + layer0.params
'''
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [
(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
'''
# compute the gradient of cost with respect to theta (sotred in params)
# the resulting gradients will be stored in a list gparams
gparams = [T.grad(cost, param) for param in params]
# specify how to update the parameters of the model as a list of
# (variable, update expression) pairs
momentum = theano.shared(numpy.cast[theano.config.floatX](0.5),
name='momentum')
updates = []
for param in params:
param_update = theano.shared(param.get_value() *
numpy.cast[theano.config.floatX](0.))
updates.append((param, param - learning_rate * param_update))
updates.append((param_update, momentum * param_update +
(numpy.cast[theano.config.floatX](1.) - momentum) *
T.grad(cost, param)))
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size],
training_enabled: numpy.cast['int32'](1)
})
# end-snippet-1
###############
# TRAIN MODEL #
###############
train_nn(train_model,
validate_model,
test_model,
n_train_batches,
n_valid_batches,
n_test_batches,
n_epochs,
verbose=True)
