def __init__(self, rng, input, n_in, n_hidden, n_out):
self.hiddenLayer = HiddenLayer(rng=rng,
input=input,
n_in=n_in,
n_out=n_hidden,
activation=T.tanh)
self.logRegressionLayer = LogisticRegressionLayer(
input=self.hiddenLayer.output, n_in=n_hidden, n_out=n_out)
self.L1 = (abs(self.hiddenLayer.W).sum() +
abs(self.logRegressionLayer.W).sum())
self.L2_sqr = ((self.hiddenLayer.W**2).sum() +
(self.hiddenLayer.W**2).sum())
self.negative_log_likelihood = (
self.logRegressionLayer.negative_log_likelihood)
self.errorRate = (self.logRegressionLayer.errorRate)
self.y_pred = (self.logRegressionLayer.y_pred)
self.params = (self.hiddenLayer.params +
self.logRegressionLayer.params)
self.input = input
def getHiddenLayers(input, n_in, layer_sizes):
input_size = n_in
index = 0
next_layer_input = input
layer_list = []
while index < len(layer_sizes):
next_layer = HiddenLayer(rng=this_rng, input=next_layer_input, \
n_in=input_size, n_out=layer_sizes[index])
layer_list.append(next_layer)
next_layer_input = next_layer.output
input_size = layer_sizes[index]
index += 1
return layer_list
def __init__(self, rng, input, n_batch, n_in, n_featureMaps, n_hidden,
n_out):
#image size is 28 x 28, with 1 feature per pixel
self.convPoolLayer1 = ConvPoolLayer(rng=rng,
input=input.reshape(
(n_batch, 1, 28, 28)),
image_shape=(n_batch, 1, 28, 28),
filter_shape=(n_featureMaps[0], 1,
5, 5),
pool_size=(2, 2))
# image size is 12 x 12, with n_featureMaps[0] features per pixel
self.convPoolLayer2 = ConvPoolLayer(
rng=rng,
input=self.convPoolLayer1.output,
image_shape=(n_batch, n_featureMaps[0], 12, 12),
filter_shape=(n_featureMaps[1], n_featureMaps[0], 5, 5),
pool_size=(2, 2))
# image size is 4 x 4, with n_featureMaps[1] features per pixel
self.hiddenLayer = HiddenLayer(
rng=rng,
input=self.convPoolLayer2.output.flatten(2),
n_in=n_featureMaps[1] * 4 * 4,
n_out=n_hidden)
self.logRegressionLayer = LogisticRegressionLayer(
input=self.hiddenLayer.output, n_in=n_hidden, n_out=n_out)
self.negative_log_likelihood = (
self.logRegressionLayer.negative_log_likelihood)
self.errorRate = (self.logRegressionLayer.errorRate)
self.y_pred = (self.logRegressionLayer.y_pred)
self.params = (self.convPoolLayer1.params +
self.convPoolLayer2.params + self.hiddenLayer.params +
self.logRegressionLayer.params)
self.input = input
def build_model(self, input_size, output_size, hidden_layer_info):
## Function to build the model ##
## Arguments:
## input_size : Number of input features
## output_size : Number of output classes
## hidden_layer_info : array with size of hidden layers
np.random.seed(42)
self.input_size = input_size
self.output_size = output_size
prev_layer_size = input_size
for hidden_layer_size in hidden_layer_info:
self.hidden_layers.append(
HiddenLayer(prev_layer_size, hidden_layer_size))
prev_layer_size = hidden_layer_size
self.output_layer = OutputLayer(prev_layer_size, self.output_size)
def __init__(self, input, n_in, n_hidden, n_out):
# Since we are dealing with a one hidden layer MLP, this will translate
# into a HiddenLayer with a tanh activation function connected to the
# LogisticRegression layer; the activation function can be replaced by
# sigmoid or any other nonlinear function
self.hiddenLayer = HiddenLayer(rng=rng, input=input,
n_in=n_in, n_out=n_hidden,
activation=T.tanh)
# The logistic regression layer gets as input the hidden units
# of the hidden layer
self.logRegressionLayer = LogisticRegression(
input=self.hiddenLayer.output,
n_in=n_hidden,
n_out=n_out)
# L1 norm ; one regularization option is to enforce L1 norm to
# be small
self.L1 = abs(self.hiddenLayer.W).sum() \
+ abs(self.logRegressionLayer.W).sum()
# square of L2 norm ; one regularization option is to enforce
# square of L2 norm to be small
self.L2_sqr = (self.hiddenLayer.W ** 2).sum() \
+ (self.logRegressionLayer.W ** 2).sum()
# negative log likelihood of the MLP is given by the negative
# log likelihood of the output of the model, computed in the
# logistic regression layer
self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood
# same holds for the function computing the number of errors
self.errors = self.logRegressionLayer.errors
# the parameters of the model are the parameters of the two layer it is
# made out of
self.params = self.hiddenLayer.params + self.logRegressionLayer.params
self.predict_class = self.logRegressionLayer.y_pred
self.predict_proba = self.logRegressionLayer.p_y_given_x
def __init__(self,
numpy_rng,
theano_rng=None,
n_ins=100,
hidden_layers_sizes=[50, 30],
corruption_levels=[0.1, 0.1]):
self.encoder = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
self.decoder = []
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
self.x = T.matrix('x')
"*************** Encoder **************************"
for i in range(self.n_layers):
if i == 0:
input_size = n_ins
layer_input = self.x
else:
input_size = hidden_layers_sizes[i - 1]
layer_input = self.encoder[-1].output
act_function = T.tanh
encoder_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=act_function)
self.encoder.append(encoder_layer)
self.params.extend(encoder_layer.params)
"**************************************************"
"""Construct a dAE that shared weights with this layer"""
dA_layer = dA(
numpy_rng=numpy_rng,
theano_rng=theano_rng,
#input = layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
W=encoder_layer.W,
bhid=encoder_layer.b)
self.dA_layers.append(dA_layer)
"*************** Decoder *****************************"
i = self.n_layers - 1
while (i >= 0):
input_size = hidden_layers_sizes[i]
if (i > 0):
output_size = hidden_layers_sizes[i - 1]
else:
output_size = n_ins
if (i == self.n_layers - 1):
layer_input = self.encoder[-1].output
else:
layer_input = self.decoder[-1].output
decoder_layer = HiddenLayer(
rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=output_size,
activation=T.tanh,
W=self.encoder[i].W.T,
b=self.dA_layers[i].b_prime) #this is bvis of dA
self.decoder.append(decoder_layer)
self.params.append(decoder_layer.b)
i = i - 1
"******************* End To End Cost function ************************"
y = self.decoder[-1].output
self.recon = (((self.x - y)**2).mean(1)).mean()
self.end2end_cost = self.recon
def build_model(datasets, batch_size, rng, learning_rate):
x = T.matrix('x')
y = T.ivector('y')
index = T.lscalar()
#reshape the image as input to the first conv pool layer
#MNIST images are 28x28
layer0_input = x.reshape((batch_size, 1, 32, 32))
layer0_conv = ConvolutionLayer(rng=rng,
input=layer0_input,
input_shape=(batch_size, 1, 32, 32),
filter_shape=(6, 1, 5, 5))
layer0_subsample = SubsampleLayer(rng=rng,
input=layer0_conv.output,
input_shape=(batch_size, 6, 28, 28),
pool_size=(2, 2))
#the custom convolution layer: C4 in lecun
layer1_conv = CustomConvLayer(rng=rng,
input=layer0_subsample.output,
input_shape=(batch_size, 6, 14, 14),
filter_shape=(16, 6, 5, 5))
layer1_subsample = SubsampleLayer(rng=rng,
input=layer1_conv.output,
input_shape=(batch_size, 16, 10, 10),
pool_size=(2, 2))
layer2_conv = ConvolutionLayer(rng=rng,
input=layer1_subsample.output,
input_shape=(batch_size, 16, 5, 5),
filter_shape=(120, 16, 5, 5))
#flatten the output of the convpool layer for input to the MLP layer
layer3_input = layer2_conv.output.flatten(2)
layer3 = HiddenLayer(rng,
input=layer3_input,
n_in=120,
n_out=84,
activation=T.tanh)
#TODO: Change to RBF
layer4 = LogisticRegression(input=layer3.output, n_in=84, n_out=10)
cost = layer4.negative_log_likelihood(y)
params = layer4.params + layer3.params + \
layer2_conv.params + \
layer1_conv.params + layer1_subsample.params + \
layer0_conv.params + layer0_subsample.params
gradients = T.grad(cost, params)
updates = [(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, gradients)]
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[1]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
test_model = theano.function(
[index],
layer4.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
train_pred = theano.function(
[index],
layer4.y_pred,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
})
test_pred = theano.function([index],
layer4.y_pred,
givens={
x:
test_set_x[index * batch_size:(index + 1) *
batch_size],
})
return (train_model, train_pred, test_model, test_pred)
def __init__(self,
numpy_rng,
n_ins,
n_outs,
hidden_layers_sizes,
corruption_levels=[0.1, 0.1],
theano_rng=None):
""" This class is made to support a variable number of layers.
:type numpy_rng: numpy.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial
weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_ins: int
:param n_ins: dimension of the input to the sdA
:type n_layers_sizes: list of ints
:param n_layers_sizes: intermediate layers size, must contain
at least one value
:type n_outs: int
:param n_outs: dimension of the output of the network
:type corruption_levels: list of float
:param corruption_levels: amount of corruption to use for each
layer
"""
self.sigmoid_layers = []
self.dA_layers = []
self.params = []
self.n_layers = len(hidden_layers_sizes)
self.n_ins = n_ins
self.n_outs = n_outs
# allocate symbolic variables for the data
self.x = T.matrix('x')
self.y = T.ivector('y')
assert self.n_layers > 0
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2**30))
for i in xrange(self.n_layers):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0:
input_size = n_ins
else:
input_size = hidden_layers_sizes[i - 1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0:
layer_input = self.x
else:
layer_input = self.sigmoid_layers[-1].output
sigmoid_layer = HiddenLayer(rng=numpy_rng,
input=layer_input,
n_in=input_size,
n_out=hidden_layers_sizes[i],
activation=T.nnet.sigmoid)
# add the layer to our list of layers
self.sigmoid_layers.append(sigmoid_layer)
self.params.append(sigmoid_layer.theta)
# Construct a denoising autoencoder that shared weights with this layer
dA_layer = dA(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=layer_input,
n_visible=input_size,
n_hidden=hidden_layers_sizes[i],
theta=sigmoid_layer.theta)
self.dA_layers.append(dA_layer)
sda_input = T.matrix('sda_input')
self.da_layers_output_size = hidden_layers_sizes[-1]
self.get_da_output = theano.function(
inputs=[sda_input],
outputs=self.sigmoid_layers[-1].output.reshape(
(-1, self.da_layers_output_size)),
givens={self.x: sda_input})
self.logLayer = LogisticRegression(
rng=numpy.random.RandomState(),
input=self.sigmoid_layers[-1].output,
n_in=hidden_layers_sizes[-1],
n_out=n_outs)
#self.params.extend(self.logLayer.params)
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
self.errors = self.logLayer.errors(self.y)