def __init__(self, batch_size, n_ins,
hidden_layers_sizes, n_outs,
corruption_levels, rng, pretrain_lr, finetune_lr):
# Just to make sure those are not modified somewhere else afterwards
hidden_layers_sizes = copy.deepcopy(hidden_layers_sizes)
corruption_levels = copy.deepcopy(corruption_levels)
update_locals(self, locals())
self.layers = []
self.pretrain_functions = []
self.params = []
# MODIF: added this so we also get the b_primes
# (not used for finetuning... still using ".params")
self.all_params = []
self.n_layers = len(hidden_layers_sizes)
self.logistic_params = []
print "Creating SdA with params:"
print "batch_size", batch_size
print "hidden_layers_sizes", hidden_layers_sizes
print "corruption_levels", corruption_levels
print "n_ins", n_ins
print "n_outs", n_outs
print "pretrain_lr", pretrain_lr
print "finetune_lr", finetune_lr
print "----"
if len(hidden_layers_sizes) < 1 :
raiseException (' You must have at least one hidden layer ')
# allocate symbolic variables for the data
#index = T.lscalar() # index to a [mini]batch
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
self.finetune_lr = T.fscalar('finetune_lr') #To get a dynamic finetune learning rate
self.pretrain_lr = T.fscalar('pretrain_lr') #To get a dynamic pretrain learning rate
for i in xrange( self.n_layers ):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0 :
input_size = n_ins
else:
input_size = hidden_layers_sizes[i-1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0 :
layer_input = self.x
else:
layer_input = self.layers[-1].output
#We have to choose between sigmoidal layer or tanh layer !
## layer = SigmoidalLayer(rng, layer_input, input_size,
## hidden_layers_sizes[i] )
layer = TanhLayer(rng, layer_input, input_size,
hidden_layers_sizes[i] )
# add the layer to the
self.layers += [layer]
self.params += layer.params
# Construct a denoising autoencoder that shared weights with this
# layer
dA_layer = dA(input_size, hidden_layers_sizes[i], \
corruption_level = corruption_levels[0],\
input = layer_input, \
shared_W = layer.W, shared_b = layer.b)
self.all_params += dA_layer.params
# Construct a function that trains this dA
# compute gradients of layer parameters
gparams = T.grad(dA_layer.cost, dA_layer.params)
# compute the list of updates
updates = {}
for param, gparam in zip(dA_layer.params, gparams):
updates[param] = param - gparam * self.pretrain_lr
# create a function that trains the dA
update_fn = theano.function([self.x, self.pretrain_lr], dA_layer.cost, \
updates = updates)#,
# givens = {
# self.x : ensemble})
# collect this function into a list
#update_fn = theano.function([index], dA_layer.cost, \
# updates = updates,
# givens = {
# self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.shared_divider})
# collect this function into a list
self.pretrain_functions += [update_fn]
# We now need to add a logistic layer on top of the SDA
self.logLayer = LogisticRegression(\
input = self.layers[-1].output,\
n_in = hidden_layers_sizes[-1], n_out = n_outs)
self.params += self.logLayer.params
self.all_params += self.logLayer.params
# construct a function that implements one step of finetunining
# compute the cost, defined as the negative log likelihood
cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
gparams = T.grad(cost, self.params)
# compute list of updates
updates = {}
for param,gparam in zip(self.params, gparams):
updates[param] = param - gparam*self.finetune_lr
self.finetune = theano.function([self.x,self.y,self.finetune_lr], cost,
updates = updates)#,
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
#STRUCTURE FOR THE FINETUNING OF THE LOGISTIC REGRESSION ON THE TOP WITH
#ALL HIDDEN LAYERS AS INPUT
all_h=[]
for i in xrange(self.n_layers):
all_h.append(self.layers[i].output)
self.all_hidden=T.concatenate(all_h,axis=1)
self.logLayer2 = LogisticRegression(\
input = self.all_hidden,\
n_in = sum(hidden_layers_sizes), n_out = n_outs)
#n_in=hidden_layers_sizes[0],n_out=n_outs)
#self.logistic_params+= self.logLayer2.params
# construct a function that implements one step of finetunining
self.logistic_params+=self.logLayer2.params
# compute the cost, defined as the negative log likelihood
cost2 = self.logLayer2.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
gparams2 = T.grad(cost2, self.logistic_params)
# compute list of updates
updates2 = {}
for param,gparam in zip(self.logistic_params, gparams2):
updates2[param] = param - gparam*finetune_lr
self.finetune2 = theano.function([self.x,self.y], cost2,
updates = updates2)
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors2 = self.logLayer2.errors(self.y)
def __init__(self, train_set_x, train_set_y, batch_size, n_ins,
hidden_layers_sizes, n_outs,
corruption_levels, rng, pretrain_lr, finetune_lr, input_divider=1.0):
# Just to make sure those are not modified somewhere else afterwards
hidden_layers_sizes = copy.deepcopy(hidden_layers_sizes)
corruption_levels = copy.deepcopy(corruption_levels)
update_locals(self, locals())
self.layers = []
self.pretrain_functions = []
self.params = []
# MODIF: added this so we also get the b_primes
# (not used for finetuning... still using ".params")
self.all_params = []
self.n_layers = len(hidden_layers_sizes)
print "Creating SdA with params:"
print "batch_size", batch_size
print "hidden_layers_sizes", hidden_layers_sizes
print "corruption_levels", corruption_levels
print "n_ins", n_ins
print "n_outs", n_outs
print "pretrain_lr", pretrain_lr
print "finetune_lr", finetune_lr
print "input_divider", input_divider
print "----"
self.shared_divider = theano.shared(numpy.asarray(input_divider, dtype=theano.config.floatX))
if len(hidden_layers_sizes) < 1 :
raiseException (' You must have at least one hidden layer ')
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
self.x = T.matrix('x') # the data is presented as rasterized images
self.y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
for i in xrange( self.n_layers ):
# construct the sigmoidal layer
# the size of the input is either the number of hidden units of
# the layer below or the input size if we are on the first layer
if i == 0 :
input_size = n_ins
else:
input_size = hidden_layers_sizes[i-1]
# the input to this layer is either the activation of the hidden
# layer below or the input of the SdA if you are on the first
# layer
if i == 0 :
layer_input = self.x
else:
layer_input = self.layers[-1].output
layer = SigmoidalLayer(rng, layer_input, input_size,
hidden_layers_sizes[i] )
# add the layer to the
self.layers += [layer]
self.params += layer.params
# Construct a denoising autoencoder that shared weights with this
# layer
dA_layer = dA(input_size, hidden_layers_sizes[i], \
corruption_level = corruption_levels[0],\
input = layer_input, \
shared_W = layer.W, shared_b = layer.b)
self.all_params += dA_layer.params
# Construct a function that trains this dA
# compute gradients of layer parameters
gparams = T.grad(dA_layer.cost, dA_layer.params)
# compute the list of updates
updates = {}
for param, gparam in zip(dA_layer.params, gparams):
updates[param] = param - gparam * pretrain_lr
# create a function that trains the dA
update_fn = theano.function([index], dA_layer.cost, \
updates = updates,
givens = {
self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.shared_divider})
# collect this function into a list
self.pretrain_functions += [update_fn]
# We now need to add a logistic layer on top of the MLP
self.logLayer = LogisticRegression(\
input = self.layers[-1].output,\
n_in = hidden_layers_sizes[-1], n_out = n_outs)
self.params += self.logLayer.params
self.all_params += self.logLayer.params
# construct a function that implements one step of finetunining
# compute the cost, defined as the negative log likelihood
cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
gparams = T.grad(cost, self.params)
# compute list of updates
updates = {}
for param,gparam in zip(self.params, gparams):
updates[param] = param - gparam*finetune_lr
self.finetune = theano.function([index], cost,
updates = updates,
givens = {
self.x : train_set_x[index*batch_size:(index+1)*batch_size]/self.shared_divider,
self.y : train_set_y[index*batch_size:(index+1)*batch_size]} )
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
