def __init__(self, batch_size, n_ins, hidden_layers_sizes, n_outs, corruption_levels, rng, pretrain_lr, finetune_lr): # Just to make sure those are not modified somewhere else afterwards hidden_layers_sizes = copy.deepcopy(hidden_layers_sizes) corruption_levels = copy.deepcopy(corruption_levels) update_locals(self, locals()) self.layers = [] self.pretrain_functions = [] self.params = [] # MODIF: added this so we also get the b_primes # (not used for finetuning... still using ".params") self.all_params = [] self.n_layers = len(hidden_layers_sizes) self.logistic_params = [] print "Creating SdA with params:" print "batch_size", batch_size print "hidden_layers_sizes", hidden_layers_sizes print "corruption_levels", corruption_levels print "n_ins", n_ins print "n_outs", n_outs print "pretrain_lr", pretrain_lr print "finetune_lr", finetune_lr print "----" if len(hidden_layers_sizes) < 1 : raiseException (' You must have at least one hidden layer ') # allocate symbolic variables for the data #index = T.lscalar() # index to a [mini]batch self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels self.finetune_lr = T.fscalar('finetune_lr') #To get a dynamic finetune learning rate self.pretrain_lr = T.fscalar('pretrain_lr') #To get a dynamic pretrain learning rate for i in xrange( self.n_layers ): # construct the sigmoidal layer # the size of the input is either the number of hidden units of # the layer below or the input size if we are on the first layer if i == 0 : input_size = n_ins else: input_size = hidden_layers_sizes[i-1] # the input to this layer is either the activation of the hidden # layer below or the input of the SdA if you are on the first # layer if i == 0 : layer_input = self.x else: layer_input = self.layers[-1].output #We have to choose between sigmoidal layer or tanh layer ! ## layer = SigmoidalLayer(rng, layer_input, input_size, ## hidden_layers_sizes[i] ) layer = TanhLayer(rng, layer_input, input_size, hidden_layers_sizes[i] ) # add the layer to the self.layers += [layer] self.params += layer.params # Construct a denoising autoencoder that shared weights with this # layer dA_layer = dA(input_size, hidden_layers_sizes[i], \ corruption_level = corruption_levels[0],\ input = layer_input, \ shared_W = layer.W, shared_b = layer.b) self.all_params += dA_layer.params # Construct a function that trains this dA # compute gradients of layer parameters gparams = T.grad(dA_layer.cost, dA_layer.params) # compute the list of updates updates = {} for param, gparam in zip(dA_layer.params, gparams): updates[param] = param - gparam * self.pretrain_lr # create a function that trains the dA update_fn = theano.function([self.x, self.pretrain_lr], dA_layer.cost, \ updates = updates)#, # givens = { # self.x : ensemble}) # collect this function into a list #update_fn = theano.function([index], dA_layer.cost, \ # updates = updates, # givens = { # self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.shared_divider}) # collect this function into a list self.pretrain_functions += [update_fn] # We now need to add a logistic layer on top of the SDA self.logLayer = LogisticRegression(\ input = self.layers[-1].output,\ n_in = hidden_layers_sizes[-1], n_out = n_outs) self.params += self.logLayer.params self.all_params += self.logLayer.params # construct a function that implements one step of finetunining # compute the cost, defined as the negative log likelihood cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters gparams = T.grad(cost, self.params) # compute list of updates updates = {} for param,gparam in zip(self.params, gparams): updates[param] = param - gparam*self.finetune_lr self.finetune = theano.function([self.x,self.y,self.finetune_lr], cost, updates = updates)#, # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logLayer.errors(self.y) #STRUCTURE FOR THE FINETUNING OF THE LOGISTIC REGRESSION ON THE TOP WITH #ALL HIDDEN LAYERS AS INPUT all_h=[] for i in xrange(self.n_layers): all_h.append(self.layers[i].output) self.all_hidden=T.concatenate(all_h,axis=1) self.logLayer2 = LogisticRegression(\ input = self.all_hidden,\ n_in = sum(hidden_layers_sizes), n_out = n_outs) #n_in=hidden_layers_sizes[0],n_out=n_outs) #self.logistic_params+= self.logLayer2.params # construct a function that implements one step of finetunining self.logistic_params+=self.logLayer2.params # compute the cost, defined as the negative log likelihood cost2 = self.logLayer2.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters gparams2 = T.grad(cost2, self.logistic_params) # compute list of updates updates2 = {} for param,gparam in zip(self.logistic_params, gparams2): updates2[param] = param - gparam*finetune_lr self.finetune2 = theano.function([self.x,self.y], cost2, updates = updates2) # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors2 = self.logLayer2.errors(self.y)
def __init__(self, train_set_x, train_set_y, batch_size, n_ins, hidden_layers_sizes, n_outs, corruption_levels, rng, pretrain_lr, finetune_lr, input_divider=1.0): # Just to make sure those are not modified somewhere else afterwards hidden_layers_sizes = copy.deepcopy(hidden_layers_sizes) corruption_levels = copy.deepcopy(corruption_levels) update_locals(self, locals()) self.layers = [] self.pretrain_functions = [] self.params = [] # MODIF: added this so we also get the b_primes # (not used for finetuning... still using ".params") self.all_params = [] self.n_layers = len(hidden_layers_sizes) print "Creating SdA with params:" print "batch_size", batch_size print "hidden_layers_sizes", hidden_layers_sizes print "corruption_levels", corruption_levels print "n_ins", n_ins print "n_outs", n_outs print "pretrain_lr", pretrain_lr print "finetune_lr", finetune_lr print "input_divider", input_divider print "----" self.shared_divider = theano.shared(numpy.asarray(input_divider, dtype=theano.config.floatX)) if len(hidden_layers_sizes) < 1 : raiseException (' You must have at least one hidden layer ') # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch self.x = T.matrix('x') # the data is presented as rasterized images self.y = T.ivector('y') # the labels are presented as 1D vector of # [int] labels for i in xrange( self.n_layers ): # construct the sigmoidal layer # the size of the input is either the number of hidden units of # the layer below or the input size if we are on the first layer if i == 0 : input_size = n_ins else: input_size = hidden_layers_sizes[i-1] # the input to this layer is either the activation of the hidden # layer below or the input of the SdA if you are on the first # layer if i == 0 : layer_input = self.x else: layer_input = self.layers[-1].output layer = SigmoidalLayer(rng, layer_input, input_size, hidden_layers_sizes[i] ) # add the layer to the self.layers += [layer] self.params += layer.params # Construct a denoising autoencoder that shared weights with this # layer dA_layer = dA(input_size, hidden_layers_sizes[i], \ corruption_level = corruption_levels[0],\ input = layer_input, \ shared_W = layer.W, shared_b = layer.b) self.all_params += dA_layer.params # Construct a function that trains this dA # compute gradients of layer parameters gparams = T.grad(dA_layer.cost, dA_layer.params) # compute the list of updates updates = {} for param, gparam in zip(dA_layer.params, gparams): updates[param] = param - gparam * pretrain_lr # create a function that trains the dA update_fn = theano.function([index], dA_layer.cost, \ updates = updates, givens = { self.x : train_set_x[index*batch_size:(index+1)*batch_size] / self.shared_divider}) # collect this function into a list self.pretrain_functions += [update_fn] # We now need to add a logistic layer on top of the MLP self.logLayer = LogisticRegression(\ input = self.layers[-1].output,\ n_in = hidden_layers_sizes[-1], n_out = n_outs) self.params += self.logLayer.params self.all_params += self.logLayer.params # construct a function that implements one step of finetunining # compute the cost, defined as the negative log likelihood cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters gparams = T.grad(cost, self.params) # compute list of updates updates = {} for param,gparam in zip(self.params, gparams): updates[param] = param - gparam*finetune_lr self.finetune = theano.function([index], cost, updates = updates, givens = { self.x : train_set_x[index*batch_size:(index+1)*batch_size]/self.shared_divider, self.y : train_set_y[index*batch_size:(index+1)*batch_size]} ) # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logLayer.errors(self.y)