def __init__(self, np_rng, theano_rng=None, n_ins=784, hidden_layer_sizes=[500, 500], n_outs=10, corruption_rates=[0.1, 0.1]): self.sigmoid_layers = [] self.dA_layers = [] self.params = [] self.n_layers = len(hidden_layer_sizes) if theano_rng is None: theano_rng = RandomStreams(np_rng.randint(2**30)) self.x = T.matrix('x') self.y = T.ivector('y') for i in range(self.n_layers): if i == 0: input_size = n_ins layer_input = self.x else: input_size = hidden_layer_sizes[i-1] layer_input = self.sigmoid_layers[-1].output hidden_layer = HiddenLayer( np_rng, layer_input, input_size, hidden_layer_sizes[i], sigmoid) self.sigmoid_layers.append(hidden_layer) self.params.extend(hidden_layer.params) # construc the denoising autoencoder layer da = dA(np_rng, theano_rng, layer_input, n_visible=input_size, n_hidden=hidden_layer_sizes[i], W=hidden_layer.W, bhid=hidden_layer.b) self.dA_layers.append(da) # LogisticRegression Layer for classification self.logLayer = LogisticRegressionLayer(self.sigmoid_layers[-1].output, n_in=hidden_layer_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) self.errors = self.logLayer.errors(self.y)
def __init__(self, np_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10): self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) if theano_rng is None: theano_rng = MRG_RandomStreams(np_rng.randint(2**30)) # allocate symbolic variables for the data self.x = T.matrix('x') self.y = T.ivector('y') # end snippet -1 for i in xrange(self.n_layers): # construct the sigmoidal layer if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # the input to this layer is either the activation of # the hidden layer below or the input of the DBN if you # are on the first layer if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=np_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) # add the sigmoid layer to layer list self.sigmoid_layers.append(sigmoid_layer) self.params.extend(sigmoid_layer.params) # construct an RBM that shared weights with this layer rbm_layer = RBM( np_rng=np_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We need to add a Logistic layer on top of MLP self.logLayer = LogisticRegressionLayer(input=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined # as the negative log likelihood of the logistic regression self.finetune_cost = self.logLayer.negative_log_likehihood(self.y) # compuate the gradientes with respect to the model parameters # symbolic variable that points to the number of errors # made on the mibibatch given by self.x and self.y self.erros = self.logLayer.errors(self.y)
class sdA(object): """Stacked denoising_autoencoder""" def __init__(self, np_rng, theano_rng=None, n_ins=784, hidden_layer_sizes=[500, 500], n_outs=10, corruption_rates=[0.1, 0.1]): self.sigmoid_layers = [] self.dA_layers = [] self.params = [] self.n_layers = len(hidden_layer_sizes) if theano_rng is None: theano_rng = RandomStreams(np_rng.randint(2**30)) self.x = T.matrix('x') self.y = T.ivector('y') for i in range(self.n_layers): if i == 0: input_size = n_ins layer_input = self.x else: input_size = hidden_layer_sizes[i-1] layer_input = self.sigmoid_layers[-1].output hidden_layer = HiddenLayer( np_rng, layer_input, input_size, hidden_layer_sizes[i], sigmoid) self.sigmoid_layers.append(hidden_layer) self.params.extend(hidden_layer.params) # construc the denoising autoencoder layer da = dA(np_rng, theano_rng, layer_input, n_visible=input_size, n_hidden=hidden_layer_sizes[i], W=hidden_layer.W, bhid=hidden_layer.b) self.dA_layers.append(da) # LogisticRegression Layer for classification self.logLayer = LogisticRegressionLayer(self.sigmoid_layers[-1].output, n_in=hidden_layer_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) self.errors = self.logLayer.errors(self.y) def pretraining_functions(self, train_set_x, batch_size): index = T.lscalar('index') corruption_rate = T.scalar('corruption') learning_rate = T.scalar('lr') batch_begin = index * batch_size batch_end = batch_begin + batch_size pretrain_fns = [] for da in self.dA_layers: cost, updates = da.get_cost_updates(corruption_rate, learning_rate) fn = theano.function( inputs=[index, theano.Param(corruption_rate, default=0.2), theano.Param(learning_rate, default=0.1)], outputs=cost, updates=updates, givens={ self.x: train_set_x[batch_begin:batch_end] } ) pretrain_fns.append(fn) return pretrain_fns def build_finetune_functions(self, datasets, batch_size, learning_rate): train_set_x, train_set_y = datasets[0] valid_set_x, valid_set_y = datasets[1] test_set_x, test_set_y = datasets[1] n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size index = T.lscalar('index') gparams = T.grad(self.finetune_cost, self.params) updates = [(p, p - learning_rate*g) for p, g in zip(self.params, gparams)] train_fn = theano.function( inputs=[index], outputs=self.finetune_cost, updates=updates, givens={ self.x: train_set_x[index*batch_size:(index+1)*batch_size], self.y: train_set_y[index*batch_size:(index+1)*batch_size] }, name='train' ) test_score_i = theano.function( [index], self.errors, givens={ self.x: test_set_x[index*batch_size:(index+1)*batch_size], self.y: test_set_y[index*batch_size:(index+1):batch_size] }, name='test' ) valid_score_i = theano.function( [index], self.errors, givens={ self.x: valid_set_x[index*batch_size:(index+1)*batch_size], self.y: valid_set_y[index*batch_size:(index+1)*batch_size] }, name='valid' ) # create a function that scans the entire validation set def valid_score(self): return [valid_score_i(i) for i in range(n_valid_batches)] def test_score(self): return [test_score_i(i) for i in range(n_test_batches)] return train_fn, valid_score, test_score
class DBN(object): """Deep belief network""" def __init__(self, np_rng, theano_rng=None, n_ins=784, hidden_layers_sizes=[500, 500], n_outs=10): self.sigmoid_layers = [] self.rbm_layers = [] self.params = [] self.n_layers = len(hidden_layers_sizes) if theano_rng is None: theano_rng = MRG_RandomStreams(np_rng.randint(2**30)) # allocate symbolic variables for the data self.x = T.matrix('x') self.y = T.ivector('y') # end snippet -1 for i in xrange(self.n_layers): # construct the sigmoidal layer if i == 0: input_size = n_ins else: input_size = hidden_layers_sizes[i - 1] # the input to this layer is either the activation of # the hidden layer below or the input of the DBN if you # are on the first layer if i == 0: layer_input = self.x else: layer_input = self.sigmoid_layers[-1].output sigmoid_layer = HiddenLayer(rng=np_rng, input=layer_input, n_in=input_size, n_out=hidden_layers_sizes[i], activation=T.nnet.sigmoid) # add the sigmoid layer to layer list self.sigmoid_layers.append(sigmoid_layer) self.params.extend(sigmoid_layer.params) # construct an RBM that shared weights with this layer rbm_layer = RBM( np_rng=np_rng, theano_rng=theano_rng, input=layer_input, n_visible=input_size, n_hidden=hidden_layers_sizes[i], W=sigmoid_layer.W, hbias=sigmoid_layer.b) self.rbm_layers.append(rbm_layer) # We need to add a Logistic layer on top of MLP self.logLayer = LogisticRegressionLayer(input=self.sigmoid_layers[-1].output, n_in=hidden_layers_sizes[-1], n_out=n_outs) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined # as the negative log likelihood of the logistic regression self.finetune_cost = self.logLayer.negative_log_likehihood(self.y) # compuate the gradientes with respect to the model parameters # symbolic variable that points to the number of errors # made on the mibibatch given by self.x and self.y self.erros = self.logLayer.errors(self.y) def pretraining_functions(self, train_set_x, batch_size, k): index = T.lscalar('index') learning_rate = T.scalar('lr') # compute number of batches batch_begin = index * batch_size batch_end = batch_begin + batch_size pretrain_fns = [] for rbm in self.rbm_layers: # get the cost and the updates list # using CD-k here cost, updates = rbm.get_cost_updates(learning_rate, persistent=None, k=k) # compile the theano function fn = theano.function(inputs=[index, theano.Param(learning_rate, default=0.1)], outputs=cost, updates=updates, givens={ self.x: train_set_x[batch_begin:batch_end] }) pretrain_fns.append(fn) return pretrain_fns def build_finetune_functions(self, datasets, batch_size, learning_rate): (train_set_x, train_set_y) = datasets[0] (valid_set_x, valid_set_y) = datasets[1] (test_set_x, test_set_y) = datasets[2] # compute number of minibatches for training, validation and testing n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] n_valid_batches /= batch_size n_test_batches = test_set_x.get_value(borrow=True).shape[0] n_test_batches /= batch_size index = T.lscalar('index') gparams = T.grad(self.finetune_cost, self.params) updates = [] for param, gparam in zip(self.params, gparams): updates.append((param, param - gparam*learning_rate)) train_fn = theano.function( inputs=[index], outputs=self.finetune_cost, updates=updates, givens={ self.x: train_set_x[index*batch_size:(index+1)*batch_size], self.y: train_set_y[index*batch_size:(index+1)*batch_size] } ) test_score_i = theano.function( [index], self.errors, givens={ self.x: test_set_x[index*batch_size:(index+1)*batch_size], self.y: test_set_y[index*batch_size:(index+1)*batch_size] } ) valid_score_i = theano.function( [index], outputs=self.erros, givens={ self.x: valid_set_x[index*batch_size:(index+1)*batch_size], self.y: valid_set_y[index*batch_size:(index+1)*batch_size] } ) # construct a function that scans the entire validation set def valid_score(): return [valid_score_i(i) for i in xrange(n_valid_batches)] def test_score(): return [test_score_i(i) for i in xrange(n_test_batches)] return train_fn, valid_score, test_score