def get_cost_updates(self, learning_rate=0.1, p=0, sigma=1): cost = self.get_cost(p, sigma) # compute the gradients of the cost of the `dA` with respect # to its parameters set_trace() gparams = T.grad(cost, self.params) # generate the list of updates updates = [ (param, param - learning_rate * gparam) for param, gparam in zip(self.params, gparams) ] return (cost, updates)
def test_model(batch_size=100, file_name='da.pkl'): # datasets = load_data(dataset) print '...loading data' datasets = load_TIMIT() train_set, valid_set, test_set = datasets print '...building model' pickle_lst = [1000] # , 500, 1000 # pickle_lst = [1, 10] for epoch in pickle_lst: print 'epoch: ', epoch file_name = "da_epoch_%d" % (epoch) w, b, b_prime = load_model_mat(file_name) # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch # generate symbolic variables for input (x and y represent a # minibatch) x = T.matrix('x') # data, presented as rasterized images rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = dA( numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=129, n_hidden=500, W=w, bhid=b, bvis=b_prime ) # test_fun = theano.function( # inputs=[index], # outputs=da.get_reconstructed_out(), # givens={ # x: test_set_x[index * batch_size:(index + 1) * batch_size] # } # ) get_outputs = theano.function( inputs=[index], outputs=da.get_active(), givens={ x: test_set[index * batch_size:(index + 1) * batch_size] } ) index = 1 hidden_value = get_outputs(index) plot_data = test_set.get_value(borrow=True)[index * batch_size:(index + 1) * batch_size] pylab.figure(); pylab.hist(plot_data.reshape(plot_data.size, 1), 50); pylab.figure();pylab.plot(numpy.mean(plot_data, axis=0), '*');pylab.xlim(0, 128);pylab.ylim(0, 1); pylab.figure();pylab.hist(hidden_value.reshape(hidden_value.size, 1), 50); pylab.figure();pylab.plot(numpy.mean(hidden_value, axis=0), '*');pylab.ylim(0, 1); pylab.show() set_trace() # pylab.title(epoch) pylab.show()
def __init__(self, numpy_rng, theano_rng=None, layers_sizes=[129, 500, 54, 500, 129], output_folder='out', p_dict=None, sigma_dict=None, file_path = '', logger=None): self.sigmoid_layers = [] self.params = [] self.finetune_train_flag = False self.n_layers = len(layers_sizes) self.n_layers_rmb = int(len(layers_sizes)/2) self.numpy_rng = numpy_rng self.output_folder = output_folder self.x = T.matrix(name='x') if not theano_rng: theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) if logger == None: self.logger = mylogger(output_folder + '/log.log') else: self.logger = logger assert self.n_layers > 0 if p_dict==None: self.p_list = [0]*self.n_layers_rmb self.sigma_list = [0]*self.n_layers_rmb self.p = 0 self.sigma = 0 elif p_dict!=None and sigma_dict==None: assert len(p_dict['p_list']) == self.n_layers_rmb self.p_list = p_dict['p_list'] self.sigma_list = [0]*self.n_layers_rmb self.p = p_dict['p'] self.sigma = 0 elif p_dict!=None and sigma_dict!=None: assert len(p_dict['p_list']) == self.n_layers_rmb assert len(sigma_dict['sigma_list']) == len(p_dict['p_list']) self.p_list = p_dict['p_list'] self.sigma_list = sigma_dict['sigma_list'] self.p = p_dict['p'] self.sigma = sigma_dict['sigma'] w = [0]*self.n_layers_rmb; hbias = [0]*self.n_layers_rmb; vbias = [0]*self.n_layers_rmb; # set_trace() w[0], hbias[0], vbias[0] = self.load_hinton_rbm(file_path+'/mnistvh.mat', 'vishid', 'hidrecbiases', 'visbiases') #rbm1 # w[1], hbias[1], vbias[1] = self.load_hinton_rbm(file_path+'/mnisthp.mat', 'hidpen', 'penrecbiases', 'hidgenbiases') #rbm2 # w[2], hbias[2], vbias[2] = self.load_hinton_rbm(file_path+'/mnisthp2.mat', 'hidpen2', 'penrecbiases2', 'hidgenbiases2') #rbm3 # w[3], hbias[3], vbias[3] = self.load_hinton_rbm(file_path+'/mnistpo.mat', 'hidtop', 'toprecbiases', 'topgenbiases') #rbm4 sigmoid_layer1 = HiddenLayer(rng=self.numpy_rng, input=self.x, n_in=w[0].get_value(borrow=True).shape[0], n_out=w[0].get_value(borrow=True).shape[1], W=w[0], b=hbias[0], activation=T.nnet.sigmoid) # sigmoid_layer2 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer1.output, # n_in=w[1].get_value(borrow=True).shape[0], n_out=w[1].get_value(borrow=True).shape[1], # W=w[1], b=hbias[1], activation=T.nnet.sigmoid) # sigmoid_layer3 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer2.output, # n_in=w[1].get_value(borrow=True).shape[1], n_out=w[1].get_value(borrow=True).shape[0], # W=theano.shared(numpy.asarray(w[1].T.eval(), dtype=theano.config.floatX), borrow=True), # b=vbias[1], activation=T.nnet.sigmoid) #sigmoid layer, inverse sigmoid_layer2 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer1.output, n_in=w[0].get_value(borrow=True).shape[1], n_out=w[0].get_value(borrow=True).shape[0], W=theano.shared(numpy.asarray(w[0].T.eval(), dtype=theano.config.floatX), borrow=True), b=vbias[0], activation=None) #output layer, inverse # m_idx = self.n_layers_rmb-1; # for i in xrange(m_idx): # if i == 0: # layer_input = self.x # else: # layer_input = self.sigmoid_layers[-1].output # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=layer_input, # n_in=w[i].get_value(borrow=True).shape[0], n_out=w[i].get_value(borrow=True).shape[1], # W=w[i], b=hbias[i], activation=T.nnet.sigmoid) #sigmoid layer # self.sigmoid_layers.append(sigmoid_layer) # # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output, # n_in=w[m_idx].get_value(borrow=True).shape[0], n_out=w[m_idx].get_value(borrow=True).shape[1], # W=w[m_idx], b=hbias[m_idx], activation=T.nnet.sigmoid) #coding layer # self.sigmoid_layers.append(sigmoid_layer) # # for i in xrange(m_idx,0,-1): # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output, # n_in=w[i].get_value(borrow=True).shape[1], n_out=w[i].get_value(borrow=True).shape[0], # W=theano.shared(numpy.asarray(w[i].T.eval(), dtype=theano.config.floatX), borrow=True), # b=vbias[i], activation=T.nnet.sigmoid) #sigmoid layer, inverse # self.sigmoid_layers.append(sigmoid_layer) # # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output, # n_in=w[0].get_value(borrow=True).shape[1], n_out=w[0].get_value(borrow=True).shape[0], # W=theano.shared(numpy.asarray(w[0].T.eval(), dtype=theano.config.floatX), borrow=True), # b=vbias[0], activation=None) #output layer, inverse # self.sigmoid_layers.append(sigmoid_layer) self.sigmoid_layers.append(sigmoid_layer1) self.sigmoid_layers.append(sigmoid_layer2) # self.sigmoid_layers.append(sigmoid_layer3) # self.sigmoid_layers.append(sigmoid_layer4) for sigmoid_layer in self.sigmoid_layers: self.params.extend(sigmoid_layer.params) set_trace() self.save_model_mat('%s/DAE_pre%s.mat'%(self.output_folder, self.get_model_file()))