def get_L1_out(train_set=None, model_file=None, N=1, shared=1): if model_file == None: model_file = 'L1_p0_s1' output_folder = 'auto_encoder_out_N%s' %(N) model_w, model_b, model_b_prime = load_model_mat(file_name=model_file, output_folder=output_folder, shared=0) # set_trace() hidden_value = 1 / (1 + numpy.exp(-(numpy.dot(train_set, model_w) + model_b))) if shared == 1: hidden_value = theano.shared(numpy.asarray(hidden_value, dtype=theano.config.floatX), borrow=True) return hidden_value
def get_hidden(train_set=None, model_file=None, shared=1): if model_file == None: model_file = 'L1_p0_s1' model_w, model_b, model_b_prime = load_model_mat(model_file) n_visible,n_hidden = model_w.get_value(borrow=True).shape rng = numpy.random.RandomState(123) da = dA(numpy_rng=rng, input=train_set, n_visible=n_visible, n_hidden=n_hidden, W=model_w, bhid=model_b, bvis=model_b_prime) # set_trace() hidden_value = da.get_active().eval() if shared == 1: hidden_value = theano.shared(numpy.asarray(hidden_value, dtype=theano.config.floatX), borrow=True) return hidden_value
def __init__(self, numpy_rng, theano_rng=None, layers_sizes=[129, 500, 54, 500, 129], w_list=None, b_list=None, output_folder='out', p_dict=None, sigma_dict=None): self.dA_layers = [] self.dA_train_flags = [] self.finetune_train_flag = False self.n_layers = len(layers_sizes) self.n_layers_dA = int(len(layers_sizes)/2) self.numpy_rng = numpy_rng self.output_folder = output_folder assert self.n_layers > 0 if p_dict==None: self.p_list = [0]*self.n_layers_dA self.sigma_list = [0]*self.n_layers_dA self.p = 0 self.sigma = 0 elif p_dict!=None and sigma_dict==None: assert len(p_dict['p_list']) == self.n_layers_dA self.p_list = p_dict['p_list'] self.sigma_list = [0]*self.n_layers_dA self.p = p_dict['p'] self.sigma = 0 elif p_dict!=None and sigma_dict!=None: assert len(p_dict['p_list']) == self.n_layers_dA assert len(sigma_dict['sigma_list']) == len(p_dict['p_list']) self.p_list = p_dict['p_list'] self.sigma_list = sigma_dict['sigma_list'] self.p = p_dict['p'] self.sigma = sigma_dict['sigma'] if not theano_rng: self.theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) self.x = T.matrix(name='x') for i in xrange(self.n_layers_dA): if i == 0: layer_input = self.x else: layer_input = self.dA_layers[-1].get_active() if(self.p_list[i] == 0): model_file = '%s/L%d.mat'%(output_folder, i) else: model_file = '%s/L%d_p%g_s%g.mat'%(output_folder, i, self.p_list[i], self.sigma_list[i]) if os.path.isfile(model_file): #this layer has been trained model_w, model_b, model_b_prime = load_model_mat(model_file) dA_layer = dA(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1], W=model_w, bhid=model_b, bvis=model_b_prime) self.dA_train_flags.append(True) #set the flag else: dA_layer = dA(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1]) self.dA_train_flags.append(False) #set the flag self.dA_layers.append(dA_layer) finetune_file = '%s/SAE%s.mat'%(output_folder, self.get_model_file()) if os.path.isfile(finetune_file): #trained self.finetune_train_flag = True
def test_model(batch_size=100, file_name='da.pkl'): # datasets = load_data(dataset) print '...loading data' datasets = load_TIMIT() train_set, valid_set, test_set = datasets print '...building model' pickle_lst = [1000] # , 500, 1000 # pickle_lst = [1, 10] for epoch in pickle_lst: print 'epoch: ', epoch file_name = "da_epoch_%d" % (epoch) w, b, b_prime = load_model_mat(file_name) # allocate symbolic variables for the data index = T.lscalar() # index to a [mini]batch # generate symbolic variables for input (x and y represent a # minibatch) x = T.matrix('x') # data, presented as rasterized images rng = numpy.random.RandomState(123) theano_rng = RandomStreams(rng.randint(2 ** 30)) da = dA( numpy_rng=rng, theano_rng=theano_rng, input=x, n_visible=129, n_hidden=500, W=w, bhid=b, bvis=b_prime ) # test_fun = theano.function( # inputs=[index], # outputs=da.get_reconstructed_out(), # givens={ # x: test_set_x[index * batch_size:(index + 1) * batch_size] # } # ) get_outputs = theano.function( inputs=[index], outputs=da.get_active(), givens={ x: test_set[index * batch_size:(index + 1) * batch_size] } ) index = 1 hidden_value = get_outputs(index) plot_data = test_set.get_value(borrow=True)[index * batch_size:(index + 1) * batch_size] pylab.figure(); pylab.hist(plot_data.reshape(plot_data.size, 1), 50); pylab.figure();pylab.plot(numpy.mean(plot_data, axis=0), '*');pylab.xlim(0, 128);pylab.ylim(0, 1); pylab.figure();pylab.hist(hidden_value.reshape(hidden_value.size, 1), 50); pylab.figure();pylab.plot(numpy.mean(hidden_value, axis=0), '*');pylab.ylim(0, 1); pylab.show() set_trace() # pylab.title(epoch) pylab.show()
def auto_encoder_finetune(datasets=None, p=0, sigma=1, param=None, training_epochs=1000): if datasets == None: datasets = load_TIMIT() train_set, valid_set, test_set = datasets def get_shared(x): return theano.shared(numpy.asarray(x, dtype=theano.config.floatX), borrow=False) layers_sizes = param['layers_sizes'] L1_file = param['L1_file'] L2_file = param['L2_file'] L3_file = param['L3_file'] L4_file = param['L4_file'] output_folder = param['output_folder'] item_str = param['item_str'] valid_flag = 0 finetune_lr = 0.1 batch_size = 100 # compute number of minibatches for training, validation and testing n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size # allocate symbolic variables for the data index = T.lscalar('index') learning_rate = T.scalar('lr') x = T.matrix('x') if not os.path.isdir(output_folder): os.makedirs(output_folder) # pickle_lst = [100,200,300,400,500,600,700,800,900,1000] L1_w, L1_b, L1_b_prime = load_model_mat(file_name=L1_file, shared=0) L2_w, L2_b, L2_b_prime = load_model_mat(file_name=L2_file, shared=0) L3_w, L3_b, L3_b_prime = load_model_mat(file_name=L3_file, shared=0) L4_w, L4_b, L4_b_prime = load_model_mat(file_name=L4_file, shared=0) w_list = [get_shared(L1_w), get_shared(L2_w), get_shared(L3_w), get_shared(L4_w), get_shared(L4_w.T), get_shared(L3_w.T), get_shared(L2_w.T), get_shared(L1_w.T)] b_list = [get_shared(L1_b), get_shared(L2_b), get_shared(L3_b), get_shared(L4_b), get_shared(L4_b_prime), get_shared(L3_b_prime), get_shared(L2_b_prime), get_shared(L1_b_prime)] rng = numpy.random.RandomState(123) sae = SAE(numpy_rng=rng, input=x, layers_sizes=layers_sizes, w_list=w_list, b_list=b_list) print '... building the model' train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size, learning_rate, p, sigma) print '... training' logger = mylogger(output_folder + '/' + item_str + '.log') logger.log('p:%g, sigma:%g, learning rate:%g' % (p, sigma, finetune_lr)) #=========================================================================== # start training #=========================================================================== patience = 100 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is found improvement_threshold = 0.999 # a relative improvement of this much is considered significant validation_frequency = min(n_train_batches, patience / 2) best_validation_loss = numpy.inf test_score = 0. start_time = strict_time() done_looping = False epoch = 0;best_epoch = 0 while (epoch < training_epochs): epoch = epoch + 1 epoch_time_s = strict_time() c = [] for minibatch_index in xrange(n_train_batches): err = train_fn(minibatch_index,finetune_lr) # err = 0 c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % 100 == 0: finetune_lr = 0.8 * finetune_lr logger.log('learning rate: %g' % (finetune_lr)) if valid_flag == 0: continue validation_losses = numpy.mean(valid_model()) logger.log('valid %.5f' % (validation_losses)) if validation_losses < best_validation_loss: best_validation_loss = validation_losses best_epoch = epoch # test it on the test set test_losses = numpy.mean(test_model()) logger.log('test %.5f' % (test_losses)) sae.save_model_mat(output_folder + '/' + item_str + '.mat') # logger.log('best validation %.5f, test error %.5f, on epoch %d'%(best_validation_loss, test_losses, best_epoch)) sae.save_model_mat(output_folder + '/' + item_str + '.mat') logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.)) return for epoch in xrange(1, training_epochs + 1): # go through trainng set c = [] epoch_time_s = strict_time() for batch_index in xrange(n_train_batches): err = train_fn(batch_index, finetune_lr) # err = 0 c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % 100 == 0: finetune_lr = 0.8 * finetune_lr logger.log('learning rate: %g' % (finetune_lr)) sae.save_model_mat(output_folder + '/' + item_str + '.mat') logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))