def train_gb_rbm(batch_size=100,epochs=50): output_folder = 'gb_rbm' data_file = 'rbm_TIMIT_dr2_(N1)_split.mat' datasets = load_TIMIT(data_file) train_set, valid_set, test_set = datasets numpy_rng = numpy.random.RandomState(123) theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) input_dim = train_set.get_value(borrow=True).shape[1] layers_sizes = [input_dim,70,input_dim] input_x = T.matrix(name='x') if not os.path.isdir(output_folder): os.makedirs(output_folder) logger = mylogger(output_folder + '/log.log') gb_rbm_layer = GBRBM(numpy_rng=numpy_rng, theano_rng=theano_rng, input=input_x, n_visible=layers_sizes[0], n_hidden=layers_sizes[1]) index = T.lscalar('index') momentum = T.scalar('momentum') learning_rate = T.scalar('lr') # number of mini-batches n_batches = train_set.get_value(borrow=True).shape[0] / batch_size # start and end index of this mini-batch batch_begin = index * batch_size batch_end = batch_begin + batch_size r_cost, fe_cost, updates = gb_rbm_layer.get_cost_updates(batch_size, learning_rate, momentum, weight_cost=0.0002, persistent=None, k = 1) # compile the theano function fn = theano.function(inputs=[index, theano.Param(learning_rate, default=0.0001), theano.Param(momentum, default=0.5)], outputs= [r_cost, fe_cost], updates=updates, givens={input_x: train_set[batch_begin:batch_end]}) r_c, fe_c = [], [] # keep record of reconstruction and free-energy cost for epoch in range(epochs): for batch_index in xrange(n_batches): # loop over mini-batches [reconstruction_cost, free_energy_cost] = fn(index=batch_index) r_c.append(reconstruction_cost) fe_c.append(free_energy_cost) logger.log('pre-training, epoch %d, r_cost %f, fe_cost %f' % (epoch, numpy.mean(r_c), numpy.mean(fe_c))) params = [] for item in gb_rbm_layer.params: params.append(item.get_value(borrow=True)) savemat(output_folder+'/gb_rbm.mat', {'params':params})
def __init__(self, numpy_rng, theano_rng=None, layers_sizes=[129, 500, 54, 500, 129], output_folder='out', p=0, sigma=0, logger=None): self.rbm_layers = [] self.rbm_train_flags = [] self.finetune_train_flag = False self.n_layers = len(layers_sizes) self.n_layers_rmb = int(len(layers_sizes) / 2) self.numpy_rng = numpy_rng self.output_folder = output_folder if logger == None: self.logger = mylogger(output_folder + '/log.log') else: self.logger = logger assert self.n_layers > 0 self.p = p self.sigma = sigma if not theano_rng: self.theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) else: self.theano_rng = theano_rng self.x = T.matrix(name='x') for i in xrange(self.n_layers_rmb): if i == 0: layer_input = self.x else: layer_input = self.rbm_layers[-1].get_active() model_file = '%s/L%d.mat' % (output_folder, i) if os.path.isfile(model_file): # this layer has been trained w, hbias, vbias = self.load_RMB_model(i) rbm_layer = GBRBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i + 1], W=w, hbias=hbias, vbias=vbias) self.rbm_train_flags.append(True) # set the flag else: rbm_layer = GBRBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i + 1]) self.rbm_train_flags.append(False) # set the flag self.rbm_layers.append(rbm_layer) finetune_file = '%s/DAE%s.mat' % (output_folder, self.get_model_file()) if os.path.isfile(finetune_file): # trained self.finetune_train_flag = True
def train_autoencoder(datasets,layers_sizes,output_folder,p_dict,sigma_dict): train_set, valid_set, test_set = datasets if not os.path.isdir(output_folder): os.makedirs(output_folder) pre_epochs=20 fine_epochs=100 finetune_lr = 0.001 batch_size = 100 down_epoch = 100 n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size logger = mylogger(output_folder + '/log.log') logger.log('layer size: '+''.join(['%d '%i for i in layers_sizes]).strip()) file_path = '../../Autoencoder_hinton' rng = numpy.random.RandomState(123) dae = autoencoder_hinton(numpy_rng=rng, layers_sizes=layers_sizes, output_folder=output_folder, logger=logger, file_path=file_path) start_time = strict_time() p = p_dict['p'] sigma = sigma_dict['sigma'] logger.log('fine tuning...') if dae.finetune_train_flag == True: logger.log('SAE trained') return logger.log('learning rate:%g, p:%g, sigma:%g' % (finetune_lr, p, sigma)) train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train for epoch in xrange(1, fine_epochs + 1): epoch_time_s = strict_time() c = [] for minibatch_index in xrange(n_train_batches): err = train_fn(minibatch_index, finetune_lr) c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % down_epoch == 0: finetune_lr = 0.8 * finetune_lr logger.log('learning rate: %g' % (finetune_lr)) # if epoch in epoch_list: # dae.save_model('%s/SAE_p%g_s%g_(e_%g).mat'%(output_folder, p, sigma, epoch)) dae.save_model() logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
def train_SAE(datasets,layers_sizes,output_folder,p_dict,sigma_dict): train_set, valid_set, test_set = datasets if not os.path.isdir(output_folder): os.makedirs(output_folder) pre_epochs=200 fine_epochs=200 epoch_list = [1, 10] finetune_lr = 0.1 batch_size = 100 down_epoch = 100 n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size # logger = mylogger(output_folder + '/log_(L-1_sx).log') logger = mylogger(output_folder + '/log.log') logger.log('layer size: '+''.join(['%d '%i for i in layers_sizes]).strip()) p_list=p_dict['p_list'] sigma_list=sigma_dict['sigma_list'] rng = numpy.random.RandomState(123) sae = SAE(numpy_rng=rng, layers_sizes=layers_sizes, output_folder=output_folder, p_dict=p_dict, sigma_dict=sigma_dict) pretraining_fns = sae.pretraining_functions(train_set, batch_size) start_time = strict_time() logger.log('pre_training...') for i in xrange(len(sae.dA_layers)): if sae.dA_train_flags[i] == True: logger.log('L%d trained'%(i)) continue pretrain_lr = 0.05 logger.log('training L%d ...'%(i)) logger.log('learning rate:%g, p:%g, sigma:%g' % (pretrain_lr, p_list[i], sigma_list[i])) for epoch in xrange(1, pre_epochs + 1): epoch_time_s = strict_time() c = [] for batch_index in xrange(n_train_batches): c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr)) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % down_epoch == 0: pretrain_lr = 0.8 * pretrain_lr logger.log('learning rate: %g' % (pretrain_lr)) if(p_list[i] == 0): model_file = '%s/L%d.mat'%(output_folder, i) else: model_file = '%s/L%d_p%g_s%g.mat'%(output_folder, i, p_list[i], sigma_list[i]) sae.dA_layers[i].save_model_mat(model_file) # train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train # return p = p_dict['p'] sigma = sigma_dict['sigma'] logger.log('fine tuning...') if sae.finetune_train_flag == True: logger.log('SAE trained') return logger.log('learning rate:%g, p:%g, sigma:%g' % (finetune_lr, p, sigma)) train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train for epoch in xrange(1, fine_epochs + 1): epoch_time_s = strict_time() c = [] for minibatch_index in xrange(n_train_batches): err = train_fn(minibatch_index, finetune_lr) c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % down_epoch == 0: finetune_lr = 0.8 * finetune_lr logger.log('learning rate: %g' % (finetune_lr)) # if epoch in epoch_list: # sae.save_model('%s/SAE_p%g_s%g_(e_%g).mat'%(output_folder, p, sigma, epoch)) sae.save_model() logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
#!/usr/bin/env python3 import math import sys import unittest from test_tools_box import Colors from texttable import Texttable import logging import logger as lg logger = lg.mylogger(__name__) class ScaraArm: STRAIGHT_ANGLE = 180 MINIMAL_X_COORDINATE = 0.0001 def __init__(self, arm_r1=0, arm_r2=0): self.length_of_arm1 = arm_r1 self.length_of_arm2 = arm_r2 self.current_angle_r1 = 0 self.current_angle_r2 = 0 self.current_x = None self.current_y = None logger.info("Scara arm initieted. Larm1:{} Larm2:{} Angle1:{} Angle2:{} currentX:{} currentY:{}".\ format(self.length_of_arm1, \ self.length_of_arm2, \ self.current_angle_r1, \ self.current_angle_r2, \ self.current_x, \ self.current_x))
def auto_encoder_Lx(train_set=None, p=0, sigma=1, param=None, training_epochs=1000): #=========================================================================== # train_set: the training data # p: penalty; sigma: sigma for Gaussian # N: number of frame; # item_str: string for model and log file # training_epochs: training epochs #=========================================================================== batch_size = 100 output_folder = param['output_folder'] item_str = param['item_str'] pretrain_lr = param['pretrain_lr'] down_epoch = param['down_epoch'] n_hidden = param['n_hidden'] # compute number of minibatches for training, validation and testing n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size n_visible = train_set.get_value(borrow=True).shape[1] # start-snippet-2 # allocate symbolic variables for the data index = T.lscalar('index') # index to a [mini]batch learning_rate = T.scalar('lr') # learning rate to use x = T.matrix('x') # the data is presented as rasterized images # end-snippet-2 if not os.path.isdir(output_folder): os.makedirs(output_folder) # pickle_lst = [1, 5, 10, 50, 100, 500, 1000] rng = numpy.random.RandomState(123) da = dA(numpy_rng=rng, input=x, n_visible=n_visible, n_hidden=n_hidden) cost, updates = da.get_cost_updates_p(learning_rate=learning_rate, p=p, sigma=sigma) print '... building the model' train_fn = theano.function( inputs=[index, learning_rate], outputs=cost, updates=updates, givens={x: train_set[index * batch_size: (index + 1) * batch_size]} ) print '... training' start_time = strict_time() logger = mylogger(output_folder + '/' + item_str + '.log') logger.log('p:%g, sigma:%g, learning rate:%g' % (p, sigma, pretrain_lr)) for epoch in xrange(1, training_epochs + 1): # go through trainng set c = [] epoch_time_s = strict_time() for batch_index in xrange(n_train_batches): err = train_fn(batch_index, pretrain_lr) # err = 0 c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % down_epoch == 0: pretrain_lr = 0.8 * pretrain_lr logger.log('learning rate: %g' % (pretrain_lr)) # if epoch in pickle_lst: # file_name = "%s_epoch_%d" % (item_str,epoch) # save_model_mat(da, file_name) # logger.info(file_name+'.mat saved') da.save_model_mat(output_folder + '/' + item_str + '.mat') # da.save_model_mat("%s_(%d)" %(item_str,training_epochs), output_folder) logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
def auto_encoder_finetune(datasets=None, p=0, sigma=1, param=None, training_epochs=1000): if datasets == None: datasets = load_TIMIT() train_set, valid_set, test_set = datasets def get_shared(x): return theano.shared(numpy.asarray(x, dtype=theano.config.floatX), borrow=False) layers_sizes = param['layers_sizes'] L1_file = param['L1_file'] L2_file = param['L2_file'] L3_file = param['L3_file'] L4_file = param['L4_file'] output_folder = param['output_folder'] item_str = param['item_str'] valid_flag = 0 finetune_lr = 0.1 batch_size = 100 # compute number of minibatches for training, validation and testing n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size # allocate symbolic variables for the data index = T.lscalar('index') learning_rate = T.scalar('lr') x = T.matrix('x') if not os.path.isdir(output_folder): os.makedirs(output_folder) # pickle_lst = [100,200,300,400,500,600,700,800,900,1000] L1_w, L1_b, L1_b_prime = load_model_mat(file_name=L1_file, shared=0) L2_w, L2_b, L2_b_prime = load_model_mat(file_name=L2_file, shared=0) L3_w, L3_b, L3_b_prime = load_model_mat(file_name=L3_file, shared=0) L4_w, L4_b, L4_b_prime = load_model_mat(file_name=L4_file, shared=0) w_list = [get_shared(L1_w), get_shared(L2_w), get_shared(L3_w), get_shared(L4_w), get_shared(L4_w.T), get_shared(L3_w.T), get_shared(L2_w.T), get_shared(L1_w.T)] b_list = [get_shared(L1_b), get_shared(L2_b), get_shared(L3_b), get_shared(L4_b), get_shared(L4_b_prime), get_shared(L3_b_prime), get_shared(L2_b_prime), get_shared(L1_b_prime)] rng = numpy.random.RandomState(123) sae = SAE(numpy_rng=rng, input=x, layers_sizes=layers_sizes, w_list=w_list, b_list=b_list) print '... building the model' train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size, learning_rate, p, sigma) print '... training' logger = mylogger(output_folder + '/' + item_str + '.log') logger.log('p:%g, sigma:%g, learning rate:%g' % (p, sigma, finetune_lr)) #=========================================================================== # start training #=========================================================================== patience = 100 * n_train_batches # look as this many examples regardless patience_increase = 2. # wait this much longer when a new best is found improvement_threshold = 0.999 # a relative improvement of this much is considered significant validation_frequency = min(n_train_batches, patience / 2) best_validation_loss = numpy.inf test_score = 0. start_time = strict_time() done_looping = False epoch = 0;best_epoch = 0 while (epoch < training_epochs): epoch = epoch + 1 epoch_time_s = strict_time() c = [] for minibatch_index in xrange(n_train_batches): err = train_fn(minibatch_index,finetune_lr) # err = 0 c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % 100 == 0: finetune_lr = 0.8 * finetune_lr logger.log('learning rate: %g' % (finetune_lr)) if valid_flag == 0: continue validation_losses = numpy.mean(valid_model()) logger.log('valid %.5f' % (validation_losses)) if validation_losses < best_validation_loss: best_validation_loss = validation_losses best_epoch = epoch # test it on the test set test_losses = numpy.mean(test_model()) logger.log('test %.5f' % (test_losses)) sae.save_model_mat(output_folder + '/' + item_str + '.mat') # logger.log('best validation %.5f, test error %.5f, on epoch %d'%(best_validation_loss, test_losses, best_epoch)) sae.save_model_mat(output_folder + '/' + item_str + '.mat') logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.)) return for epoch in xrange(1, training_epochs + 1): # go through trainng set c = [] epoch_time_s = strict_time() for batch_index in xrange(n_train_batches): err = train_fn(batch_index, finetune_lr) # err = 0 c.append(err) logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % 100 == 0: finetune_lr = 0.8 * finetune_lr logger.log('learning rate: %g' % (finetune_lr)) sae.save_model_mat(output_folder + '/' + item_str + '.mat') logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
def __init__(self, numpy_rng, theano_rng=None, layers_sizes=[129, 500, 54, 500, 129], output_folder='out', p_dict=None, sigma_dict=None, file_path = '', logger=None): self.sigmoid_layers = [] self.params = [] self.finetune_train_flag = False self.n_layers = len(layers_sizes) self.n_layers_rmb = int(len(layers_sizes)/2) self.numpy_rng = numpy_rng self.output_folder = output_folder self.x = T.matrix(name='x') if not theano_rng: theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) if logger == None: self.logger = mylogger(output_folder + '/log.log') else: self.logger = logger assert self.n_layers > 0 if p_dict==None: self.p_list = [0]*self.n_layers_rmb self.sigma_list = [0]*self.n_layers_rmb self.p = 0 self.sigma = 0 elif p_dict!=None and sigma_dict==None: assert len(p_dict['p_list']) == self.n_layers_rmb self.p_list = p_dict['p_list'] self.sigma_list = [0]*self.n_layers_rmb self.p = p_dict['p'] self.sigma = 0 elif p_dict!=None and sigma_dict!=None: assert len(p_dict['p_list']) == self.n_layers_rmb assert len(sigma_dict['sigma_list']) == len(p_dict['p_list']) self.p_list = p_dict['p_list'] self.sigma_list = sigma_dict['sigma_list'] self.p = p_dict['p'] self.sigma = sigma_dict['sigma'] w = [0]*self.n_layers_rmb; hbias = [0]*self.n_layers_rmb; vbias = [0]*self.n_layers_rmb; # set_trace() w[0], hbias[0], vbias[0] = self.load_hinton_rbm(file_path+'/mnistvh.mat', 'vishid', 'hidrecbiases', 'visbiases') #rbm1 # w[1], hbias[1], vbias[1] = self.load_hinton_rbm(file_path+'/mnisthp.mat', 'hidpen', 'penrecbiases', 'hidgenbiases') #rbm2 # w[2], hbias[2], vbias[2] = self.load_hinton_rbm(file_path+'/mnisthp2.mat', 'hidpen2', 'penrecbiases2', 'hidgenbiases2') #rbm3 # w[3], hbias[3], vbias[3] = self.load_hinton_rbm(file_path+'/mnistpo.mat', 'hidtop', 'toprecbiases', 'topgenbiases') #rbm4 sigmoid_layer1 = HiddenLayer(rng=self.numpy_rng, input=self.x, n_in=w[0].get_value(borrow=True).shape[0], n_out=w[0].get_value(borrow=True).shape[1], W=w[0], b=hbias[0], activation=T.nnet.sigmoid) # sigmoid_layer2 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer1.output, # n_in=w[1].get_value(borrow=True).shape[0], n_out=w[1].get_value(borrow=True).shape[1], # W=w[1], b=hbias[1], activation=T.nnet.sigmoid) # sigmoid_layer3 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer2.output, # n_in=w[1].get_value(borrow=True).shape[1], n_out=w[1].get_value(borrow=True).shape[0], # W=theano.shared(numpy.asarray(w[1].T.eval(), dtype=theano.config.floatX), borrow=True), # b=vbias[1], activation=T.nnet.sigmoid) #sigmoid layer, inverse sigmoid_layer2 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer1.output, n_in=w[0].get_value(borrow=True).shape[1], n_out=w[0].get_value(borrow=True).shape[0], W=theano.shared(numpy.asarray(w[0].T.eval(), dtype=theano.config.floatX), borrow=True), b=vbias[0], activation=None) #output layer, inverse # m_idx = self.n_layers_rmb-1; # for i in xrange(m_idx): # if i == 0: # layer_input = self.x # else: # layer_input = self.sigmoid_layers[-1].output # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=layer_input, # n_in=w[i].get_value(borrow=True).shape[0], n_out=w[i].get_value(borrow=True).shape[1], # W=w[i], b=hbias[i], activation=T.nnet.sigmoid) #sigmoid layer # self.sigmoid_layers.append(sigmoid_layer) # # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output, # n_in=w[m_idx].get_value(borrow=True).shape[0], n_out=w[m_idx].get_value(borrow=True).shape[1], # W=w[m_idx], b=hbias[m_idx], activation=T.nnet.sigmoid) #coding layer # self.sigmoid_layers.append(sigmoid_layer) # # for i in xrange(m_idx,0,-1): # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output, # n_in=w[i].get_value(borrow=True).shape[1], n_out=w[i].get_value(borrow=True).shape[0], # W=theano.shared(numpy.asarray(w[i].T.eval(), dtype=theano.config.floatX), borrow=True), # b=vbias[i], activation=T.nnet.sigmoid) #sigmoid layer, inverse # self.sigmoid_layers.append(sigmoid_layer) # # sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output, # n_in=w[0].get_value(borrow=True).shape[1], n_out=w[0].get_value(borrow=True).shape[0], # W=theano.shared(numpy.asarray(w[0].T.eval(), dtype=theano.config.floatX), borrow=True), # b=vbias[0], activation=None) #output layer, inverse # self.sigmoid_layers.append(sigmoid_layer) self.sigmoid_layers.append(sigmoid_layer1) self.sigmoid_layers.append(sigmoid_layer2) # self.sigmoid_layers.append(sigmoid_layer3) # self.sigmoid_layers.append(sigmoid_layer4) for sigmoid_layer in self.sigmoid_layers: self.params.extend(sigmoid_layer.params) set_trace() self.save_model_mat('%s/DAE_pre%s.mat'%(self.output_folder, self.get_model_file()))
def __init__(self, numpy_rng, theano_rng=None, layers_sizes=[129, 500, 54, 500, 129], output_folder='out', p_dict=None, sigma_dict=None, logger=None): self.rbm_layers = [] self.rbm_train_flags = [] self.finetune_train_flag = False self.n_layers = len(layers_sizes) self.n_layers_rmb = int(len(layers_sizes)/2) self.numpy_rng = numpy_rng self.output_folder = output_folder if logger == None: self.logger = mylogger(output_folder + '/log.log') else: self.logger = logger assert self.n_layers > 0 if p_dict==None: self.p_list = [0]*self.n_layers_rmb self.sigma_list = [0]*self.n_layers_rmb self.p = 0 self.sigma = 0 elif p_dict!=None and sigma_dict==None: assert len(p_dict['p_list']) == self.n_layers_rmb self.p_list = p_dict['p_list'] self.sigma_list = [0]*self.n_layers_rmb self.p = p_dict['p'] self.sigma = 0 elif p_dict!=None and sigma_dict!=None: assert len(p_dict['p_list']) == self.n_layers_rmb assert len(sigma_dict['sigma_list']) == len(p_dict['p_list']) self.p_list = p_dict['p_list'] self.sigma_list = sigma_dict['sigma_list'] self.p = p_dict['p'] self.sigma = sigma_dict['sigma'] if not theano_rng: theano_rng = RandomStreams(numpy_rng.randint(2 ** 30)) self.x = T.matrix(name='x') for i in xrange(self.n_layers_rmb): # if i == (self.n_layers_rmb - 1): # linear_flag = True # else: # linear_flag = False linear_flag = False self.log('layer: %d, linear_flag: %d'%(i, linear_flag)) if i == 0: layer_input = self.x else: layer_input = self.rbm_layers[-1].get_active() if(self.p_list[i] == 0): model_file = '%s/L%d.mat'%(output_folder, i) else: model_file = '%s/L%d_p%g_s%g.mat'%(output_folder, i, self.p_list[i], self.sigma_list[i]) if os.path.isfile(model_file): #this layer has been trained w, hbias, vbias = self.load_RMB_model(i) rbm_layer = RBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1], W=w, hbias=hbias, vbias=vbias, linear_flag=linear_flag) self.rbm_train_flags.append(True) #set the flag else: rbm_layer = RBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1], linear_flag=linear_flag) self.rbm_train_flags.append(False) #set the flag self.rbm_layers.append(rbm_layer) finetune_file = '%s/DAE%s.mat'%(output_folder, self.get_model_file()) if os.path.isfile(finetune_file): #trained self.finetune_train_flag = True
def train_DAE(datasets, layers_sizes, output_folder, p, sigma): train_set, valid_set, test_set = datasets if not os.path.isdir(output_folder): os.makedirs(output_folder) pre_epochs = 10 fine_epochs = 500 finetune_lr = 0.01 batch_size = 100 down_epoch = 50 save_epoch = 10 MAX_TRAIN_SET = 100000 train_set_N = train_set.get_value(borrow=True).shape[0] n_train_batches = train_set_N / batch_size if train_set_N < MAX_TRAIN_SET else MAX_TRAIN_SET / batch_size logger = mylogger('%s/log_p%g_s%g.log' % (output_folder, p, sigma)) logger.log('train_set size: %d'%(n_train_batches*batch_size)) logger.log('layer size: ' + ''.join(['%d ' % i for i in layers_sizes]).strip()) rng = numpy.random.RandomState(123) dae = deep_auto_encoder(numpy_rng=rng, layers_sizes=layers_sizes, output_folder=output_folder, p=p, sigma=sigma, logger=logger) pretraining_fns = dae.pretraining_functions(train_set, batch_size) start_time = strict_time() logger.log('pre_training...') for i in xrange(len(dae.rbm_layers)): if dae.rbm_train_flags[i] == True: logger.log('L%d trained' % (i)) continue pretrain_lr = 0.0001 logger.log('training L%d ...' % (i)) logger.log('learning rate:%g' % (pretrain_lr)) for epoch in xrange(1, pre_epochs + 1): epoch_time_s = strict_time() r_c, fe_c = [], [] for batch_index in xrange(n_train_batches): [reconstruction_cost, free_energy_cost] = pretraining_fns[i](index=batch_index, lr=pretrain_lr) r_c.append(reconstruction_cost) fe_c.append(free_energy_cost) logger.log('epoch %d, r_cost %.5f, fe_c %.5f, took %f seconds ' % (epoch, numpy.mean(r_c), numpy.mean(fe_c), (strict_time() - epoch_time_s))) dae.save_RBM_model(i) # train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train logger.log('fine tuning...') if dae.finetune_train_flag == True: logger.log('DAE trained') return logger.log('learning rate:%g, p:%g, sigma:%g' % (finetune_lr, p, sigma)) train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size, 'DAE_p-1_s0.3.mat') # must be here! after pre_train # train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size) # must be here! after pre_train for epoch in xrange(1, fine_epochs + 1): epoch_time_s = strict_time() c = [] for minibatch_index in xrange(n_train_batches): err = train_fn(minibatch_index, finetune_lr) c.append(err) # set_trace() logger.log('epoch %d, cost %.5f, took %.2f seconds' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s))) if epoch % down_epoch == 0: finetune_lr = 0.9 * finetune_lr logger.log('learning rate:%g' % (finetune_lr)) if epoch % save_epoch == 0: dae.save_model() dae.save_model() logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))