def train_gb_rbm(batch_size=100,epochs=50):
output_folder = 'gb_rbm'
data_file = 'rbm_TIMIT_dr2_(N1)_split.mat'
datasets = load_TIMIT(data_file)
train_set, valid_set, test_set = datasets
numpy_rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
input_dim = train_set.get_value(borrow=True).shape[1]
layers_sizes = [input_dim,70,input_dim]
input_x = T.matrix(name='x')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
logger = mylogger(output_folder + '/log.log')
gb_rbm_layer = GBRBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=input_x,
n_visible=layers_sizes[0],
n_hidden=layers_sizes[1])
index = T.lscalar('index')
momentum = T.scalar('momentum')
learning_rate = T.scalar('lr')
# number of mini-batches
n_batches = train_set.get_value(borrow=True).shape[0] / batch_size
# start and end index of this mini-batch
batch_begin = index * batch_size
batch_end = batch_begin + batch_size
r_cost, fe_cost, updates = gb_rbm_layer.get_cost_updates(batch_size, learning_rate,
momentum, weight_cost=0.0002,
persistent=None, k = 1)
# compile the theano function
fn = theano.function(inputs=[index,
theano.Param(learning_rate, default=0.0001),
theano.Param(momentum, default=0.5)],
outputs= [r_cost, fe_cost],
updates=updates,
givens={input_x: train_set[batch_begin:batch_end]})
r_c, fe_c = [], [] # keep record of reconstruction and free-energy cost
for epoch in range(epochs):
for batch_index in xrange(n_batches): # loop over mini-batches
[reconstruction_cost, free_energy_cost] = fn(index=batch_index)
r_c.append(reconstruction_cost)
fe_c.append(free_energy_cost)
logger.log('pre-training, epoch %d, r_cost %f, fe_cost %f' % (epoch, numpy.mean(r_c), numpy.mean(fe_c)))
params = []
for item in gb_rbm_layer.params:
params.append(item.get_value(borrow=True))
savemat(output_folder+'/gb_rbm.mat', {'params':params})
def __init__(self,
numpy_rng,
theano_rng=None,
layers_sizes=[129, 500, 54, 500, 129],
output_folder='out',
p=0,
sigma=0,
logger=None):
self.rbm_layers = []
self.rbm_train_flags = []
self.finetune_train_flag = False
self.n_layers = len(layers_sizes)
self.n_layers_rmb = int(len(layers_sizes) / 2)
self.numpy_rng = numpy_rng
self.output_folder = output_folder
if logger == None:
self.logger = mylogger(output_folder + '/log.log')
else:
self.logger = logger
assert self.n_layers > 0
self.p = p
self.sigma = sigma
if not theano_rng:
self.theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
else:
self.theano_rng = theano_rng
self.x = T.matrix(name='x')
for i in xrange(self.n_layers_rmb):
if i == 0:
layer_input = self.x
else:
layer_input = self.rbm_layers[-1].get_active()
model_file = '%s/L%d.mat' % (output_folder, i)
if os.path.isfile(model_file): # this layer has been trained
w, hbias, vbias = self.load_RMB_model(i)
rbm_layer = GBRBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i + 1],
W=w, hbias=hbias, vbias=vbias)
self.rbm_train_flags.append(True) # set the flag
else:
rbm_layer = GBRBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i + 1])
self.rbm_train_flags.append(False) # set the flag
self.rbm_layers.append(rbm_layer)
finetune_file = '%s/DAE%s.mat' % (output_folder, self.get_model_file())
if os.path.isfile(finetune_file): # trained
self.finetune_train_flag = True
def train_autoencoder(datasets,layers_sizes,output_folder,p_dict,sigma_dict):
train_set, valid_set, test_set = datasets
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
pre_epochs=20
fine_epochs=100
finetune_lr = 0.001
batch_size = 100
down_epoch = 100
n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size
logger = mylogger(output_folder + '/log.log')
logger.log('layer size: '+''.join(['%d '%i for i in layers_sizes]).strip())
file_path = '../../Autoencoder_hinton'
rng = numpy.random.RandomState(123)
dae = autoencoder_hinton(numpy_rng=rng, layers_sizes=layers_sizes, output_folder=output_folder, logger=logger, file_path=file_path)
start_time = strict_time()
p = p_dict['p']
sigma = sigma_dict['sigma']
logger.log('fine tuning...')
if dae.finetune_train_flag == True:
logger.log('SAE trained')
return
logger.log('learning rate:%g, p:%g, sigma:%g' % (finetune_lr, p, sigma))
train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train
for epoch in xrange(1, fine_epochs + 1):
epoch_time_s = strict_time()
c = []
for minibatch_index in xrange(n_train_batches):
err = train_fn(minibatch_index, finetune_lr)
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % down_epoch == 0:
finetune_lr = 0.8 * finetune_lr
logger.log('learning rate: %g' % (finetune_lr))
# if epoch in epoch_list:
# dae.save_model('%s/SAE_p%g_s%g_(e_%g).mat'%(output_folder, p, sigma, epoch))
dae.save_model()
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
def train_SAE(datasets,layers_sizes,output_folder,p_dict,sigma_dict):
train_set, valid_set, test_set = datasets
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
pre_epochs=200
fine_epochs=200
epoch_list = [1, 10]
finetune_lr = 0.1
batch_size = 100
down_epoch = 100
n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size
# logger = mylogger(output_folder + '/log_(L-1_sx).log')
logger = mylogger(output_folder + '/log.log')
logger.log('layer size: '+''.join(['%d '%i for i in layers_sizes]).strip())
p_list=p_dict['p_list']
sigma_list=sigma_dict['sigma_list']
rng = numpy.random.RandomState(123)
sae = SAE(numpy_rng=rng, layers_sizes=layers_sizes, output_folder=output_folder, p_dict=p_dict, sigma_dict=sigma_dict)
pretraining_fns = sae.pretraining_functions(train_set, batch_size)
start_time = strict_time()
logger.log('pre_training...')
for i in xrange(len(sae.dA_layers)):
if sae.dA_train_flags[i] == True:
logger.log('L%d trained'%(i))
continue
pretrain_lr = 0.05
logger.log('training L%d ...'%(i))
logger.log('learning rate:%g, p:%g, sigma:%g' % (pretrain_lr, p_list[i], sigma_list[i]))
for epoch in xrange(1, pre_epochs + 1):
epoch_time_s = strict_time()
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index, lr=pretrain_lr))
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % down_epoch == 0:
pretrain_lr = 0.8 * pretrain_lr
logger.log('learning rate: %g' % (pretrain_lr))
if(p_list[i] == 0):
model_file = '%s/L%d.mat'%(output_folder, i)
else:
model_file = '%s/L%d_p%g_s%g.mat'%(output_folder, i, p_list[i], sigma_list[i])
sae.dA_layers[i].save_model_mat(model_file)
# train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train
# return
p = p_dict['p']
sigma = sigma_dict['sigma']
logger.log('fine tuning...')
if sae.finetune_train_flag == True:
logger.log('SAE trained')
return
logger.log('learning rate:%g, p:%g, sigma:%g' % (finetune_lr, p, sigma))
train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train
for epoch in xrange(1, fine_epochs + 1):
epoch_time_s = strict_time()
c = []
for minibatch_index in xrange(n_train_batches):
err = train_fn(minibatch_index, finetune_lr)
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % down_epoch == 0:
finetune_lr = 0.8 * finetune_lr
logger.log('learning rate: %g' % (finetune_lr))
# if epoch in epoch_list:
# sae.save_model('%s/SAE_p%g_s%g_(e_%g).mat'%(output_folder, p, sigma, epoch))
sae.save_model()
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
#!/usr/bin/env python3
import math
import sys
import unittest
from test_tools_box import Colors
from texttable import Texttable
import logging
import logger as lg
logger = lg.mylogger(__name__)
class ScaraArm:
STRAIGHT_ANGLE = 180
MINIMAL_X_COORDINATE = 0.0001
def __init__(self, arm_r1=0, arm_r2=0):
self.length_of_arm1 = arm_r1
self.length_of_arm2 = arm_r2
self.current_angle_r1 = 0
self.current_angle_r2 = 0
self.current_x = None
self.current_y = None
logger.info("Scara arm initieted. Larm1:{} Larm2:{} Angle1:{} Angle2:{} currentX:{} currentY:{}".\
format(self.length_of_arm1, \
self.length_of_arm2, \
self.current_angle_r1, \
self.current_angle_r2, \
self.current_x, \
self.current_x))
def auto_encoder_Lx(train_set=None, p=0, sigma=1, param=None, training_epochs=1000):
#===========================================================================
# train_set: the training data
# p: penalty; sigma: sigma for Gaussian
# N: number of frame;
# item_str: string for model and log file
# training_epochs: training epochs
#===========================================================================
batch_size = 100
output_folder = param['output_folder']
item_str = param['item_str']
pretrain_lr = param['pretrain_lr']
down_epoch = param['down_epoch']
n_hidden = param['n_hidden']
# compute number of minibatches for training, validation and testing
n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size
n_visible = train_set.get_value(borrow=True).shape[1]
# start-snippet-2
# allocate symbolic variables for the data
index = T.lscalar('index') # index to a [mini]batch
learning_rate = T.scalar('lr') # learning rate to use
x = T.matrix('x') # the data is presented as rasterized images
# end-snippet-2
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# pickle_lst = [1, 5, 10, 50, 100, 500, 1000]
rng = numpy.random.RandomState(123)
da = dA(numpy_rng=rng, input=x, n_visible=n_visible, n_hidden=n_hidden)
cost, updates = da.get_cost_updates_p(learning_rate=learning_rate, p=p, sigma=sigma)
print '... building the model'
train_fn = theano.function(
inputs=[index,
learning_rate],
outputs=cost,
updates=updates,
givens={x: train_set[index * batch_size: (index + 1) * batch_size]}
)
print '... training'
start_time = strict_time()
logger = mylogger(output_folder + '/' + item_str + '.log')
logger.log('p:%g, sigma:%g, learning rate:%g' % (p, sigma, pretrain_lr))
for epoch in xrange(1, training_epochs + 1):
# go through trainng set
c = []
epoch_time_s = strict_time()
for batch_index in xrange(n_train_batches):
err = train_fn(batch_index, pretrain_lr)
# err = 0
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % down_epoch == 0:
pretrain_lr = 0.8 * pretrain_lr
logger.log('learning rate: %g' % (pretrain_lr))
# if epoch in pickle_lst:
# file_name = "%s_epoch_%d" % (item_str,epoch)
# save_model_mat(da, file_name)
# logger.info(file_name+'.mat saved')
da.save_model_mat(output_folder + '/' + item_str + '.mat')
# da.save_model_mat("%s_(%d)" %(item_str,training_epochs), output_folder)
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
def auto_encoder_finetune(datasets=None, p=0, sigma=1, param=None, training_epochs=1000):
if datasets == None:
datasets = load_TIMIT()
train_set, valid_set, test_set = datasets
def get_shared(x):
return theano.shared(numpy.asarray(x, dtype=theano.config.floatX), borrow=False)
layers_sizes = param['layers_sizes']
L1_file = param['L1_file']
L2_file = param['L2_file']
L3_file = param['L3_file']
L4_file = param['L4_file']
output_folder = param['output_folder']
item_str = param['item_str']
valid_flag = 0
finetune_lr = 0.1
batch_size = 100
# compute number of minibatches for training, validation and testing
n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size
# allocate symbolic variables for the data
index = T.lscalar('index')
learning_rate = T.scalar('lr')
x = T.matrix('x')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# pickle_lst = [100,200,300,400,500,600,700,800,900,1000]
L1_w, L1_b, L1_b_prime = load_model_mat(file_name=L1_file, shared=0)
L2_w, L2_b, L2_b_prime = load_model_mat(file_name=L2_file, shared=0)
L3_w, L3_b, L3_b_prime = load_model_mat(file_name=L3_file, shared=0)
L4_w, L4_b, L4_b_prime = load_model_mat(file_name=L4_file, shared=0)
w_list = [get_shared(L1_w), get_shared(L2_w), get_shared(L3_w), get_shared(L4_w),
get_shared(L4_w.T), get_shared(L3_w.T), get_shared(L2_w.T), get_shared(L1_w.T)]
b_list = [get_shared(L1_b), get_shared(L2_b), get_shared(L3_b), get_shared(L4_b),
get_shared(L4_b_prime), get_shared(L3_b_prime), get_shared(L2_b_prime), get_shared(L1_b_prime)]
rng = numpy.random.RandomState(123)
sae = SAE(numpy_rng=rng,
input=x,
layers_sizes=layers_sizes,
w_list=w_list,
b_list=b_list)
print '... building the model'
train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size, learning_rate, p, sigma)
print '... training'
logger = mylogger(output_folder + '/' + item_str + '.log')
logger.log('p:%g, sigma:%g, learning rate:%g' % (p, sigma, finetune_lr))
#===========================================================================
# start training
#===========================================================================
patience = 100 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is found
improvement_threshold = 0.999 # a relative improvement of this much is considered significant
validation_frequency = min(n_train_batches, patience / 2)
best_validation_loss = numpy.inf
test_score = 0.
start_time = strict_time()
done_looping = False
epoch = 0;best_epoch = 0
while (epoch < training_epochs):
epoch = epoch + 1
epoch_time_s = strict_time()
c = []
for minibatch_index in xrange(n_train_batches):
err = train_fn(minibatch_index,finetune_lr)
# err = 0
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % 100 == 0:
finetune_lr = 0.8 * finetune_lr
logger.log('learning rate: %g' % (finetune_lr))
if valid_flag == 0:
continue
validation_losses = numpy.mean(valid_model())
logger.log('valid %.5f' % (validation_losses))
if validation_losses < best_validation_loss:
best_validation_loss = validation_losses
best_epoch = epoch
# test it on the test set
test_losses = numpy.mean(test_model())
logger.log('test %.5f' % (test_losses))
sae.save_model_mat(output_folder + '/' + item_str + '.mat')
# logger.log('best validation %.5f, test error %.5f, on epoch %d'%(best_validation_loss, test_losses, best_epoch))
sae.save_model_mat(output_folder + '/' + item_str + '.mat')
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
return
for epoch in xrange(1, training_epochs + 1):
# go through trainng set
c = []
epoch_time_s = strict_time()
for batch_index in xrange(n_train_batches):
err = train_fn(batch_index, finetune_lr)
# err = 0
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % 100 == 0:
finetune_lr = 0.8 * finetune_lr
logger.log('learning rate: %g' % (finetune_lr))
sae.save_model_mat(output_folder + '/' + item_str + '.mat')
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
def __init__(self,
numpy_rng,
theano_rng=None,
layers_sizes=[129, 500, 54, 500, 129],
output_folder='out',
p_dict=None,
sigma_dict=None,
file_path = '',
logger=None):
self.sigmoid_layers = []
self.params = []
self.finetune_train_flag = False
self.n_layers = len(layers_sizes)
self.n_layers_rmb = int(len(layers_sizes)/2)
self.numpy_rng = numpy_rng
self.output_folder = output_folder
self.x = T.matrix(name='x')
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
if logger == None:
self.logger = mylogger(output_folder + '/log.log')
else:
self.logger = logger
assert self.n_layers > 0
if p_dict==None:
self.p_list = [0]*self.n_layers_rmb
self.sigma_list = [0]*self.n_layers_rmb
self.p = 0
self.sigma = 0
elif p_dict!=None and sigma_dict==None:
assert len(p_dict['p_list']) == self.n_layers_rmb
self.p_list = p_dict['p_list']
self.sigma_list = [0]*self.n_layers_rmb
self.p = p_dict['p']
self.sigma = 0
elif p_dict!=None and sigma_dict!=None:
assert len(p_dict['p_list']) == self.n_layers_rmb
assert len(sigma_dict['sigma_list']) == len(p_dict['p_list'])
self.p_list = p_dict['p_list']
self.sigma_list = sigma_dict['sigma_list']
self.p = p_dict['p']
self.sigma = sigma_dict['sigma']
w = [0]*self.n_layers_rmb; hbias = [0]*self.n_layers_rmb; vbias = [0]*self.n_layers_rmb;
# set_trace()
w[0], hbias[0], vbias[0] = self.load_hinton_rbm(file_path+'/mnistvh.mat', 'vishid', 'hidrecbiases', 'visbiases') #rbm1
# w[1], hbias[1], vbias[1] = self.load_hinton_rbm(file_path+'/mnisthp.mat', 'hidpen', 'penrecbiases', 'hidgenbiases') #rbm2
# w[2], hbias[2], vbias[2] = self.load_hinton_rbm(file_path+'/mnisthp2.mat', 'hidpen2', 'penrecbiases2', 'hidgenbiases2') #rbm3
# w[3], hbias[3], vbias[3] = self.load_hinton_rbm(file_path+'/mnistpo.mat', 'hidtop', 'toprecbiases', 'topgenbiases') #rbm4
sigmoid_layer1 = HiddenLayer(rng=self.numpy_rng, input=self.x,
n_in=w[0].get_value(borrow=True).shape[0], n_out=w[0].get_value(borrow=True).shape[1],
W=w[0], b=hbias[0], activation=T.nnet.sigmoid)
# sigmoid_layer2 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer1.output,
# n_in=w[1].get_value(borrow=True).shape[0], n_out=w[1].get_value(borrow=True).shape[1],
# W=w[1], b=hbias[1], activation=T.nnet.sigmoid)
# sigmoid_layer3 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer2.output,
# n_in=w[1].get_value(borrow=True).shape[1], n_out=w[1].get_value(borrow=True).shape[0],
# W=theano.shared(numpy.asarray(w[1].T.eval(), dtype=theano.config.floatX), borrow=True),
# b=vbias[1], activation=T.nnet.sigmoid) #sigmoid layer, inverse
sigmoid_layer2 = HiddenLayer(rng=self.numpy_rng, input=sigmoid_layer1.output,
n_in=w[0].get_value(borrow=True).shape[1], n_out=w[0].get_value(borrow=True).shape[0],
W=theano.shared(numpy.asarray(w[0].T.eval(), dtype=theano.config.floatX), borrow=True),
b=vbias[0], activation=None) #output layer, inverse
# m_idx = self.n_layers_rmb-1;
# for i in xrange(m_idx):
# if i == 0:
# layer_input = self.x
# else:
# layer_input = self.sigmoid_layers[-1].output
# sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=layer_input,
# n_in=w[i].get_value(borrow=True).shape[0], n_out=w[i].get_value(borrow=True).shape[1],
# W=w[i], b=hbias[i], activation=T.nnet.sigmoid) #sigmoid layer
# self.sigmoid_layers.append(sigmoid_layer)
#
# sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output,
# n_in=w[m_idx].get_value(borrow=True).shape[0], n_out=w[m_idx].get_value(borrow=True).shape[1],
# W=w[m_idx], b=hbias[m_idx], activation=T.nnet.sigmoid) #coding layer
# self.sigmoid_layers.append(sigmoid_layer)
#
# for i in xrange(m_idx,0,-1):
# sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output,
# n_in=w[i].get_value(borrow=True).shape[1], n_out=w[i].get_value(borrow=True).shape[0],
# W=theano.shared(numpy.asarray(w[i].T.eval(), dtype=theano.config.floatX), borrow=True),
# b=vbias[i], activation=T.nnet.sigmoid) #sigmoid layer, inverse
# self.sigmoid_layers.append(sigmoid_layer)
#
# sigmoid_layer = HiddenLayer(rng=self.numpy_rng, input=self.sigmoid_layers[-1].output,
# n_in=w[0].get_value(borrow=True).shape[1], n_out=w[0].get_value(borrow=True).shape[0],
# W=theano.shared(numpy.asarray(w[0].T.eval(), dtype=theano.config.floatX), borrow=True),
# b=vbias[0], activation=None) #output layer, inverse
# self.sigmoid_layers.append(sigmoid_layer)
self.sigmoid_layers.append(sigmoid_layer1)
self.sigmoid_layers.append(sigmoid_layer2)
# self.sigmoid_layers.append(sigmoid_layer3)
# self.sigmoid_layers.append(sigmoid_layer4)
for sigmoid_layer in self.sigmoid_layers:
self.params.extend(sigmoid_layer.params)
set_trace()
self.save_model_mat('%s/DAE_pre%s.mat'%(self.output_folder, self.get_model_file()))
def __init__(self,
numpy_rng,
theano_rng=None,
layers_sizes=[129, 500, 54, 500, 129],
output_folder='out',
p_dict=None,
sigma_dict=None,
logger=None):
self.rbm_layers = []
self.rbm_train_flags = []
self.finetune_train_flag = False
self.n_layers = len(layers_sizes)
self.n_layers_rmb = int(len(layers_sizes)/2)
self.numpy_rng = numpy_rng
self.output_folder = output_folder
if logger == None:
self.logger = mylogger(output_folder + '/log.log')
else:
self.logger = logger
assert self.n_layers > 0
if p_dict==None:
self.p_list = [0]*self.n_layers_rmb
self.sigma_list = [0]*self.n_layers_rmb
self.p = 0
self.sigma = 0
elif p_dict!=None and sigma_dict==None:
assert len(p_dict['p_list']) == self.n_layers_rmb
self.p_list = p_dict['p_list']
self.sigma_list = [0]*self.n_layers_rmb
self.p = p_dict['p']
self.sigma = 0
elif p_dict!=None and sigma_dict!=None:
assert len(p_dict['p_list']) == self.n_layers_rmb
assert len(sigma_dict['sigma_list']) == len(p_dict['p_list'])
self.p_list = p_dict['p_list']
self.sigma_list = sigma_dict['sigma_list']
self.p = p_dict['p']
self.sigma = sigma_dict['sigma']
if not theano_rng:
theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
self.x = T.matrix(name='x')
for i in xrange(self.n_layers_rmb):
# if i == (self.n_layers_rmb - 1):
# linear_flag = True
# else:
# linear_flag = False
linear_flag = False
self.log('layer: %d, linear_flag: %d'%(i, linear_flag))
if i == 0:
layer_input = self.x
else:
layer_input = self.rbm_layers[-1].get_active()
if(self.p_list[i] == 0):
model_file = '%s/L%d.mat'%(output_folder, i)
else:
model_file = '%s/L%d_p%g_s%g.mat'%(output_folder, i, self.p_list[i], self.sigma_list[i])
if os.path.isfile(model_file): #this layer has been trained
w, hbias, vbias = self.load_RMB_model(i)
rbm_layer = RBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1],
W=w, hbias=hbias, vbias=vbias, linear_flag=linear_flag)
self.rbm_train_flags.append(True) #set the flag
else:
rbm_layer = RBM(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1],
linear_flag=linear_flag)
self.rbm_train_flags.append(False) #set the flag
self.rbm_layers.append(rbm_layer)
finetune_file = '%s/DAE%s.mat'%(output_folder, self.get_model_file())
if os.path.isfile(finetune_file): #trained
self.finetune_train_flag = True
def train_DAE(datasets, layers_sizes, output_folder, p, sigma):
train_set, valid_set, test_set = datasets
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
pre_epochs = 10
fine_epochs = 500
finetune_lr = 0.01
batch_size = 100
down_epoch = 50
save_epoch = 10
MAX_TRAIN_SET = 100000
train_set_N = train_set.get_value(borrow=True).shape[0]
n_train_batches = train_set_N / batch_size if train_set_N < MAX_TRAIN_SET else MAX_TRAIN_SET / batch_size
logger = mylogger('%s/log_p%g_s%g.log' % (output_folder, p, sigma))
logger.log('train_set size: %d'%(n_train_batches*batch_size))
logger.log('layer size: ' + ''.join(['%d ' % i for i in layers_sizes]).strip())
rng = numpy.random.RandomState(123)
dae = deep_auto_encoder(numpy_rng=rng, layers_sizes=layers_sizes, output_folder=output_folder, p=p, sigma=sigma, logger=logger)
pretraining_fns = dae.pretraining_functions(train_set, batch_size)
start_time = strict_time()
logger.log('pre_training...')
for i in xrange(len(dae.rbm_layers)):
if dae.rbm_train_flags[i] == True:
logger.log('L%d trained' % (i))
continue
pretrain_lr = 0.0001
logger.log('training L%d ...' % (i))
logger.log('learning rate:%g' % (pretrain_lr))
for epoch in xrange(1, pre_epochs + 1):
epoch_time_s = strict_time()
r_c, fe_c = [], []
for batch_index in xrange(n_train_batches):
[reconstruction_cost, free_energy_cost] = pretraining_fns[i](index=batch_index, lr=pretrain_lr)
r_c.append(reconstruction_cost)
fe_c.append(free_energy_cost)
logger.log('epoch %d, r_cost %.5f, fe_c %.5f, took %f seconds ' % (epoch, numpy.mean(r_c), numpy.mean(fe_c), (strict_time() - epoch_time_s)))
dae.save_RBM_model(i)
# train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size)#must be here! after pre_train
logger.log('fine tuning...')
if dae.finetune_train_flag == True:
logger.log('DAE trained')
return
logger.log('learning rate:%g, p:%g, sigma:%g' % (finetune_lr, p, sigma))
train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size, 'DAE_p-1_s0.3.mat') # must be here! after pre_train
# train_fn, valid_model, test_model = dae.build_finetune_functions(datasets, batch_size) # must be here! after pre_train
for epoch in xrange(1, fine_epochs + 1):
epoch_time_s = strict_time()
c = []
for minibatch_index in xrange(n_train_batches):
err = train_fn(minibatch_index, finetune_lr)
c.append(err)
# set_trace()
logger.log('epoch %d, cost %.5f, took %.2f seconds' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % down_epoch == 0:
finetune_lr = 0.9 * finetune_lr
logger.log('learning rate:%g' % (finetune_lr))
if epoch % save_epoch == 0:
dae.save_model()
dae.save_model()
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
