def get_L1_out(train_set=None, model_file=None, N=1, shared=1):
if model_file == None:
model_file = 'L1_p0_s1'
output_folder = 'auto_encoder_out_N%s' %(N)
model_w, model_b, model_b_prime = load_model_mat(file_name=model_file, output_folder=output_folder, shared=0)
# set_trace()
hidden_value = 1 / (1 + numpy.exp(-(numpy.dot(train_set, model_w) + model_b)))
if shared == 1:
hidden_value = theano.shared(numpy.asarray(hidden_value, dtype=theano.config.floatX), borrow=True)
return hidden_value
def get_hidden(train_set=None, model_file=None, shared=1):
if model_file == None:
model_file = 'L1_p0_s1'
model_w, model_b, model_b_prime = load_model_mat(model_file)
n_visible,n_hidden = model_w.get_value(borrow=True).shape
rng = numpy.random.RandomState(123)
da = dA(numpy_rng=rng, input=train_set, n_visible=n_visible, n_hidden=n_hidden, W=model_w, bhid=model_b, bvis=model_b_prime)
# set_trace()
hidden_value = da.get_active().eval()
if shared == 1:
hidden_value = theano.shared(numpy.asarray(hidden_value, dtype=theano.config.floatX), borrow=True)
return hidden_value
def __init__(self,
numpy_rng,
theano_rng=None,
layers_sizes=[129, 500, 54, 500, 129],
w_list=None,
b_list=None,
output_folder='out',
p_dict=None,
sigma_dict=None):
self.dA_layers = []
self.dA_train_flags = []
self.finetune_train_flag = False
self.n_layers = len(layers_sizes)
self.n_layers_dA = int(len(layers_sizes)/2)
self.numpy_rng = numpy_rng
self.output_folder = output_folder
assert self.n_layers > 0
if p_dict==None:
self.p_list = [0]*self.n_layers_dA
self.sigma_list = [0]*self.n_layers_dA
self.p = 0
self.sigma = 0
elif p_dict!=None and sigma_dict==None:
assert len(p_dict['p_list']) == self.n_layers_dA
self.p_list = p_dict['p_list']
self.sigma_list = [0]*self.n_layers_dA
self.p = p_dict['p']
self.sigma = 0
elif p_dict!=None and sigma_dict!=None:
assert len(p_dict['p_list']) == self.n_layers_dA
assert len(sigma_dict['sigma_list']) == len(p_dict['p_list'])
self.p_list = p_dict['p_list']
self.sigma_list = sigma_dict['sigma_list']
self.p = p_dict['p']
self.sigma = sigma_dict['sigma']
if not theano_rng:
self.theano_rng = RandomStreams(numpy_rng.randint(2 ** 30))
self.x = T.matrix(name='x')
for i in xrange(self.n_layers_dA):
if i == 0:
layer_input = self.x
else:
layer_input = self.dA_layers[-1].get_active()
if(self.p_list[i] == 0):
model_file = '%s/L%d.mat'%(output_folder, i)
else:
model_file = '%s/L%d_p%g_s%g.mat'%(output_folder, i, self.p_list[i], self.sigma_list[i])
if os.path.isfile(model_file): #this layer has been trained
model_w, model_b, model_b_prime = load_model_mat(model_file)
dA_layer = dA(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1],
W=model_w, bhid=model_b, bvis=model_b_prime)
self.dA_train_flags.append(True) #set the flag
else:
dA_layer = dA(numpy_rng=numpy_rng, input=layer_input, n_visible=layers_sizes[i], n_hidden=layers_sizes[i+1])
self.dA_train_flags.append(False) #set the flag
self.dA_layers.append(dA_layer)
finetune_file = '%s/SAE%s.mat'%(output_folder, self.get_model_file())
if os.path.isfile(finetune_file): #trained
self.finetune_train_flag = True
def test_model(batch_size=100, file_name='da.pkl'):
# datasets = load_data(dataset)
print '...loading data'
datasets = load_TIMIT()
train_set, valid_set, test_set = datasets
print '...building model'
pickle_lst = [1000] # , 500, 1000
# pickle_lst = [1, 10]
for epoch in pickle_lst:
print 'epoch: ', epoch
file_name = "da_epoch_%d" % (epoch)
w, b, b_prime = load_model_mat(file_name)
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
# generate symbolic variables for input (x and y represent a
# minibatch)
x = T.matrix('x') # data, presented as rasterized images
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2 ** 30))
da = dA(
numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=129,
n_hidden=500,
W=w,
bhid=b,
bvis=b_prime
)
# test_fun = theano.function(
# inputs=[index],
# outputs=da.get_reconstructed_out(),
# givens={
# x: test_set_x[index * batch_size:(index + 1) * batch_size]
# }
# )
get_outputs = theano.function(
inputs=[index],
outputs=da.get_active(),
givens={
x: test_set[index * batch_size:(index + 1) * batch_size]
}
)
index = 1
hidden_value = get_outputs(index)
plot_data = test_set.get_value(borrow=True)[index * batch_size:(index + 1) * batch_size]
pylab.figure(); pylab.hist(plot_data.reshape(plot_data.size, 1), 50);
pylab.figure();pylab.plot(numpy.mean(plot_data, axis=0), '*');pylab.xlim(0, 128);pylab.ylim(0, 1);
pylab.figure();pylab.hist(hidden_value.reshape(hidden_value.size, 1), 50);
pylab.figure();pylab.plot(numpy.mean(hidden_value, axis=0), '*');pylab.ylim(0, 1);
pylab.show()
set_trace()
# pylab.title(epoch)
pylab.show()
def auto_encoder_finetune(datasets=None, p=0, sigma=1, param=None, training_epochs=1000):
if datasets == None:
datasets = load_TIMIT()
train_set, valid_set, test_set = datasets
def get_shared(x):
return theano.shared(numpy.asarray(x, dtype=theano.config.floatX), borrow=False)
layers_sizes = param['layers_sizes']
L1_file = param['L1_file']
L2_file = param['L2_file']
L3_file = param['L3_file']
L4_file = param['L4_file']
output_folder = param['output_folder']
item_str = param['item_str']
valid_flag = 0
finetune_lr = 0.1
batch_size = 100
# compute number of minibatches for training, validation and testing
n_train_batches = train_set.get_value(borrow=True).shape[0] / batch_size
# allocate symbolic variables for the data
index = T.lscalar('index')
learning_rate = T.scalar('lr')
x = T.matrix('x')
if not os.path.isdir(output_folder):
os.makedirs(output_folder)
# pickle_lst = [100,200,300,400,500,600,700,800,900,1000]
L1_w, L1_b, L1_b_prime = load_model_mat(file_name=L1_file, shared=0)
L2_w, L2_b, L2_b_prime = load_model_mat(file_name=L2_file, shared=0)
L3_w, L3_b, L3_b_prime = load_model_mat(file_name=L3_file, shared=0)
L4_w, L4_b, L4_b_prime = load_model_mat(file_name=L4_file, shared=0)
w_list = [get_shared(L1_w), get_shared(L2_w), get_shared(L3_w), get_shared(L4_w),
get_shared(L4_w.T), get_shared(L3_w.T), get_shared(L2_w.T), get_shared(L1_w.T)]
b_list = [get_shared(L1_b), get_shared(L2_b), get_shared(L3_b), get_shared(L4_b),
get_shared(L4_b_prime), get_shared(L3_b_prime), get_shared(L2_b_prime), get_shared(L1_b_prime)]
rng = numpy.random.RandomState(123)
sae = SAE(numpy_rng=rng,
input=x,
layers_sizes=layers_sizes,
w_list=w_list,
b_list=b_list)
print '... building the model'
train_fn, valid_model, test_model = sae.build_finetune_functions(datasets, batch_size, learning_rate, p, sigma)
print '... training'
logger = mylogger(output_folder + '/' + item_str + '.log')
logger.log('p:%g, sigma:%g, learning rate:%g' % (p, sigma, finetune_lr))
#===========================================================================
# start training
#===========================================================================
patience = 100 * n_train_batches # look as this many examples regardless
patience_increase = 2. # wait this much longer when a new best is found
improvement_threshold = 0.999 # a relative improvement of this much is considered significant
validation_frequency = min(n_train_batches, patience / 2)
best_validation_loss = numpy.inf
test_score = 0.
start_time = strict_time()
done_looping = False
epoch = 0;best_epoch = 0
while (epoch < training_epochs):
epoch = epoch + 1
epoch_time_s = strict_time()
c = []
for minibatch_index in xrange(n_train_batches):
err = train_fn(minibatch_index,finetune_lr)
# err = 0
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % 100 == 0:
finetune_lr = 0.8 * finetune_lr
logger.log('learning rate: %g' % (finetune_lr))
if valid_flag == 0:
continue
validation_losses = numpy.mean(valid_model())
logger.log('valid %.5f' % (validation_losses))
if validation_losses < best_validation_loss:
best_validation_loss = validation_losses
best_epoch = epoch
# test it on the test set
test_losses = numpy.mean(test_model())
logger.log('test %.5f' % (test_losses))
sae.save_model_mat(output_folder + '/' + item_str + '.mat')
# logger.log('best validation %.5f, test error %.5f, on epoch %d'%(best_validation_loss, test_losses, best_epoch))
sae.save_model_mat(output_folder + '/' + item_str + '.mat')
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
return
for epoch in xrange(1, training_epochs + 1):
# go through trainng set
c = []
epoch_time_s = strict_time()
for batch_index in xrange(n_train_batches):
err = train_fn(batch_index, finetune_lr)
# err = 0
c.append(err)
logger.log('Training epoch %d, cost %.5f, took %f seconds ' % (epoch, numpy.mean(c), (strict_time() - epoch_time_s)))
if epoch % 100 == 0:
finetune_lr = 0.8 * finetune_lr
logger.log('learning rate: %g' % (finetune_lr))
sae.save_model_mat(output_folder + '/' + item_str + '.mat')
logger.log('ran for %.2f m ' % ((strict_time() - start_time) / 60.))
