def test_DAfinetune(finetune_lr=0.1,
training_epochs=5,
hidden_layers_sizes=[1000, 500, 250, 30],
dataset='/Users/jon/Data/mnist/mnist.pkl.gz',
batch_size=10):
"""
Demonstrates how to train and test a Deep Autoencoder.
This is demonstrated on MNIST.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:default: 0.1
:type training_epochs: int
:param training_epochs: maximal number of iterations to run the optimizer.
:default: 1000
:type dataset: string
:param dataset: path the the pickled dataset
:default: 'mnist.pkl.gz'
:type batch_size: int
:param batch_size: the size of a minibatch
:default: 10
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the Deep Autoencoder
dafinetune = DAfinetune(numpy_rng=numpy_rng,
n_ins=28 * 28,
hidden_layers_sizes=hidden_layers_sizes)
### jdy code block
# print dafinetune.params
# print 'layer0'
# print dafinetune.params[0].get_value()[0:3, 0:3]
# print 'layer1'
# print dafinetune.params[2].get_value()[0:3, 0:3]
# print 'layer2'
# print dafinetune.params[4].get_value()[0:3, 0:3]
###
# # save_short(dafinetune, '/Users/jon/models/DBNDA_theano/model_test.pkl')
# # save_med_pkl(dafinetune, '/Users/jon/models/DBNDA_theano/model_test2.pkl')
# # save_med_npy(dafinetune, '/Users/jon/models/DBNDA_theano/model_test3.npy')
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
training_fn, testing_fn = dafinetune.build_finetune_functions(
train_set_x=train_set_x,
valid_set_x=valid_set_x,
test_set_x=test_set_x,
batch_size=batch_size)
print '... finetuning the model'
start_time = time.time()
# for each epoch
for epoch in xrange(training_epochs):
c = [] #list to collect costs
e = [] #list to collect train errors
# TRAIN SET COST: for each batch, append the cost for that batch (should
# be 5000 costs in c). Also append mse to e.
for batch_index in xrange(n_train_batches):
cost, err = training_fn(index=batch_index, lr=finetune_lr)
c.append(cost)
e.append(err)
# TEST SET ERROR: calculate test set error for each epoch
te = [] #list to collect test errors
for i in xrange(n_test_batches):
te.append(testing_fn(index=i))
# print results to screen
print(
'Training epoch %d, Train cost %0.3f, Train MSE %0.3f, Test '
'MSE %0.3f' %
(epoch, numpy.mean(c), numpy.mean(e), numpy.mean(te)))
end_time = time.time()
print >> sys.stderr, ('The fine tuning code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((end_time - start_time) / 60.))
def test_SRBM_SA(finetune_lr=0.1, pretraining_epochs=1,
pretrain_lr=0.1, k=1, training_epochs=1,
dataset='/Users/jon/Data/mnist/mnist.pkl.gz', batch_size=10):
### finetune_lr and training_epochs not needed for SRBM
"""
Demonstrates how to train and test a Deep Belief Network.
This is demonstrated on MNIST.
:type finetune_lr: float
:param finetune_lr: learning rate used in the finetune stage
:default: 0.1
:type pretraining_epochs: int
:param pretraining_epochs: number of epoch to do pretraining
:default: 100
:type pretrain_lr: float
:param pretrain_lr: learning rate to be used during pre-training
:default: 0.01
:type k: int
:param k: number of Gibbs steps in CD/PCD
:default: 1
:type training_epochs: int
:param training_epochs: maximal number of iterations to run the optimizer.
:default: 1000
:type dataset: string
:param dataset: path the the pickled dataset
:default: 'mnist.pkl.gz'
:type batch_size: int
:param batch_size: the size of a minibatch
:default: 10
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
#print n_train_batches
# numpy random generator
numpy_rng = numpy.random.RandomState(123)
print '... building the model'
# construct the SRBM_SA
srbm_sa = SRBM_SA(numpy_rng=numpy_rng, n_ins=28 * 28,
hidden_layers_sizes=[1000, 500, 250, 30],
n_outs=10)
### jdy code block
print srbm_sa.params
print 'layer0'
print srbm_sa.params[0].get_value()[0:3, 0:3]
print 'layer1'
print srbm_sa.params[2].get_value()[0:3, 0:3]
print 'layer2'
print srbm_sa.params[4].get_value()[0:3, 0:3]
###
#########################
# PRETRAINING THE MODEL #
#########################
print '... getting the pretraining functions'
### creates a list of pretraining fxns for each layer in the SRBM_SA. This is
### where the self.sigmoid_layer[-1].output is needed -- to create the
### appropriate equation/function for pretraining
pretraining_fns = srbm_sa.pretraining_functions(train_set_x=train_set_x,
batch_size=batch_size,
k=k)
### *note
'''Now any function pretrain_fns[i] takes as arguments index and optionally
lr - the learning rate. Note that the names of the parameters are the names
given to the Theano variables (e.g. lr) when they are constructed and not
the python variables (e.g. learning_rate).'''
print '... pre-training the model'
start_time = time.time() ###changed time.clock() to time.time() because
### getting times that are way too long with time.clock()
## Pre-train layer-wise
for i in xrange(srbm_sa.n_layers):
# go through pretraining epochs
for epoch in xrange(pretraining_epochs):
# go through the training set
c = []
for batch_index in xrange(n_train_batches):
c.append(pretraining_fns[i](index=batch_index,
lr=pretrain_lr))
### see *note above
### for each function call of the pretraining fns, the states
### of the shared variables (e.g. self.params) are updated.
### pretraining_fns = list of cost/update functions for each
### layer of SRBM_SA. Each call to this fxn returns the cost and
### updates the parameters for that layer. See 'Shared Variable'
### section here: http://deeplearning.net/software/theano/tutorial/examples.html#logistic-function
print 'Pre-training layer %i, epoch %d, cost ' % (i, epoch),
print numpy.mean(c)
### jdy code block
# print srbm_sa.params
# print 'layer %i, epoch %d' % (i,epoch)
# jdy_params0 = srbm_sa.params[i * 2].get_value()
# print jdy_params0.shape
# print jdy_params0[0:3, 0:3]
###
end_time = time.time() ###changed time.clock() to time.time()
print >> sys.stderr, ('The pretraining code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))
### jdy code block
# print i, epoch, batch_index
# ### temp = pretraining_fns[i](index=batch_index,lr=pretrain_lr))
# ### Running the line above changes the weights in layer 3 by a very small
# ### amount leading to a minimal change in cost. any time pretraining_fns[i]
# ### is called it will update the shared variables for that layer only.
# params = type(srbm_sa.params[0])
# print params
# print srbm_sa.params
# print 'layer0'
# print srbm_sa.params[0].get_value()[0:3, 0:3]
# print 'layer1'
# print srbm_sa.params[2].get_value()[0:3, 0:3]
# print 'layer2'
# print srbm_sa.params[4].get_value()[0:3, 0:3]
###
# save_short(srbm_sa, '/Users/jon/models/DBNDA_theano/model_test.pkl')
# save_med_pkl(srbm_sa, '/Users/jon/models/DBNDA_theano/model_test2.pkl')
# save_med_npy(srbm_sa, '/Users/jon/models/DBNDA_theano/model_test3.npy')
########################
# FINETUNING THE MODEL #
########################
# get the training, validation and testing function for the model
print '... getting the finetuning functions'
training_fns = srbm_sa.build_finetune_functions(train_set_x=train_set_x,
valid_set_x=valid_set_x,
test_set_x=test_set_x,
batch_size=batch_size)
print '... finetuning the model'
s_time = time.time()
## Finetune train layer-wise
for layer in xrange(srbm_sa.n_layers):
# go through training epochs
for ep in xrange(training_epochs):
# go through the training set
t_cost = []
for b_index in xrange(n_train_batches):
t_cost.append(training_fns[layer](index=b_index,
lr=finetune_lr))
### see *note above
### for each function call of the training fns, the states
### of the shared variables (e.g. self.params) are updated.
### training_fns = list of cost/update functions for each
### layer of SRBM_SA. Each call to this fxn returns the cost and
### updates the parameters for that layer. See 'Shared Variable'
### section here: http://deeplearning.net/software/theano/tutorial/examples.html#logistic-function
print 'Pre-training layer %i, epoch %d, cost ' % (layer, ep),
print numpy.mean(t_cost)
### jdy code block
# print srbm_sa.params
# print 'layer %i, epoch %d' % (layer,ep)
# jdy_params0train = srbm_sa.params[layer * 2].get_value()
# print jdy_params0train.shape
# print jdy_params0train[0:3, 0:3]
###
e_time = time.time() ###changed time.clock() to time.time()
print >> sys.stderr, ('The finetuning code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((e_time - s_time) / 60.))
def test_A(learning_rate=0.1,
training_epochs=2,
dataset='/Users/jon/Data/mnist/mnist.pkl.gz',
batch_size=20,
output_folder='dA_plots',
testing=0):
"""
This demo is tested on MNIST
:type learning_rate: float
:param learning_rate: learning rate used for training the AutoEncoder
:type training_epochs: int
:param training_epochs: number of epochs used for training
:type dataset: string
:param dataset: path to the picked dataset
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as rasterized images
# if not os.path.isdir(output_folder):
# os.makedirs(output_folder)
# os.chdir(output_folder)
####################################
# BUILDING THE MODEL NO CORRUPTION #
####################################
rng = numpy.random.RandomState(123)
theano_rng = RandomStreams(rng.randint(2**30))
a = A(numpy_rng=rng,
theano_rng=theano_rng,
input=x,
n_visible=28 * 28,
n_hidden=500)
### jdy code block
print a.params
print 'layer0'
print a.params[0].get_value()[0:3, 0:3]
###
cost, updates = a.get_cost_updates(learning_rate=learning_rate)
###removed 'corruption level'
train_a = theano.function(
[index],
cost,
updates=updates,
givens={x: train_set_x[index * batch_size:(index + 1) * batch_size]})
print '...training the model'
start_time = time.time() ###changed time.clock() to time.time()
############
# TRAINING #
############
# go through training epochs
for epoch in xrange(training_epochs):
# go through trainng set
c = []
for batch_index in xrange(n_train_batches):
c.append(train_a(batch_index))
print 'Training epoch %d, cost ' % epoch, numpy.mean(c)
### jdy code block
print a.params
print 'epoch %d' % (epoch)
jdy_params0 = a.params[0].get_value()
print jdy_params0.shape
print jdy_params0[0:3, 0:3]
###
end_time = time.time() ###changed time.clock() to time.time()
training_time = (end_time - start_time)
print >> sys.stderr, ('The autoencoder code for file ' +
os.path.split(__file__)[1] + ' ran for %.2fm' %
((training_time) / 60.))
#save_med_pkl(a, '/Users/jon/models/DBNDA_theano/modelx.pkl')
if testing:
return a