def reconstruct(
autoencoder_in=None,
dataset='mnist.pkl.gz',
batch_size=8
):
# Loads the parameters of a trained model
# Takes a test case, run it through the
# trained autoencoder, and plot the reconstructed version
# to see how our autoencoder generalizes an arbitrary case.
if not autoencoder_in:
W, hbias, vbias = load_parameters()
autoencoder_in = autoencoder(W = W, hbias = hbias, vbias = vbias)
test_set_x, test_set_y = load_data(dataset)[2]
x = T.matrix('x')
index = T.iscalar('index')
pre_output = autoencoder_in.test_prop(input = x, params = autoencoder_in.params)
reconstruct = autoencoder_in.layer_info[0](pre_output)
error = autoencoder_in.gradient_reconstruction_error(input = x, phase = 'test', params = autoencoder_in.params)
sgd_test = theano.function(
[index],
[reconstruct, error],
givens = {
x : test_set_x[
index * batch_size :
(index + 1) * batch_size
]
},
name = 'sgd_test'
)
original = test_set_x.get_value(borrow=True)[8:16]
reconstructed, error = sgd_test(1)
print 'Reconstruction error is %3f' % error
images = np.append(
original, reconstructed
).reshape((batch_size * 2, 28*28))
print images.shape[0], images.shape[1]
images = Image.fromarray(
tile_raster_images(
X = images,
img_shape = (28, 28),
tile_shape = (2, batch_size),
tile_spacing = (2,2)
)
)
images.save('autoencoder_reconstructed_images.png')
pass
def Sample(dataset="mnist.pkl.gz", random_initialization=True, sample_every=1000, no_samples=1):
# Initialize sample randomly if random_initialization = True.
# For a Gibbs chain, take a sample after (sample_every) steps.
# no_samples: int, how many samples to be taken.
RBMin = RBM(resume=True)
datasets = load_data(dataset)
test_set_x, test_set_y = datasets[2] # Chose test set
nrg = np.random.RandomState()
if random_initialization:
chain_start = theano.shared(nrg.uniform(low=0.0, high=1.0, size=(28 * 28,)).astype("float32"))
else:
chain_start = theano.shared(
test_set_x.get_value(borrow=True)[np.floor(28 * 28 * nrg.uniform(low=0.0, high=5.0)).astype("int32")]
)
# Run a single round of Gibbs sampler
([h0_pre, h0_mean, h0, v1_pre, v1_mean, v1], updates) = theano.scan(
RBMin.GS_vhv, outputs_info=[None, None, None, None, chain_start, None], n_steps=sample_every
)
# Update the updates dictionary
updates[chain_start] = v1_mean[-1]
GS = theano.function([], [v1_mean[-1], v1[-1]], updates=updates, name="Gibbs Sampler")
# Plot samples.
# Flattened and reshaped accordingly so that each row
# represents an image
start_time = timeit.default_timer()
images = np.array([GS() for i in range(no_samples)], "float32").flatten().reshape((2 * no_samples, 28 * 28))
images = Image.fromarray(
tile_raster_images(X=images, img_shape=(28, 28), tile_shape=(no_samples, 2), tile_spacing=(1, 5))
)
images.save("RBM_generated_samples.png")
end_time = timeit.default_timer()
print "Sampling took %f minutes" % ((end_time - start_time) / 60.0)
def test_mlp(learning_rate=0.01,
L1_reg=0.00,
L2_reg=0.0001,
n_epochs=1000,
dataset='../data/mnist.pkl.gz',
batch_size=20,
n_hidden=500):
"""
Demonstrate stochastic gradient descent optimization for a multilayer
perceptron
This is demonstrated on MNIST
:type learning_rate: float
:paran learning_rate: learning rate used (factor for the stochastic gradient)
:type L1_reg: float
:param L1_reg: L1-norm's weight when added to the cost (see
regularization)
:type L2_reg: float
:param L2_reg: L2-norm's weight when added to the cost (see
regularization)
:type n_epochs: int
:params n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
datasets = load_data(
dataset) # use the load_data()from logisticRegression module
# datasets[0]:train info, datasets[1]:valid info, datasets[2]:test info, all of them are tuple,
# details in Theano
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
#####################
#BUILD ACTUAL MODEL #
#####################
print '... building the model'
# allocate symbolic variables for the data,info include index, raterized images
# labels.
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as raterized images
y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
rng = numpy.random.RandomState(1234)
# construct the MLP class, n_in the dimensionty of input,n_out: the number
# of the label, in this experiment it is 10 (digit:0-9)
classifier = MLP(rng=rng,
input=x,
n_in=28 * 28,
n_hidden=n_hidden,
n_out=10)
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost = classifier.negative_log_likelihood(y) \
+ L1_reg * classifier.L1 \
+ L2_reg * classifier.L2_sqr
# compiling a Theano function that computes the mistakes that are made
# by the model on a minibatch,计算每个minibatch中与model的误差
test_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: test_set_x[index * batch_size:(index + 1) * batch_size],
y: test_set_y[index * batch_size:(index + 1) * batch_size]
})
validate_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size:(index + 1) * batch_size],
y: valid_set_y[index * batch_size:(index + 1) * batch_size]
})
# compute the gradient of cost with respect to theta (sorted in params)
# the resulting gradients will be stored in a list gparas，计算响应的梯度，
# 用于mlp反向传输来调整各个节点的权值
gparams = []
for param in classifier.params:
gparam = T.grad(cost, param)
gparams.append(gparam)
# specify how to upate the parameters of the model as a list of
# (variable, update expression) pairs
updates = []
# given two list the zip A =[a1, a2, a3, a4] and B=[b1, b2, b3, b4] of
# same length, zip generates a list C of same size, where each element
# is a pair formed from the two lists:
# c = [(a1, b1),(a2,b2),(a3,b3),(a4,b4)]
# create a rule defined by the code
for param, gparam in zip(classifier.params, gparams):
updates.append(
(param, param - learning_rate *
gparam)) # in logisticRegression, it called gradient descent
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
train_model = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={
x: train_set_x[index * batch_size:(index + 1) * batch_size],
y: train_set_y[index * batch_size:(index + 1) * batch_size]
})
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this muchi is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
best_params = None
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = time.clock()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_model(
minibatch_index) # 利用前面使用theano创建的train_model函数得到相应地cost函数
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# computer zero-one loss on validation set
validation_losses = [
validate_model(i) for i in xrange(n_valid_batches)
]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %o/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
# improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [
test_model(i) for i in xrange(n_test_batches)
]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete. Best validation score of %f %%'
'obtained at iteration %i, with test performance %f %%') %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file' + os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.0))
seq_hidden = [200,150,100,100,100],
)
params = auto.params
params_moving = [theano.shared(param.get_value()) for param in params]
gradient_batch = T.matrix('batch')
curvature_batch = gradient_batch
pre_output, curvature_cost = auto.curvature_reconstruction_error(
input = gradient_batch,
phase = 'train',
params = params )
gradient_cost = lambda params: auto.gradient_reconstruction_error(
input = gradient_batch,
phase = 'train',
params = params )
datasets = load_data('mnist.pkl.gz')
data = datasets[0][0]
data_size = data.get_value(borrow=True).shape[0]
gradient_batch_size = 1000
curvature_batch_size = 1000
damping_constant = theano.shared(np.array(1., 'float32'))
n_epochs = 20
no_iterations = 30
HF = HF(params = params,
params_moving = params_moving,
gradient_batch = gradient_batch,
curvature_batch = curvature_batch,
pre_output = pre_output,
curvature_cost = curvature_cost,
gradient_cost = gradient_cost,
def test_mlp(
learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, dataset="mnist.pkl.gz", batch_size=20, n_hidden=500
):
"""
Demonstrate stochastic gradient descent optimization for a multilayer
perceptron
This is demonstrated on MNIST.
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient
:type L1_reg: float
:param L1_reg: L1-norm's weight when added to the cost (see
regularization)
:type L2_reg: float
:param L2_reg: L2-norm's weight when added to the cost (see
regularization)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size
print "... building the model"
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
x = T.matrix("x") # the data is presented as rasterized images
y = T.ivector("y") # the labels are presented as 1D vector of
# [int] labels
rng = numpy.random.RandomState(1234)
# construct the MLP class
classifier = MLP(rng=rng, input=x, n_in=28 * 28, n_hidden=n_hidden, n_out=10)
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost = classifier.negative_log_likelihood(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr
# compiling a Theano function that computes the mistakes that are made
# by the model on a minibatch
test_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: test_set_x[index * batch_size : (index + 1) * batch_size],
y: test_set_y[index * batch_size : (index + 1) * batch_size],
},
)
validate_model = theano.function(
inputs=[index],
outputs=classifier.errors(y),
givens={
x: valid_set_x[index * batch_size : (index + 1) * batch_size],
y: valid_set_y[index * batch_size : (index + 1) * batch_size],
},
)
gparams = [T.grad(cost, param) for param in classifier.params]
# specify how to update the parameters of the model as a list of
# (variable, update expression) pairs
# given two lists of the same length, A = [a1, a2, a3, a4] and
# B = [b1, b2, b3, b4], zip generates a list C of same size, where each
# element is a pair formed from the two lists :
# C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)]
updates = [(param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams)]
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
train_model = theano.function(
inputs=[index],
outputs=cost,
updates=updates,
givens={
x: train_set_x[index * batch_size : (index + 1) * batch_size],
y: train_set_y[index * batch_size : (index + 1) * batch_size],
},
)
print "... training"
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.0
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index)
# iteration number
iter = (epoch - 1) * n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print (
"epoch %i, minibatch %i/%i, validation error %f %%"
% (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.0)
)
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
# improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience, iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [test_model(i) for i in xrange(n_test_batches)]
test_score = numpy.mean(test_losses)
print (
(" epoch %i, minibatch %i/%i, test error of " "best model %f %%")
% (epoch, minibatch_index + 1, n_train_batches, test_score * 100.0)
)
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print (
(
"Optimization complete. Best validation score of %f %% "
"obtained at iteration %i, with test performance %f %%"
)
% (best_validation_loss * 100.0, best_iter + 1, test_score * 100.0)
)
print >> sys.stderr, (
"The code for file " + os.path.split(__file__)[1] + " ran for %.2fm" % ((end_time - start_time) / 60.0)
)
def test_mlp(learning_rate = 0.01, L1_reg = 0.00, L2_reg = 0.0001, n_epochs=1000,
dataset = '../data/mnist.pkl.gz', batch_size = 20, n_hidden = 500):
"""
Demonstrate stochastic gradient descent optimization for a multilayer
perceptron
This is demonstrated on MNIST
:type learning_rate: float
:paran learning_rate: learning rate used (factor for the stochastic gradient)
:type L1_reg: float
:param L1_reg: L1-norm's weight when added to the cost (see
regularization)
:type L2_reg: float
:param L2_reg: L2-norm's weight when added to the cost (see
regularization)
:type n_epochs: int
:params n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: the path of the MNIST dataset file from
http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz
"""
datasets = load_data(dataset) # use the load_data()from logisticRegression module
# datasets[0]:train info, datasets[1]:valid info, datasets[2]:test info, all of them are tuple,
# details in Theano
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow = True).shape[0] / batch_size
n_valid_batches = valid_set_x.get_value(borrow = True).shape[0] / batch_size
n_test_batches = test_set_x.get_value(borrow = True).shape[0] / batch_size
#####################
#BUILD ACTUAL MODEL #
#####################
print '... building the model'
# allocate symbolic variables for the data,info include index, raterized images
# labels.
index = T.lscalar() # index to a [mini]batch
x = T.matrix('x') # the data is presented as raterized images
y = T.ivector('y') # the labels are presented as 1D vector of [int] labels
rng = numpy.random.RandomState(1234)
# construct the MLP class, n_in the dimensionty of input,n_out: the number
# of the label, in this experiment it is 10 (digit:0-9)
classifier = MLP(rng = rng, input = x, n_in =28*28,
n_hidden = n_hidden, n_out = 10)
# the cost we minimize during training is the negative log likelihood of
# the model plus the regularization terms (L1 and L2); cost is expressed
# here symbolically
cost = classifier.negative_log_likelihood(y) \
+ L1_reg * classifier.L1 \
+ L2_reg * classifier.L2_sqr
# compiling a Theano function that computes the mistakes that are made
# by the model on a minibatch,计算每个minibatch中与model的误差
test_model = theano.function(inputs = [index],
outputs = classifier.errors(y),
givens = {
x: test_set_x[index * batch_size:(index+1)*batch_size],
y: test_set_y[index * batch_size:(index+1)*batch_size]})
validate_model = theano.function(inputs = [index],
outputs = classifier.errors(y),
givens = {
x: valid_set_x[index * batch_size:(index+1)*batch_size],
y: valid_set_y[index * batch_size:(index+1)*batch_size]})
# compute the gradient of cost with respect to theta (sorted in params)
# the resulting gradients will be stored in a list gparas，计算响应的梯度，
# 用于mlp反向传输来调整各个节点的权值
gparams = []
for param in classifier.params:
gparam = T.grad(cost, param)
gparams.append(gparam)
# specify how to upate the parameters of the model as a list of
# (variable, update expression) pairs
updates = []
# given two list the zip A =[a1, a2, a3, a4] and B=[b1, b2, b3, b4] of
# same length, zip generates a list C of same size, where each element
# is a pair formed from the two lists:
# c = [(a1, b1),(a2,b2),(a3,b3),(a4,b4)]
# create a rule defined by the code
for param, gparam in zip(classifier.params, gparams):
updates.append((param, param - learning_rate * gparam)) # in logisticRegression, it called gradient descent
# compiling a Theano function `train_model` that returns the cost, but
# in the same time updates the parameter of the model based on the rules
# defined in `updates`
train_model = theano.function(inputs = [index], outputs = cost,
updates = updates,
givens = {
x: train_set_x[index * batch_size:(index + 1)*batch_size],
y: train_set_y[index * batch_size:(index + 1)*batch_size]})
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is found
improvement_threshold = 0.995 # a relative improvement of this muchi is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
best_params = None
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = time.clock()
epoch = 0
done_looping =False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
minibatch_avg_cost = train_model(minibatch_index) # 利用前面使用theano创建的train_model函数得到相应地cost函数
# iteration number
iter = (epoch -1 )* n_train_batches + minibatch_index
if (iter + 1) % validation_frequency == 0:
# computer zero-one loss on validation set
validation_losses = [validate_model(i) for i
in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %o/%i, validation error %f %%' %
(epoch,minibatch_index + 1,n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
# improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * improvement_threshold:
patience = max(patience,iter * patience_increase)
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [test_model(i) for i in xrange(n_test_batches)]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = time.clock()
print(('Optimization complete. Best validation score of %f %%'
'obtained at iteration %i, with test performance %f %%') %
(best_validation_loss *100., best_iter + 1, test_score *100.))
print >> sys.stderr,('The code for file' +
os.path.split(__file__)[1]+
' ran for %.2fm' % ((end_time - start_time)/60.0))
def SGD(autoencoder_in=None,
resume=False,
batch_size = 20,
n_epochs=200,
improvement_ratio=1.,
patience=10,
dropout=[],
validation_freq=1,
lr=0.01,
lr_decay=0.1,
momentum=0.9,
dataset='mnist.pkl.gz'
):
# param autoencoder: autoencoder class instance
# type resume: bool
# param resume: whether to resume training from
# saved parameters
# type batch_size: int
# param batch_size: batch size
# type n_epochs: int
# param n_epochs: maximum # of epochs to train for
# type improvement_ratio: float
# param imporvement_ratio: if validation error decreases
# by this much, then consider an improvement.
# type patience: int
# param patience: How many times to persevere until
# dropping the learning rate
# type validation_freq: int
# param validation_freq: How often the validation will
# be performed (every epoch, e.g.)
# type lr: float
# param lr: learning rate
# type lr_decay: float
# param lr_decay: learning rate decay constant
# type momentum: float
# param momentum: momentum
start_compile = timeit.default_timer()
if resume:
W, hbias, vbias = load_parameters(
filename='autoencoder_params.save'
)
autoencoder_in = autoencoder(
W = W,
hbias = hbias,
vbias = vbias,
seq_hidden = [
h.get_value(borrow=True).shape[0]
for h in hbias
],
n_hidden = len(hbias),
dropout = dropout
)
if not autoencoder_in:
autoencoder_in = autoencoder() # initialize with a single hidden layer autoencoder
# Load MNIST dataset
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
# # batches
n_batch_train = train_set_x.get_value(borrow=True).shape[0] / batch_size
n_batch_valid = valid_set_x.get_value(borrow=True).shape[0] / batch_size
# Define train, valid, test optimizers for a single iteration
index = T.iscalar('index')
x = T.matrix('x')
autoencoder_in.learning_rate.set_value(
np.array(lr, 'float32')
)
cost_train, updates = autoencoder_in.param_cost_updates(
input = x,
momentum = momentum,
params = autoencoder_in.params
)
cost_valid = autoencoder_in.gradient_reconstruction_error(
input = x,
phase = 'valid',
params = autoencoder_in.params
)
sgd_train = theano.function(
[index],
cost_train,
updates = updates,
givens = {
x : train_set_x[
index * batch_size :
(index + 1) * batch_size
]
},
name = 'sgd_train'
)
sgd_valid = theano.function(
[index],
cost_valid,
givens = {
x : valid_set_x[
index * batch_size :
(index + 1) * batch_size
]
},
name = 'sgd_valid'
)
# theano.printing.pydotprint(cost_train, outfile='symbolic_graph_unopt.png', var_with_name_simple=True)
# theano.printing.pydotprint(sgd_train, outfile='symbolic_graph_opt.png', var_with_name_simple=True)
end_compile = timeit.default_timer()
print 'Compiling took %2f seconds' % (end_compile - start_compile)
continue_time = 0. # parameter in case training is continued
start_train = timeit.default_timer()
# Begin training
if resume:
histories = pickle.load(file('autoencoder_error.save', 'rb'))
train_history, valid_history, test_history = \
[h if h else [] for h in histories]
continue_time = train_history[-1][0]
else:
train_history, valid_history, test_history = [], [], []
patience_init = patience
for epoch in range(n_epochs):
train_error, valid_error, test_error = [], [], []
if patience_init == 0:
break
for iter in range(n_batch_train):
error = sgd_train(iter)
train_error.append(error)
if iter % 50 == 0:
print 'training error at iter %d is...' % iter, error
intermediate_time = timeit.default_timer()
train_history.append(
(intermediate_time - start_train + continue_time,
sum(train_error) / len(train_error))
)
if epoch % validation_freq == 0:
# Validation
for iter in range(n_batch_valid):
error = sgd_valid(iter)
valid_error.append(error)
intermediate_time_ = timeit.default_timer()
valid_history.append(
(intermediate_time_ - start_train + continue_time,
sum(valid_error) / len(valid_error))
)
print 'validation error at epoch %d is...' % epoch, valid_history[-1]
patience_init -= 1
if epoch > 0 and valid_history[-1][-1] < improvement_ratio * valid_history[-2][-1]:
patience_init += patience
else:
autoencoder_in.learning_rate.set_value(
autoencoder_in.learning_rate.get_value() *
np.array(lr_decay, 'float32') )
if autoencoder_in.learning_rate.get_value() == lr * 1e-4:
break
save_parameters(params = autoencoder_in.params)
save_history(
train_history = train_history,
valid_history = valid_history
)
end_train = timeit.default_timer()
print 'Training took %3f seconds' % (end_train - start_train)
pass
def train_RBM(
RBMin=None,
lr=0.01,
lr_decay=0.1,
momentum=0.9,
improvement_ratio=0.95,
batch_size=20,
dataset="mnist.pkl.gz",
epochs=15,
n_hidden=500,
n_chains=20,
n_samples=10,
):
# n_chains: int, # of parallel Gibbs chains to be used for sampling
# n_samples: int, # samples to plot for each chain
lr_init = lr
if not RBMin:
RBMin = RBM()
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
test_set_x, test_set_y = datasets[2]
index = T.iscalar()
x = T.matrix()
xent, updates = CD_k(RBMin, input=x, lr=lr)
optimize = theano.function(
[index],
xent,
updates=updates,
givens={x: train_set_x[index * batch_size : (index + 1) * batch_size]},
name="optimize",
)
n_batch = train_set_x.get_value().shape[0] / batch_size
train_error = np.array([], "float64")
learning_rate = []
start_time = timeit.default_timer()
for epoch in range(epochs):
print "epoch %d:" % epoch, "\n"
for iter in range(n_batch):
error = optimize(iter)
if iter % 250 == 0:
# print Recon error every 5000 examples & save in train_error for later plotting
print "Reconstruction error at iteration %d:" % (iter * batch_size), error
train_error = np.append(train_error, error)
# Check if the recon error of the last epoch has improved.
# If yes, maintain the current learning rate.
# Otherwise, lr = lr * lr_decay
# Check last 25,000 examples.
# Save the learning rate
print "\n", "learning rate for epoch %d: %f" % (epoch, lr), "\n"
learning_rate.append(lr)
if epoch > 0:
if (
train_error[-5:].mean()
> improvement_ratio
* train_error[-5 - 50000.0 / (250 * batch_size) : -50000.0 / (250 * batch_size)].mean()
):
lr = lr * lr_decay
xent, updates = CD_k(RBMin, input=x, lr=lr)
# Save models & train_error each epoch
f = file("RBM_weights.save", "wb")
pickle.dump(RBMin.W.get_value(borrow=True), f, protocol=pickle.HIGHEST_PROTOCOL)
f.close()
f = file("RBM_hbias.save", "wb")
pickle.dump(RBMin.hbias.get_value(borrow=True), f, protocol=pickle.HIGHEST_PROTOCOL)
f.close()
f = file("RBM_vbias.save", "wb")
pickle.dump(RBMin.vbias.get_value(borrow=True), f, protocol=pickle.HIGHEST_PROTOCOL)
f.close()
f = file("train_error.save", "wb")
pickle.dump(train_error, f, protocol=pickle.HIGHEST_PROTOCOL)
f.close()
f = file("learning_rate.save", "wb")
pickle.dump(learning_rate, f, protocol=pickle.HIGHEST_PROTOCOL)
f.close()
# plot weights at each epoch
image = Image.fromarray(
tile_raster_images(
X=RBMin.W.get_value(borrow=True).T, img_shape=(28, 28), tile_shape=(20, 20), tile_spacing=(1, 1)
)
)
image.save("Weights_at_epoch_%d.png" % (epoch + 2))
# Stop training if learning rate becomes too low
# This means objective is simply not improving
if lr <= lr_init * 0.001:
break
# plot the training procedure
_, axis = pylab.subplots()
grid = np.arange(len(train_error))
axis.plot(grid, train_error)
pylab.plot()
pylab.show()
end_time = timeit.default_timer()
pretraining_time = end_time - start_time
print "Pretraining took %f minutes" % (pretraining_time / 60.0)
def evaluate_lenet5(learning_rate=0.1, n_epochs=200,
dataset='mnist.pkl.gz',
nkerns=[20, 50], batch_size=500):
""" Demonstrates lenet on MNIST dataset
:type learning_rate: float
:param learning_rate: learning rate used (factor for the stochastic
gradient)
:type n_epochs: int
:param n_epochs: maximal number of epochs to run the optimizer
:type dataset: string
:param dataset: path to the dataset used for training /testing (MNIST here)
:type nkerns: list of ints
:param nkerns: number of kernels on each layer
"""
rng = numpy.random.RandomState(23455)
datasets = load_data(dataset)
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# compute number of minibatches for training, validation and testing
n_train_batches = train_set_x.get_value(borrow=True).shape[0]
n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
n_test_batches = test_set_x.get_value(borrow=True).shape[0]
n_train_batches /= batch_size
n_valid_batches /= batch_size
n_test_batches /= batch_size
# allocate symbolic variables for the data
index = T.lscalar() # index to a [mini]batch
# start-snippet-1
x = T.matrix('x') # the data is presented as rasterized images
y = T.ivector('y') # the labels are presented as 1D vector of
# [int] labels
######################
# BUILD ACTUAL MODEL #
######################
print '... building the model'
# Reshape matrix of rasterized images of shape (batch_size, 28 * 28)
# to a 4D tensor, compatible with our LeNetConvPoolLayer
# (28, 28) is the size of MNIST images.
layer0_input = x.reshape((batch_size, 1, 28, 28))
# Construct the first convolutional pooling layer:
# filtering reduces the image size to (28-5+1 , 28-5+1) = (24, 24)
# maxpooling reduces this further to (24/2, 24/2) = (12, 12)
# 4D output tensor is thus of shape (batch_size, nkerns[0], 12, 12)
layer0 = LeNetConvPoolLayer(
rng,
input=layer0_input,
image_shape=(batch_size, 1, 28, 28),
filter_shape=(nkerns[0], 1, 5, 5),
poolsize=(2, 2)
)
# Construct the second convolutional pooling layer
# filtering reduces the image size to (12-5+1, 12-5+1) = (8, 8)
# maxpooling reduces this further to (8/2, 8/2) = (4, 4)
# 4D output tensor is thus of shape (batch_size, nkerns[1], 4, 4)
layer1 = LeNetConvPoolLayer(
rng,
input=layer0.output,
image_shape=(batch_size, nkerns[0], 12, 12),
filter_shape=(nkerns[1], nkerns[0], 5, 5),
poolsize=(2, 2)
)
# the HiddenLayer being fully-connected, it operates on 2D matrices of
# shape (batch_size, num_pixels) (i.e matrix of rasterized images).
# This will generate a matrix of shape (batch_size, nkerns[1] * 4 * 4),
# or (500, 50 * 4 * 4) = (500, 800) with the default values.
layer2_input = layer1.output.flatten(2)
# construct a fully-connected sigmoidal layer
layer2 = HiddenLayer(
rng,
input=layer2_input,
n_in=nkerns[1] * 4 * 4,
n_out=500,
activation=T.tanh
)
# classify the values of the fully-connected sigmoidal layer
layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
# the cost we minimize during training is the NLL of the model
cost = layer3.negative_log_likelihood(y)
# create a function to compute the mistakes that are made by the model
test_model = theano.function(
[index],
layer3.errors(y),
givens={
x: test_set_x[index * batch_size: (index + 1) * batch_size],
y: test_set_y[index * batch_size: (index + 1) * batch_size]
}
)
validate_model = theano.function(
[index],
layer3.errors(y),
givens={
x: valid_set_x[index * batch_size: (index + 1) * batch_size],
y: valid_set_y[index * batch_size: (index + 1) * batch_size]
}
)
# create a list of all model parameters to be fit by gradient descent
params = layer3.params + layer2.params + layer1.params + layer0.params
# create a list of gradients for all model parameters
grads = T.grad(cost, params)
# train_model is a function that updates the model parameters by
# SGD Since this model has many parameters, it would be tedious to
# manually create an update rule for each model parameter. We thus
# create the updates list by automatically looping over all
# (params[i], grads[i]) pairs.
updates = [
(param_i, param_i - learning_rate * grad_i)
for param_i, grad_i in zip(params, grads)
]
train_model = theano.function(
[index],
cost,
updates=updates,
givens={
x: train_set_x[index * batch_size: (index + 1) * batch_size],
y: train_set_y[index * batch_size: (index + 1) * batch_size]
}
)
# end-snippet-1
###############
# TRAIN MODEL #
###############
print '... training'
# early-stopping parameters
patience = 10000 # look as this many examples regardless
patience_increase = 2 # wait this much longer when a new best is
# found
improvement_threshold = 0.995 # a relative improvement of this much is
# considered significant
validation_frequency = min(n_train_batches, patience / 2)
# go through this many
# minibatche before checking the network
# on the validation set; in this case we
# check every epoch
best_validation_loss = numpy.inf
best_iter = 0
test_score = 0.
start_time = timeit.default_timer()
epoch = 0
done_looping = False
while (epoch < n_epochs) and (not done_looping):
epoch = epoch + 1
for minibatch_index in xrange(n_train_batches):
iter = (epoch - 1) * n_train_batches + minibatch_index
if iter % 100 == 0:
print 'training @ iter = ', iter
cost_ij = train_model(minibatch_index)
if (iter + 1) % validation_frequency == 0:
# compute zero-one loss on validation set
validation_losses = [validate_model(i) for i
in xrange(n_valid_batches)]
this_validation_loss = numpy.mean(validation_losses)
print('epoch %i, minibatch %i/%i, validation error %f %%' %
(epoch, minibatch_index + 1, n_train_batches,
this_validation_loss * 100.))
# if we got the best validation score until now
if this_validation_loss < best_validation_loss:
#improve patience if loss improvement is good enough
if this_validation_loss < best_validation_loss * \
improvement_threshold:
patience = max(patience, iter * patience_increase)
# save best validation score and iteration number
best_validation_loss = this_validation_loss
best_iter = iter
# test it on the test set
test_losses = [
test_model(i)
for i in xrange(n_test_batches)
]
test_score = numpy.mean(test_losses)
print((' epoch %i, minibatch %i/%i, test error of '
'best model %f %%') %
(epoch, minibatch_index + 1, n_train_batches,
test_score * 100.))
if patience <= iter:
done_looping = True
break
end_time = timeit.default_timer()
print('Optimization complete.')
print('Best validation score of %f %% obtained at iteration %i, '
'with test performance %f %%' %
(best_validation_loss * 100., best_iter + 1, test_score * 100.))
print >> sys.stderr, ('The code for file ' +
os.path.split(__file__)[1] +
' ran for %.2fm' % ((end_time - start_time) / 60.))