train_images = load_MNIST.load_MNIST_images(
'data/mnist/train-images-idx3-ubyte')
train_labels = load_MNIST.load_MNIST_labels(
'data/mnist/train-labels-idx1-ubyte')
##======================================================================
## STEP 2: Train the first sparse autoencoder
# This trains the first sparse autoencoder on the unlabelled STL training
# images.
# If you've correctly implemented sparseAutoencoderCost.m, you don't need
# to change anything here.
# Randomly initialize the parameters
sae1_theta = sparse_autoencoder.initialize(hidden_size_L1, input_size)
J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(
x, input_size, hidden_size_L1, lambda_, sparsity_param, beta, train_images)
options_ = {'maxiter': 400, 'disp': True}
result = scipy.optimize.minimize(J,
sae1_theta,
method='L-BFGS-B',
jac=True,
options=options_)
sae1_opt_theta = result.x
print(result)
##======================================================================
## STEP 3: Train the second sparse autoencoder
# This trains the second sparse autoencoder on the first autoencoder
# featurse.
plt.figure()
plt.imsave('sparse_autoencoder_minist_patches.png', image, cmap=plt.cm.gray)
plt.imshow(image, cmap=plt.cm.gray)
visible_size = patches.shape[0] # Number of input units
hidden_size = 196 # Number of hidden units
weight_decay_param = 3e-3 # Weight decay parameter, which is the lambda in lecture notes
beta = 3 # Weight of sparsity penalty term
sparsity_param = 0.1 # Desired average activation of the hidden units.
# Randomly initialize the fitting parameters
theta = initialize_parameters(hidden_size, visible_size)
J = lambda theta: sparse_autoencoder_cost(
theta, visible_size, hidden_size, weight_decay_param, sparsity_param, beta,
patches)
# The number of maximun iterations is set as 400,
# which is good enough to get reasonable results.
options = {'maxiter': 400, 'disp': True, 'gtol': 1e-5, 'ftol': 2e-9}
results = scipy.optimize.minimize(J,
theta,
method='L-BFGS-B',
jac=True,
options=options)
opt_theta = results['x']
print("Show the results of optimization as following.\n")
print(results)
#
# (b) Add in the weight decay term (in both the cost function and the derivative
# calculations), then re-run Gradient Checking to verify correctness.
#
# (c) Add in the sparsity penalty term, then re-run Gradient Checking to
# verify correctness.
#
# Feel free to change the training settings when debugging your
# code. (For example, reducing the training set size or
# number of hidden units may make your code run faster; and setting beta
# and/or lambda to zero may be helpful for debugging.) However, in your
# final submission of the visualized weights, please use parameters we
# gave in Step 0 above.
(cost, grad) = sparse_autoencoder.sparse_autoencoder_cost(theta, visible_size,
hidden_size, lambda_,
sparsity_param, beta, patches)
print cost, grad
##======================================================================
## STEP 3: Gradient Checking
#
# Hint: If you are debugging your code, performing gradient checking on smaller models
# and smaller training sets (e.g., using only 10 training examples and 1-2 hidden
# units) may speed things up.
# First, lets make sure your numerical gradient computation is correct for a
# simple function. After you have implemented computeNumericalGradient.m,
# run the following:
if debug:
train_images = load_MNIST.load_MNIST_images('data/mnist/train-images-idx3-ubyte')
train_labels = load_MNIST.load_MNIST_labels('data/mnist/train-labels-idx1-ubyte')
##======================================================================
## STEP 2: Train the first sparse autoencoder
# This trains the first sparse autoencoder on the unlabelled STL training
# images.
# If you've correctly implemented sparseAutoencoderCost.m, you don't need
# to change anything here.
# Randomly initialize the parameters
sae1_theta = sparse_autoencoder.initialize(hidden_size_L1, input_size)
J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, input_size, hidden_size_L1,
lambda_, sparsity_param,
beta, train_images)
options_ = {'maxiter': 400, 'disp': True}
result = scipy.optimize.minimize(J, sae1_theta, method='L-BFGS-B', jac=True, options=options_)
sae1_opt_theta = result.x
print result
##======================================================================
## STEP 3: Train the second sparse autoencoder
# This trains the second sparse autoencoder on the first autoencoder
# featurse.
# If you've correctly implemented sparseAutoencoderCost.m, you don't need
# to change anything here.
test_labels = labels[test_index]
print '# examples in unlabeled set: {0:d}\n'.format(unlabeled_data.shape[1])
print '# examples in supervised training set: {0:d}\n'.format(train_data.shape[1])
print '# examples in supervised testing set: {0:d}\n'.format(test_data.shape[1])
## ======================================================================
# STEP 2: Train the sparse autoencoder
# This trains the sparse autoencoder on the unlabeled training
# images.
# Randomly initialize the parameters
theta = sparse_autoencoder.initialize(hidden_size, input_size)
J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, input_size, hidden_size,
lambda_, sparsity_param,
beta, unlabeled_data)
options_ = {'maxiter': 400, 'disp': True}
result = scipy.optimize.minimize(J, theta, method='L-BFGS-B', jac=True, options=options_)
opt_theta = result.x
print result
# Visualize the weights
W1 = opt_theta[0:hidden_size * input_size].reshape(hidden_size, input_size).transpose()
display_network.display_network(W1)
##======================================================================
## STEP 3: Extract Features from the Supervised Dataset
#
"""
STEP 2: Train the first sparse autoencoder
This trains the first sparse autoencoder on the unlabelled STL training images.
If you've correctly implemented sparse_autoencoder_cost, you don't need
to change anything here.
"""
# Randomly initialize the parameters
sae1_theta = initialize_parameters(hidden_size_L1, input_size)
# Instructions: Train the first layer sparse autoencoder, this layer has
# an hidden size of "hidden_size_L1"
# You should store the optimal parameters in sae1_opt_theta
J = lambda theta : sparse_autoencoder_cost(theta, input_size, hidden_size_L1, lambda_, sparsity_param, beta, train_data)
options = {'maxiter': maxiter, 'disp': True}
results = scipy.optimize.minimize(J, sae1_theta, method='L-BFGS-B', jac=True, options=options)
sae1_opt_theta = results['x']
print("Show the results of optimization as following.\n")
print(results)
# Visualize weights
visualize_weights = False
if visualize_weights:
W1 = sae1_opt_theta[0:hidden_size_L1*input_size].reshape((hidden_size_L1, input_size))
image = display_network(W1.T)
plt.figure()
plt.figure()
plt.imsave('sparse_autoencoder_minist_patches.png', image, cmap=plt.cm.gray)
plt.imshow(image, cmap=plt.cm.gray)
visible_size = patches.shape[0] # Number of input units
hidden_size = 196 # Number of hidden units
weight_decay_param = 3e-3 # Weight decay parameter, which is the lambda in lecture notes
beta = 3 # Weight of sparsity penalty term
sparsity_param = 0.1 # Desired average activation of the hidden units.
# Randomly initialize the fitting parameters
theta = initialize_parameters(hidden_size, visible_size)
J = lambda theta : sparse_autoencoder_cost(theta, visible_size, hidden_size, weight_decay_param, sparsity_param, beta, patches)
# The number of maximun iterations is set as 400,
# which is good enough to get reasonable results.
options = {'maxiter': 400, 'disp': True, 'gtol': 1e-5, 'ftol': 2e-9}
results = scipy.optimize.minimize(J, theta, method='L-BFGS-B', jac=True, options=options)
opt_theta = results['x']
print("Show the results of optimization as following.\n")
print(results)
# Visualization
W1 = opt_theta[0:hidden_size*visible_size].reshape((hidden_size, visible_size))
image = display_network(W1.T)
plt.figure()
