train_images = load_MNIST.load_MNIST_images( 'data/mnist/train-images-idx3-ubyte') train_labels = load_MNIST.load_MNIST_labels( 'data/mnist/train-labels-idx1-ubyte') ##====================================================================== ## STEP 2: Train the first sparse autoencoder # This trains the first sparse autoencoder on the unlabelled STL training # images. # If you've correctly implemented sparseAutoencoderCost.m, you don't need # to change anything here. # Randomly initialize the parameters sae1_theta = sparse_autoencoder.initialize(hidden_size_L1, input_size) J = lambda x: sparse_autoencoder.sparse_autoencoder_cost( x, input_size, hidden_size_L1, lambda_, sparsity_param, beta, train_images) options_ = {'maxiter': 400, 'disp': True} result = scipy.optimize.minimize(J, sae1_theta, method='L-BFGS-B', jac=True, options=options_) sae1_opt_theta = result.x print(result) ##====================================================================== ## STEP 3: Train the second sparse autoencoder # This trains the second sparse autoencoder on the first autoencoder # featurse.
plt.figure() plt.imsave('sparse_autoencoder_minist_patches.png', image, cmap=plt.cm.gray) plt.imshow(image, cmap=plt.cm.gray) visible_size = patches.shape[0] # Number of input units hidden_size = 196 # Number of hidden units weight_decay_param = 3e-3 # Weight decay parameter, which is the lambda in lecture notes beta = 3 # Weight of sparsity penalty term sparsity_param = 0.1 # Desired average activation of the hidden units. # Randomly initialize the fitting parameters theta = initialize_parameters(hidden_size, visible_size) J = lambda theta: sparse_autoencoder_cost( theta, visible_size, hidden_size, weight_decay_param, sparsity_param, beta, patches) # The number of maximun iterations is set as 400, # which is good enough to get reasonable results. options = {'maxiter': 400, 'disp': True, 'gtol': 1e-5, 'ftol': 2e-9} results = scipy.optimize.minimize(J, theta, method='L-BFGS-B', jac=True, options=options) opt_theta = results['x'] print("Show the results of optimization as following.\n") print(results)
# # (b) Add in the weight decay term (in both the cost function and the derivative # calculations), then re-run Gradient Checking to verify correctness. # # (c) Add in the sparsity penalty term, then re-run Gradient Checking to # verify correctness. # # Feel free to change the training settings when debugging your # code. (For example, reducing the training set size or # number of hidden units may make your code run faster; and setting beta # and/or lambda to zero may be helpful for debugging.) However, in your # final submission of the visualized weights, please use parameters we # gave in Step 0 above. (cost, grad) = sparse_autoencoder.sparse_autoencoder_cost(theta, visible_size, hidden_size, lambda_, sparsity_param, beta, patches) print cost, grad ##====================================================================== ## STEP 3: Gradient Checking # # Hint: If you are debugging your code, performing gradient checking on smaller models # and smaller training sets (e.g., using only 10 training examples and 1-2 hidden # units) may speed things up. # First, lets make sure your numerical gradient computation is correct for a # simple function. After you have implemented computeNumericalGradient.m, # run the following: if debug:
train_images = load_MNIST.load_MNIST_images('data/mnist/train-images-idx3-ubyte') train_labels = load_MNIST.load_MNIST_labels('data/mnist/train-labels-idx1-ubyte') ##====================================================================== ## STEP 2: Train the first sparse autoencoder # This trains the first sparse autoencoder on the unlabelled STL training # images. # If you've correctly implemented sparseAutoencoderCost.m, you don't need # to change anything here. # Randomly initialize the parameters sae1_theta = sparse_autoencoder.initialize(hidden_size_L1, input_size) J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, input_size, hidden_size_L1, lambda_, sparsity_param, beta, train_images) options_ = {'maxiter': 400, 'disp': True} result = scipy.optimize.minimize(J, sae1_theta, method='L-BFGS-B', jac=True, options=options_) sae1_opt_theta = result.x print result ##====================================================================== ## STEP 3: Train the second sparse autoencoder # This trains the second sparse autoencoder on the first autoencoder # featurse. # If you've correctly implemented sparseAutoencoderCost.m, you don't need # to change anything here.
test_labels = labels[test_index] print '# examples in unlabeled set: {0:d}\n'.format(unlabeled_data.shape[1]) print '# examples in supervised training set: {0:d}\n'.format(train_data.shape[1]) print '# examples in supervised testing set: {0:d}\n'.format(test_data.shape[1]) ## ====================================================================== # STEP 2: Train the sparse autoencoder # This trains the sparse autoencoder on the unlabeled training # images. # Randomly initialize the parameters theta = sparse_autoencoder.initialize(hidden_size, input_size) J = lambda x: sparse_autoencoder.sparse_autoencoder_cost(x, input_size, hidden_size, lambda_, sparsity_param, beta, unlabeled_data) options_ = {'maxiter': 400, 'disp': True} result = scipy.optimize.minimize(J, theta, method='L-BFGS-B', jac=True, options=options_) opt_theta = result.x print result # Visualize the weights W1 = opt_theta[0:hidden_size * input_size].reshape(hidden_size, input_size).transpose() display_network.display_network(W1) ##====================================================================== ## STEP 3: Extract Features from the Supervised Dataset #
""" STEP 2: Train the first sparse autoencoder This trains the first sparse autoencoder on the unlabelled STL training images. If you've correctly implemented sparse_autoencoder_cost, you don't need to change anything here. """ # Randomly initialize the parameters sae1_theta = initialize_parameters(hidden_size_L1, input_size) # Instructions: Train the first layer sparse autoencoder, this layer has # an hidden size of "hidden_size_L1" # You should store the optimal parameters in sae1_opt_theta J = lambda theta : sparse_autoencoder_cost(theta, input_size, hidden_size_L1, lambda_, sparsity_param, beta, train_data) options = {'maxiter': maxiter, 'disp': True} results = scipy.optimize.minimize(J, sae1_theta, method='L-BFGS-B', jac=True, options=options) sae1_opt_theta = results['x'] print("Show the results of optimization as following.\n") print(results) # Visualize weights visualize_weights = False if visualize_weights: W1 = sae1_opt_theta[0:hidden_size_L1*input_size].reshape((hidden_size_L1, input_size)) image = display_network(W1.T) plt.figure()
plt.figure() plt.imsave('sparse_autoencoder_minist_patches.png', image, cmap=plt.cm.gray) plt.imshow(image, cmap=plt.cm.gray) visible_size = patches.shape[0] # Number of input units hidden_size = 196 # Number of hidden units weight_decay_param = 3e-3 # Weight decay parameter, which is the lambda in lecture notes beta = 3 # Weight of sparsity penalty term sparsity_param = 0.1 # Desired average activation of the hidden units. # Randomly initialize the fitting parameters theta = initialize_parameters(hidden_size, visible_size) J = lambda theta : sparse_autoencoder_cost(theta, visible_size, hidden_size, weight_decay_param, sparsity_param, beta, patches) # The number of maximun iterations is set as 400, # which is good enough to get reasonable results. options = {'maxiter': 400, 'disp': True, 'gtol': 1e-5, 'ftol': 2e-9} results = scipy.optimize.minimize(J, theta, method='L-BFGS-B', jac=True, options=options) opt_theta = results['x'] print("Show the results of optimization as following.\n") print(results) # Visualization W1 = opt_theta[0:hidden_size*visible_size].reshape((hidden_size, visible_size)) image = display_network(W1.T) plt.figure()