def learn(self): def f(theta): return self.sparse_autoencoder(theta) theta = self.initialize_parameters() same_theta = theta.copy() x, f, d = scipy.optimize.fmin_l_bfgs_b(f, theta, maxfun= self.options.max_iterations, iprint=1, m=20) W1, W2, b1, b2 = self.unflatten(x) utils.save_as_figure(W1.T, "%s/network.png" % self.options.output_dir) return SparseAutoEncoderSolution(W1, W2, b1, b2)
#!/usr/bin/python import pickle import utils import sac import sys if len(sys.argv) != 2: print "Usage: ./display_saved_network.py somefile.pickle" sys.exit(1) fname = sys.argv[1] f = open(fname, "r") solution = pickle.load(f) utils.save_as_figure((solution.W1 + solution.b1).T, "loadedW1.png") utils.save_as_figure(solution.W2, "loadedW2.png") images = utils.load_images("data/train-images-idx3-ubyte") labels = utils.load_labels("data/train-labels-idx1-ubyte") utils.save_as_figure(images[:, 0:100], "output/input.png") patches = images[:, 0:10000] visible_size = 28*28 hidden_size = 196 options = sac.SparseAutoEncoderOptions(visible_size, hidden_size, output_dir="output",
#!/usr/bin/python import utils import numpy as np import scipy.optimize import struct import sac import pickle images = utils.load_images("data/train-images-idx3-ubyte") labels = utils.load_labels("data/train-labels-idx1-ubyte") utils.save_as_figure(images[:, 0:100], "output/input.png") patches = images[:, 0:10000] visible_size = 28 * 28 hidden_size = 196 options = sac.SparseAutoEncoderOptions(visible_size, hidden_size, output_dir="output", max_iterations=400) network = sac.SparseAutoEncoder(options, patches) answer = network.learn() output = open("network.pickle", "w") output.write(pickle.dumps(answer)) output.close()
#!/usr/bin/python import utils import numpy as np import scipy.optimize import struct import sac import pickle images = utils.load_images("data/train-images-idx3-ubyte") labels = utils.load_labels("data/train-labels-idx1-ubyte") utils.save_as_figure(images[:, 0:100], "output/input.png") patches = images[:, 0:10000] visible_size = 28*28 hidden_size = 196 options = sac.SparseAutoEncoderOptions(visible_size, hidden_size, output_dir="output", max_iterations = 400) network = sac.SparseAutoEncoder(options, patches) answer = network.learn() output = open("network.pickle", "w") output.write(pickle.dumps(answer)) output.close()
def sparse_autoencoder(self, theta): visible_size = self.options.visible_size hidden_size = self.options.hidden_size lamb = self.options.learning_rate rho = self.options.sparsity_param beta = self.options.beta x = self.data m = x.shape[1] W1, W2, b1, b2 = self.unflatten(theta) ASSERT_SIZE(W1, (hidden_size, visible_size)) ASSERT_SIZE(W2, (visible_size, hidden_size)) ASSERT_SIZE(b1, (hidden_size, 1)) ASSERT_SIZE(b2, (visible_size, 1)) if self.frame_number % 100 == 0: utils.save_as_figure(W1.T, "%s/w1frame%03d.png" % (self.options.output_dir, self.frame_number)) utils.save_as_figure(W2.T, "%s/w2frame%03d.png" % (self.options.output_dir, self.frame_number)) self.frame_number += 1 a2, a3 = self.feed_forward(x, W1, W2, b1, b2) # Compute average activation for an edge over all data rho_hat = np.mean(a2, 1)[:, np.newaxis] ASSERT_SIZE(rho_hat, (hidden_size, 1)) kl = rho*np.log(rho/rho_hat) + (1-rho)*np.log((1-rho)/(1-rho_hat)) cost = 0.5/m * np.sum((a3 - x)**2) + \ (lamb/2.)*(np.sum(W1**2) + np.sum(W2**2)) + \ beta*np.sum(kl) # We set <span class='math'>y</span> equal to the input since we're learning # an identity function y = x delta3 = -(y - a3) * a3*(1-a3) ASSERT_SIZE(delta3, (visible_size, m)) sparsity = -rho/rho_hat + (1-rho)/(1-rho_hat) ASSERT_SIZE(sparsity, (hidden_size, 1)) delta2 = (np.dot(W2.T, delta3) + beta * sparsity) * a2 * (1-a2) ASSERT_SIZE(delta2, (hidden_size, m)) W2_grad = 1./m * np.dot(delta3, a2.T) + lamb * W2 ASSERT_SIZE(W2_grad, (visible_size, hidden_size)) # [:, newaxis] makes this into a matrix b2_grad = 1./m * np.sum(delta3, 1)[:, np.newaxis] ASSERT_SIZE(b2_grad, (visible_size, 1)) # sum the rows of delta3 and then mult by 1/m W1_grad = 1./m * np.dot(delta2, x.T) + lamb * W1 ASSERT_SIZE(W1_grad, (hidden_size, visible_size)) b1_grad = 1./m * np.sum(delta2, 1)[:, np.newaxis] ASSERT_SIZE(b1_grad, (hidden_size, 1)) grad = self.flatten(W1_grad, W2_grad, b1_grad, b2_grad) return (cost, grad)
b1 = x[ios:].reshape([output_size, 1], order='F') ASSERT_NO_NAN(W1) ASSERT_NO_NAN(b1) print "W1", W1 print "b1", b1 test_number = 0 test_patch = images[:, test_number] features, identity = network.feed_forward(test_patch[:,np.newaxis], solution.W1, solution.W2, solution.b1, solution.b2) z2 = np.dot(W1, features) + b1 a2 = sigmoid(z2) ASSERT_SIZE(a2, (output_size, 1)) utils.save_as_figure(test_patch.T[:,np.newaxis], "output/test_patch.png") utils.save_as_figure(features.T, "output/features.png") print np.max(W1) print "a2", a2 print "theta=", theta print labels_[test_number] print np.argmax(a2) answer = sac.SparseAutoEncoderSolution(W1,None,b1,None) output = open("recognizer_network.pickle", "w") output.write(pickle.dumps(answer))
ios = input_size * output_size W1 = x[0:ios].reshape([output_size, input_size], order="F") b1 = x[ios:].reshape([output_size, 1], order="F") ASSERT_NO_NAN(W1) ASSERT_NO_NAN(b1) print "W1", W1 print "b1", b1 test_number = 0 test_patch = images[:, test_number] features, identity = network.feed_forward(test_patch[:, np.newaxis], solution.W1, solution.W2, solution.b1, solution.b2) z2 = np.dot(W1, features) + b1 a2 = sigmoid(z2) ASSERT_SIZE(a2, (output_size, 1)) utils.save_as_figure(test_patch.T[:, np.newaxis], "output/test_patch.png") utils.save_as_figure(features.T, "output/features.png") print np.max(W1) print "a2", a2 print "theta=", theta print labels_[test_number] print np.argmax(a2) answer = sac.SparseAutoEncoderSolution(W1, None, b1, None) output = open("recognizer_network.pickle", "w") output.write(pickle.dumps(answer))
#!/usr/bin/python import pickle import utils import sac import sys if len(sys.argv) != 2: print "Usage: ./display_saved_network.py somefile.pickle" sys.exit(1) fname = sys.argv[1] f = open(fname, "r") solution = pickle.load(f) utils.save_as_figure((solution.W1 + solution.b1).T, "loadedW1.png") utils.save_as_figure(solution.W2, "loadedW2.png") images = utils.load_images("data/train-images-idx3-ubyte") labels = utils.load_labels("data/train-labels-idx1-ubyte") utils.save_as_figure(images[:, 0:100], "output/input.png") patches = images[:, 0:10000] visible_size = 28 * 28 hidden_size = 196 options = sac.SparseAutoEncoderOptions(visible_size, hidden_size, output_dir="output", max_iterations=400)