def learn(self):
def f(theta):
return self.sparse_autoencoder(theta)
theta = self.initialize_parameters()
same_theta = theta.copy()
x, f, d = scipy.optimize.fmin_l_bfgs_b(f, theta,
maxfun= self.options.max_iterations,
iprint=1, m=20)
W1, W2, b1, b2 = self.unflatten(x)
utils.save_as_figure(W1.T, "%s/network.png" % self.options.output_dir)
return SparseAutoEncoderSolution(W1, W2, b1, b2)
#!/usr/bin/python
import pickle
import utils
import sac
import sys
if len(sys.argv) != 2:
print "Usage: ./display_saved_network.py somefile.pickle"
sys.exit(1)
fname = sys.argv[1]
f = open(fname, "r")
solution = pickle.load(f)
utils.save_as_figure((solution.W1 + solution.b1).T, "loadedW1.png")
utils.save_as_figure(solution.W2, "loadedW2.png")
images = utils.load_images("data/train-images-idx3-ubyte")
labels = utils.load_labels("data/train-labels-idx1-ubyte")
utils.save_as_figure(images[:, 0:100], "output/input.png")
patches = images[:, 0:10000]
visible_size = 28*28
hidden_size = 196
options = sac.SparseAutoEncoderOptions(visible_size,
hidden_size,
output_dir="output",
#!/usr/bin/python
import utils
import numpy as np
import scipy.optimize
import struct
import sac
import pickle
images = utils.load_images("data/train-images-idx3-ubyte")
labels = utils.load_labels("data/train-labels-idx1-ubyte")
utils.save_as_figure(images[:, 0:100], "output/input.png")
patches = images[:, 0:10000]
visible_size = 28 * 28
hidden_size = 196
options = sac.SparseAutoEncoderOptions(visible_size,
hidden_size,
output_dir="output",
max_iterations=400)
network = sac.SparseAutoEncoder(options, patches)
answer = network.learn()
output = open("network.pickle", "w")
output.write(pickle.dumps(answer))
output.close()
#!/usr/bin/python
import utils
import numpy as np
import scipy.optimize
import struct
import sac
import pickle
images = utils.load_images("data/train-images-idx3-ubyte")
labels = utils.load_labels("data/train-labels-idx1-ubyte")
utils.save_as_figure(images[:, 0:100], "output/input.png")
patches = images[:, 0:10000]
visible_size = 28*28
hidden_size = 196
options = sac.SparseAutoEncoderOptions(visible_size,
hidden_size,
output_dir="output",
max_iterations = 400)
network = sac.SparseAutoEncoder(options, patches)
answer = network.learn()
output = open("network.pickle", "w")
output.write(pickle.dumps(answer))
output.close()
def sparse_autoencoder(self, theta):
visible_size = self.options.visible_size
hidden_size = self.options.hidden_size
lamb = self.options.learning_rate
rho = self.options.sparsity_param
beta = self.options.beta
x = self.data
m = x.shape[1]
W1, W2, b1, b2 = self.unflatten(theta)
ASSERT_SIZE(W1, (hidden_size, visible_size))
ASSERT_SIZE(W2, (visible_size, hidden_size))
ASSERT_SIZE(b1, (hidden_size, 1))
ASSERT_SIZE(b2, (visible_size, 1))
if self.frame_number % 100 == 0:
utils.save_as_figure(W1.T,
"%s/w1frame%03d.png" % (self.options.output_dir,
self.frame_number))
utils.save_as_figure(W2.T,
"%s/w2frame%03d.png" % (self.options.output_dir,
self.frame_number))
self.frame_number += 1
a2, a3 = self.feed_forward(x, W1, W2, b1, b2)
# Compute average activation for an edge over all data
rho_hat = np.mean(a2, 1)[:, np.newaxis]
ASSERT_SIZE(rho_hat, (hidden_size, 1))
kl = rho*np.log(rho/rho_hat) + (1-rho)*np.log((1-rho)/(1-rho_hat))
cost = 0.5/m * np.sum((a3 - x)**2) + \
(lamb/2.)*(np.sum(W1**2) + np.sum(W2**2)) + \
beta*np.sum(kl)
# We set <span class='math'>y</span> equal to the input since we're learning
# an identity function
y = x
delta3 = -(y - a3) * a3*(1-a3)
ASSERT_SIZE(delta3, (visible_size, m))
sparsity = -rho/rho_hat + (1-rho)/(1-rho_hat)
ASSERT_SIZE(sparsity, (hidden_size, 1))
delta2 = (np.dot(W2.T, delta3) + beta * sparsity) * a2 * (1-a2)
ASSERT_SIZE(delta2, (hidden_size, m))
W2_grad = 1./m * np.dot(delta3, a2.T) + lamb * W2
ASSERT_SIZE(W2_grad, (visible_size, hidden_size))
# [:, newaxis] makes this into a matrix
b2_grad = 1./m * np.sum(delta3, 1)[:, np.newaxis]
ASSERT_SIZE(b2_grad, (visible_size, 1))
# sum the rows of delta3 and then mult by 1/m
W1_grad = 1./m * np.dot(delta2, x.T) + lamb * W1
ASSERT_SIZE(W1_grad, (hidden_size, visible_size))
b1_grad = 1./m * np.sum(delta2, 1)[:, np.newaxis]
ASSERT_SIZE(b1_grad, (hidden_size, 1))
grad = self.flatten(W1_grad, W2_grad, b1_grad, b2_grad)
return (cost, grad)
b1 = x[ios:].reshape([output_size, 1], order='F')
ASSERT_NO_NAN(W1)
ASSERT_NO_NAN(b1)
print "W1", W1
print "b1", b1
test_number = 0
test_patch = images[:, test_number]
features, identity = network.feed_forward(test_patch[:,np.newaxis],
solution.W1, solution.W2,
solution.b1, solution.b2)
z2 = np.dot(W1, features) + b1
a2 = sigmoid(z2)
ASSERT_SIZE(a2, (output_size, 1))
utils.save_as_figure(test_patch.T[:,np.newaxis], "output/test_patch.png")
utils.save_as_figure(features.T, "output/features.png")
print np.max(W1)
print "a2", a2
print "theta=", theta
print labels_[test_number]
print np.argmax(a2)
answer = sac.SparseAutoEncoderSolution(W1,None,b1,None)
output = open("recognizer_network.pickle", "w")
output.write(pickle.dumps(answer))
ios = input_size * output_size
W1 = x[0:ios].reshape([output_size, input_size], order="F")
b1 = x[ios:].reshape([output_size, 1], order="F")
ASSERT_NO_NAN(W1)
ASSERT_NO_NAN(b1)
print "W1", W1
print "b1", b1
test_number = 0
test_patch = images[:, test_number]
features, identity = network.feed_forward(test_patch[:, np.newaxis], solution.W1, solution.W2, solution.b1, solution.b2)
z2 = np.dot(W1, features) + b1
a2 = sigmoid(z2)
ASSERT_SIZE(a2, (output_size, 1))
utils.save_as_figure(test_patch.T[:, np.newaxis], "output/test_patch.png")
utils.save_as_figure(features.T, "output/features.png")
print np.max(W1)
print "a2", a2
print "theta=", theta
print labels_[test_number]
print np.argmax(a2)
answer = sac.SparseAutoEncoderSolution(W1, None, b1, None)
output = open("recognizer_network.pickle", "w")
output.write(pickle.dumps(answer))
#!/usr/bin/python
import pickle
import utils
import sac
import sys
if len(sys.argv) != 2:
print "Usage: ./display_saved_network.py somefile.pickle"
sys.exit(1)
fname = sys.argv[1]
f = open(fname, "r")
solution = pickle.load(f)
utils.save_as_figure((solution.W1 + solution.b1).T, "loadedW1.png")
utils.save_as_figure(solution.W2, "loadedW2.png")
images = utils.load_images("data/train-images-idx3-ubyte")
labels = utils.load_labels("data/train-labels-idx1-ubyte")
utils.save_as_figure(images[:, 0:100], "output/input.png")
patches = images[:, 0:10000]
visible_size = 28 * 28
hidden_size = 196
options = sac.SparseAutoEncoderOptions(visible_size,
hidden_size,
output_dir="output",
max_iterations=400)
