def main(testing=False):
# STEP 0: Parameters
patchsize = 8
num_patches = testing and 10 or 10000
visible_size = patchsize * patchsize
hidden_size = testing and 3 or 25
target_activation = 0.01
lamb = 0.0001
beta = 3
# STEP 1: Get data
patches = sampleIMAGES(patchsize, num_patches)
util.display_network(patches[:, np.random.randint(0, num_patches, 200)])
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
# STEP 2: Implement sparseAutoencoderLoss
def sal(theta):
return autoencoder.sparse_autoencoder_loss(theta, visible_size,
hidden_size, lamb,
target_activation, beta,
patches)
loss, grad = sal(theta)
# STEP 3: Gradient Checking
if testing:
numgrad = util.compute_numerical_gradient(lambda x: sal(x)[0], theta)
# Eyeball the gradients
print np.hstack([numgrad, grad])
diff = linalg.norm(numgrad - grad) / linalg.norm(numgrad + grad)
print "Normed difference: %f" % diff
# STEP 4: Run sparse_autoencoder_loss with L-BFGS
# Initialize random theta
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
print "Starting..."
x, f, d = scipy.optimize.fmin_l_bfgs_b(sal,
theta,
maxfun=400,
iprint=25,
m=20)
print "Done"
print x
print f
print d
W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size)
print "W1.shape=%s" % (W1.shape, )
util.display_network(W1.T)
def main(testing=False):
# STEP 0: Parameters
patchsize = 8
num_patches = testing and 10 or 10000
visible_size = patchsize * patchsize
hidden_size = testing and 3 or 25
target_activation = 0.01
lamb = 0.0001
beta = 3
# STEP 1: Get data
patches = sampleIMAGES(patchsize, num_patches)
util.display_network(patches[:,np.random.randint(0, num_patches, 200)])
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
# STEP 2: Implement sparseAutoencoderLoss
def sal(theta):
return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb,
target_activation, beta, patches)
loss, grad = sal(theta)
# STEP 3: Gradient Checking
if testing:
numgrad = util.compute_numerical_gradient(lambda x: sal(x)[0], theta)
# Eyeball the gradients
print np.hstack([numgrad, grad])
diff = linalg.norm(numgrad-grad) / linalg.norm(numgrad+grad)
print "Normed difference: %f" % diff
# STEP 4: Run sparse_autoencoder_loss with L-BFGS
# Initialize random theta
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
print "Starting..."
x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=25, m=20)
print "Done"
print x
print f
print d
W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size)
print "W1.shape=%s" % (W1.shape,)
util.display_network(W1.T)
def main(testing=True):
images = mnist.load_images('../data/train-images-idx3-ubyte') # 784 x 60000
labels = mnist.load_labels('../data/train-labels-idx1-ubyte') # 60000 x 1
util.display_network(images[:,0:100]) # Show the first 100 images
visible_size = 28*28
hidden_size = 196
sparsity_param = 0.1
lamb = 3e-3
beta = 3
patches = images[:,0:10000]
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
def sal(theta):
return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb,
sparsity_param, beta, patches)
x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=1, m=20)
W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size)
util.display_network(W1.T)
# inputSize, hiddenSizeL1, ...
# lambda, sparsityParam, ...
# beta, trainData), ...
# sae1Theta, options);
fn = lambda theta: autoencoder.sparse_autoencoder_loss(
theta, input_size, hidden_size_l1, lamb, sparsity_param, beta, train_data)
sae1_opt_theta, loss, d = (scipy.optimize.fmin_l_bfgs_b(fn,
sae1_theta,
maxfun=maxfun,
iprint=1))
if DISPLAY:
W1, W2, b1, b2 = autoencoder.unflatten(sae1_opt_theta, input_size,
hidden_size_l1)
util.display_network(W1.T)
# === Step 3: Train the second sparse autoencoder ===
#
# Train the second sparse autoencoder on the first autoencoder features.
sae1_features = autoencoder.feedforward_autoencoder(sae1_opt_theta,
hidden_size_l1, input_size,
train_data)
# Randomly initialize the parameters
sae2_theta = autoencoder.initialize_parameters(hidden_size_l2, hidden_size_l1)
fn = lambda theta: autoencoder.sparse_autoencoder_loss(
theta, hidden_size_l1, hidden_size_l2, lamb, sparsity_param, beta,
sae1_features)
sae2_opt_theta, loss, d = (scipy.optimize.fmin_l_bfgs_b(fn,
plt.figure()
plt.imshow(covar)
plt.title('Visualization of covariance matrix')
plt.show()
# STEP 2: Find k, the number of components to retain
SD = S
k = SD[(np.cumsum(SD) / np.sum(SD)) < 0.99].size
# STEP 3: Implement PCA with dimension reduction
xRot = U[:,0:k].T.dot(x)
xHat = U[:,0:k].dot(xRot)
util.display_network(x[:, randsel], 'Raw images', show=False)
util.display_network(xHat[:, randsel], 'PCA processed images')
# STEP 4a: Implement PCA with whitening and regularisation
epsilon = 0.1
xPCAWhite = np.diag(1.0 / np.sqrt(S + epsilon)).dot(U.T).dot(x)
# STEP 4b: Check your implementation of PCA whitening
covar = xPCAWhite.dot(xPCAWhite.T) / n
plt.figure()
plt.imshow(covar)
plt.title('Visualization of covariance matrix')
plt.show()
#!/usr/bin/env python
import sys
sys.path.append('..')
import numpy as np
from library import stl10
from library import softmax
from library import autoencoder
from library import util
import scipy.optimize
if __name__ == '__main__':
r = np.load("result.npz")
W11 = r["W11"]
print W11.shape
util.display_network(W11.T)
# print W12.shape
# util.display_network(W12.T)
stack.layers[0].size = inputSize stack.layers[1].type = 'hidden' stack.layers[1].size = 50 stack.layers[2].type = 'output' stack.layers[2].size = numOfClasses opts = util.Empty() opts.alpha = 0.1 opts.batchsize = 100 opts.momentun = 0 opts.numepochs = 10 # The nn setup & training stack = nn.setup(stack) stack = nn.train(stack, trainData, trainLabels, opts) if DISPLAY: util.display_network(stack.layers[1].W.T) pred = nn.predict(stack, testData) print pred print testLabels acc = (testLabels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
y = np.random.randint(width - patchSize + 1) x = np.random.randint(height - patchSize + 1) sample = IMAGES[y:y+patchSize, x:x+patchSize, idx] patches[:, p] = np.array(sample).flatten() p = p + 1 return patches if __name__ == '__main__': # Step 0a: Load data x = sampleIMAGESRAW() inputSize, numOfSamples = x.shape randsel = np.random.randint(0, numOfSamples, 200) util.display_network(x[:, randsel], 'Raw data') # Step 0b: Zero-mean the data (by column) avg = np.mean(x, axis = 0) x = x - avg # Step 1a: Implement PCA to obtain xRot sigma = x.dot(x.T) / numOfSamples U, S, V = np.linalg.svd(sigma) xRot = U.T.dot(x) # Step 1b: Check your implementation of PCA covar = xRot.dot(xRot.T) / numOfSamples plt.figure() plt.imshow(covar)