def main(testing=False): # STEP 0: Parameters patchsize = 8 num_patches = testing and 10 or 10000 visible_size = patchsize * patchsize hidden_size = testing and 3 or 25 target_activation = 0.01 lamb = 0.0001 beta = 3 # STEP 1: Get data patches = sampleIMAGES(patchsize, num_patches) util.display_network(patches[:, np.random.randint(0, num_patches, 200)]) theta = autoencoder.initialize_parameters(hidden_size, visible_size) # STEP 2: Implement sparseAutoencoderLoss def sal(theta): return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb, target_activation, beta, patches) loss, grad = sal(theta) # STEP 3: Gradient Checking if testing: numgrad = util.compute_numerical_gradient(lambda x: sal(x)[0], theta) # Eyeball the gradients print np.hstack([numgrad, grad]) diff = linalg.norm(numgrad - grad) / linalg.norm(numgrad + grad) print "Normed difference: %f" % diff # STEP 4: Run sparse_autoencoder_loss with L-BFGS # Initialize random theta theta = autoencoder.initialize_parameters(hidden_size, visible_size) print "Starting..." x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=25, m=20) print "Done" print x print f print d W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size) print "W1.shape=%s" % (W1.shape, ) util.display_network(W1.T)
def main(testing=False): # STEP 0: Parameters patchsize = 8 num_patches = testing and 10 or 10000 visible_size = patchsize * patchsize hidden_size = testing and 3 or 25 target_activation = 0.01 lamb = 0.0001 beta = 3 # STEP 1: Get data patches = sampleIMAGES(patchsize, num_patches) util.display_network(patches[:,np.random.randint(0, num_patches, 200)]) theta = autoencoder.initialize_parameters(hidden_size, visible_size) # STEP 2: Implement sparseAutoencoderLoss def sal(theta): return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb, target_activation, beta, patches) loss, grad = sal(theta) # STEP 3: Gradient Checking if testing: numgrad = util.compute_numerical_gradient(lambda x: sal(x)[0], theta) # Eyeball the gradients print np.hstack([numgrad, grad]) diff = linalg.norm(numgrad-grad) / linalg.norm(numgrad+grad) print "Normed difference: %f" % diff # STEP 4: Run sparse_autoencoder_loss with L-BFGS # Initialize random theta theta = autoencoder.initialize_parameters(hidden_size, visible_size) print "Starting..." x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=25, m=20) print "Done" print x print f print d W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size) print "W1.shape=%s" % (W1.shape,) util.display_network(W1.T)
def main(testing=True): images = mnist.load_images('../data/train-images-idx3-ubyte') # 784 x 60000 labels = mnist.load_labels('../data/train-labels-idx1-ubyte') # 60000 x 1 util.display_network(images[:,0:100]) # Show the first 100 images visible_size = 28*28 hidden_size = 196 sparsity_param = 0.1 lamb = 3e-3 beta = 3 patches = images[:,0:10000] theta = autoencoder.initialize_parameters(hidden_size, visible_size) def sal(theta): return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb, sparsity_param, beta, patches) x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=1, m=20) W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size) util.display_network(W1.T)
# inputSize, hiddenSizeL1, ... # lambda, sparsityParam, ... # beta, trainData), ... # sae1Theta, options); fn = lambda theta: autoencoder.sparse_autoencoder_loss( theta, input_size, hidden_size_l1, lamb, sparsity_param, beta, train_data) sae1_opt_theta, loss, d = (scipy.optimize.fmin_l_bfgs_b(fn, sae1_theta, maxfun=maxfun, iprint=1)) if DISPLAY: W1, W2, b1, b2 = autoencoder.unflatten(sae1_opt_theta, input_size, hidden_size_l1) util.display_network(W1.T) # === Step 3: Train the second sparse autoencoder === # # Train the second sparse autoencoder on the first autoencoder features. sae1_features = autoencoder.feedforward_autoencoder(sae1_opt_theta, hidden_size_l1, input_size, train_data) # Randomly initialize the parameters sae2_theta = autoencoder.initialize_parameters(hidden_size_l2, hidden_size_l1) fn = lambda theta: autoencoder.sparse_autoencoder_loss( theta, hidden_size_l1, hidden_size_l2, lamb, sparsity_param, beta, sae1_features) sae2_opt_theta, loss, d = (scipy.optimize.fmin_l_bfgs_b(fn,
plt.figure() plt.imshow(covar) plt.title('Visualization of covariance matrix') plt.show() # STEP 2: Find k, the number of components to retain SD = S k = SD[(np.cumsum(SD) / np.sum(SD)) < 0.99].size # STEP 3: Implement PCA with dimension reduction xRot = U[:,0:k].T.dot(x) xHat = U[:,0:k].dot(xRot) util.display_network(x[:, randsel], 'Raw images', show=False) util.display_network(xHat[:, randsel], 'PCA processed images') # STEP 4a: Implement PCA with whitening and regularisation epsilon = 0.1 xPCAWhite = np.diag(1.0 / np.sqrt(S + epsilon)).dot(U.T).dot(x) # STEP 4b: Check your implementation of PCA whitening covar = xPCAWhite.dot(xPCAWhite.T) / n plt.figure() plt.imshow(covar) plt.title('Visualization of covariance matrix') plt.show()
#!/usr/bin/env python import sys sys.path.append('..') import numpy as np from library import stl10 from library import softmax from library import autoencoder from library import util import scipy.optimize if __name__ == '__main__': r = np.load("result.npz") W11 = r["W11"] print W11.shape util.display_network(W11.T) # print W12.shape # util.display_network(W12.T)
stack.layers[0].size = inputSize stack.layers[1].type = 'hidden' stack.layers[1].size = 50 stack.layers[2].type = 'output' stack.layers[2].size = numOfClasses opts = util.Empty() opts.alpha = 0.1 opts.batchsize = 100 opts.momentun = 0 opts.numepochs = 10 # The nn setup & training stack = nn.setup(stack) stack = nn.train(stack, trainData, trainLabels, opts) if DISPLAY: util.display_network(stack.layers[1].W.T) pred = nn.predict(stack, testData) print pred print testLabels acc = (testLabels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
y = np.random.randint(width - patchSize + 1) x = np.random.randint(height - patchSize + 1) sample = IMAGES[y:y+patchSize, x:x+patchSize, idx] patches[:, p] = np.array(sample).flatten() p = p + 1 return patches if __name__ == '__main__': # Step 0a: Load data x = sampleIMAGESRAW() inputSize, numOfSamples = x.shape randsel = np.random.randint(0, numOfSamples, 200) util.display_network(x[:, randsel], 'Raw data') # Step 0b: Zero-mean the data (by column) avg = np.mean(x, axis = 0) x = x - avg # Step 1a: Implement PCA to obtain xRot sigma = x.dot(x.T) / numOfSamples U, S, V = np.linalg.svd(sigma) xRot = U.T.dot(x) # Step 1b: Check your implementation of PCA covar = xRot.dot(xRot.T) / numOfSamples plt.figure() plt.imshow(covar)