num_grad = util.compute_numerical_gradient(cost_func, theta) num_grad = num_grad.ravel('F') # Visually compare the gradients side by side print np.vstack([grad, num_grad]).T # Compare numerically computed gradients with those computed analytically diff = linalg.norm(num_grad - grad) / linalg.norm(num_grad + grad); print(diff) # === Step 4: Learning parameters === # # Once the gradients are correct, we start training using # [softmax.train()](softmax.html#section-5). softmax_model = softmax.train(input_size, num_classes, lamb, input_data, labels, maxfun=100) # === Step 5: Testing === # # Test the model against the test images, using # [softmax.predict()](softmax.html#section-6), which returns # predictions given a softmax model and the input data. images = mnist.load_images('../data/t10k-images-idx3-ubyte') labels = mnist.load_labels('../data/t10k-labels-idx1-ubyte') input_data = images if DEBUG: input_data = input_data[:, :100] labels = labels[:100] input_data = np.asfortranarray(input_data[indices, :])
# === Step 4: Train the softmax classifier === # Train the sparse autoencoder on the second autoencoder features. sae2_features = autoencoder.feedforward_autoencoder(sae2_opt_theta, hidden_size_l2, hidden_size_l1, sae1_features) sae_softmax_theta = 0.005 * np.random.randn(hidden_size_l2 * num_classes, 1) # Train the softmax classifier. The classifier takes in input of # dimension `hidden_size_l2` corresponding to the hidden layer size of # the 2nd layer. softmax_model = softmax.train(hidden_size_l2, num_classes, 1e-4, sae2_features, train_labels, maxfun=maxfun) sae_softmax_opt_theta = softmax_model['opt_theta'].ravel('F') # === Step 5: Finetune softmax model === # # Implemented in stackedae.cost() # Initialize the stack using the parameters learned stack = [util.Empty(), util.Empty()] W11, W21, b11, b21 = autoencoder.unflatten(sae1_opt_theta, input_size, hidden_size_l1) W12, W22, b12, b22 = autoencoder.unflatten(sae2_opt_theta, hidden_size_l1, hidden_size_l2) stack[0].w = W11
#!/usr/bin/env python import sys sys.path.append('..') import numpy as np from library import mnist from library import softmax if __name__ == '__main__': inputSize = 28 * 28 numOfClasses = 10 lamb = 1e-4 trainImages = mnist.load_images('../data/train-images-idx3-ubyte') trainLabels = mnist.load_labels('../data/train-labels-idx1-ubyte') softmaxModel = softmax.train(inputSize, numOfClasses, lamb, trainImages, trainLabels, maxfun=100) testImages = mnist.load_images('../data/t10k-images-idx3-ubyte') testLabels = mnist.load_labels('../data/t10k-labels-idx1-ubyte') pred = softmax.predict(softmaxModel, testImages) acc = (testLabels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
# A = np.transpose(pooledFeaturesTrain, (2, 3, 0, 1)) # A = A[:, :, :, 1] # A = A.reshape([9, 400]) # util.display_network(A) # Train Softmax Classifier softmaxLambda = 1e-4; numOfClasses = 4; inputSize = np.size(pooledFeaturesTrain) / numTrainImages softmaxX = np.transpose(pooledFeaturesTrain, (0, 2, 3, 1)) softmaxX = softmaxX.reshape([inputSize, numTrainImages]) softmaxY = np.int_(trainLabels) - 1 softmaxModel = softmax.train(inputSize, numOfClasses, softmaxLambda, softmaxX, softmaxY, maxfun=400) # Test inputSize = np.size(pooledFeaturesTest) / numTestImages softmaxX = np.transpose(pooledFeaturesTest, (0, 2, 3, 1)) softmaxX = softmaxX.reshape([inputSize, numTestImages]) softmaxY = np.int_(testLabels) - 1 softmaxY = softmaxY.reshape([1, numTestImages]) pred = softmax.predict(softmaxModel, softmaxX) acc = (softmaxY == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
if DISPLAY: W11, W21, b11, b21 = autoencoder.unflatten(sae1_opt_theta, input_size, hidden_size_l1) W12, W22, b12, b22 = autoencoder.unflatten(sae2_opt_theta, hidden_size_l1, hidden_size_l2) # TODO(zellyn): figure out how to display a 2-level network # display_network(log(W11' ./ (1-W11')) * W12'); # === Step 4: Train the softmax classifier === # Train the sparse autoencoder on the second autoencoder features. sae2_features = autoencoder.feedforward_autoencoder(sae2_opt_theta, hidden_size_l2, hidden_size_l1, sae1_features) sae_softmax_theta = 0.005 * np.random.randn(hidden_size_l2 * num_classes, 1) # Train the softmax classifier. The classifier takes in input of # dimension `hidden_size_l2` corresponding to the hidden layer size of # the 2nd layer. softmax_model = softmax.train(hidden_size_l2, num_classes, 1e-4, sae2_features, train_labels, maxfun=maxfun) sae_softmax_opt_theta = softmax_model["opt_theta"].ravel("F") # === Step 5: Finetune softmax model === # # Implemented in stackedae.cost() # Initialize the stack using the parameters learned stack = [util.Empty(), util.Empty()] W11, W21, b11, b21 = autoencoder.unflatten(sae1_opt_theta, input_size, hidden_size_l1) W12, W22, b12, b22 = autoencoder.unflatten(sae2_opt_theta, hidden_size_l1, hidden_size_l2) stack[0].w = W11 stack[0].b = b11 stack[1].w = W12 stack[1].b = b12
fn = lambda theta: autoencoder.sparse_autoencoder_loss( theta, input_size, hidden_size,lamb, sparsity_param, beta, unlabeled_data) # Find `opt_theta` by running the sparse autoencoder on unlabeled # training images. opt_theta, loss, d = ( scipy.optimize.fmin_l_bfgs_b(fn, theta, maxfun=maxfun, iprint=1, m=20)) # Visualize weights W1, W2, b1, b2 = autoencoder.unflatten(opt_theta, input_size, hidden_size) util.display_network(W1.T) # === Step 3: Extract Features from the Supervised Dataset === train_features = autoencoder.feedforward_autoencoder( opt_theta, hidden_size, input_size, train_data) test_features = autoencoder.feedforward_autoencoder( opt_theta, hidden_size, input_size, test_data) # === Step 4: Train the softmax classifier === lamb = 1e-4 num_classes = len(set(train_labels)) softmax_model = softmax.train(hidden_size, num_classes, lamb, train_features, train_labels, maxfun=100) # === Step 5: Testing === # # Compute Predictions on the test set (testFeatures) using # `softmax.predict`. pred = softmax.predict(softmax_model, test_features) acc = (test_labels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
opts.batchsize = 100 opts.momentun = 0 opts.numepochs = 50 # The cnn setup & training # starttime = datetime.datetime.now() stack = dbn.setup(stack, trainData, opts) stack = dbn.train(stack, trainData, opts) if DISPLAY: util.display_network(stack.layers[1].W.T) util.display_network(stack.layers[2].W.T) trainFeature = dbn.feedforward(stack, trainData) testFeature = dbn.feedforward(stack, testData) lamb = 1e-4 maxfun = 400 softmaxModel = softmax.train(trainFeature.shape[0], numOfClasses, lamb, trainFeature, trainLabels, maxfun=maxfun) pred = softmax.predict(softmaxModel, testFeature) acc = (testLabels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100) # endtime = datetime.datetime.now() # print 'training stackedcnn last time : %s s' % (endtime-starttime)
iprint=1, m=20)) # Visualize weights W1, W2, b1, b2 = autoencoder.unflatten(opt_theta, input_size, hidden_size) util.display_network(W1.T) # === Step 3: Extract Features from the Supervised Dataset === train_features = autoencoder.feedforward_autoencoder(opt_theta, hidden_size, input_size, train_data) test_features = autoencoder.feedforward_autoencoder(opt_theta, hidden_size, input_size, test_data) # === Step 4: Train the softmax classifier === lamb = 1e-4 num_classes = len(set(train_labels)) softmax_model = softmax.train(hidden_size, num_classes, lamb, train_features, train_labels, maxfun=100) # === Step 5: Testing === # # Compute Predictions on the test set (testFeatures) using # `softmax.predict`. pred = softmax.predict(softmax_model, test_features) acc = (test_labels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
fn = lambda theta: autoencoder.sparseAutoencoderCost(theta, hiddenSizeL1, hiddenSizeL2, lamb, sparsityParam, beta, sae1Features) sae2OptTheta, loss, d = scipy.optimize.fmin_l_bfgs_b(fn, sae2Theta, maxfun=maxfun, iprint=1) if DISPLAY: W11, W21, b11, b21 = autoencoder.unflatten(sae1OptTheta, inputSize, hiddenSizeL1) W12, W22, b12, b22 = autoencoder.unflatten(sae2OptTheta, hiddenSizeL1, hiddenSizeL2) # figure out how to display a 2-level network # util.display_network( np.log(W11.T / (1-W11.T)).dot(W12.T) ) sae2Features = autoencoder.feedforwardAutoencoder(sae2OptTheta, hiddenSizeL2, hiddenSizeL1, sae1Features) # Train the softmax classifier. saeSoftmaxTheta = 0.005 * np.random.randn(hiddenSizeL2 * numOfClasses, 1) softmaxModel = softmax.train(hiddenSizeL2, numOfClasses, 1e-4, sae2Features, trainLabels, maxfun=maxfun) saeSoftmaxOptTheta = softmaxModel['optTheta'].ravel('F') stack = [util.Empty(), util.Empty()] W11, W21, b11, b21 = autoencoder.unflatten(sae1OptTheta, inputSize, hiddenSizeL1) W12, W22, b12, b22 = autoencoder.unflatten(sae2OptTheta, hiddenSizeL1, hiddenSizeL2) stack[0].w = W11 stack[0].b = b11 stack[1].w = W12 stack[1].b = b12 np.savez("result.npz", W11 = W11, W12 = W12, b11 = b11, b12 = b12, saeSoftmaxOptTheta = saeSoftmaxOptTheta) stackParams, netconfig = stackedae.stack2params(stack) stackedaeTheta = np.append(saeSoftmaxOptTheta, stackParams).ravel('F')