num_grad = util.compute_numerical_gradient(cost_func, theta)
num_grad = num_grad.ravel('F')
# Visually compare the gradients side by side
print np.vstack([grad, num_grad]).T
# Compare numerically computed gradients with those computed analytically
diff = linalg.norm(num_grad - grad) / linalg.norm(num_grad + grad);
print(diff)
# === Step 4: Learning parameters ===
#
# Once the gradients are correct, we start training using
# [softmax.train()](softmax.html#section-5).
softmax_model = softmax.train(input_size, num_classes, lamb,
input_data, labels, maxfun=100)
# === Step 5: Testing ===
#
# Test the model against the test images, using
# [softmax.predict()](softmax.html#section-6), which returns
# predictions given a softmax model and the input data.
images = mnist.load_images('../data/t10k-images-idx3-ubyte')
labels = mnist.load_labels('../data/t10k-labels-idx1-ubyte')
input_data = images
if DEBUG:
input_data = input_data[:, :100]
labels = labels[:100]
input_data = np.asfortranarray(input_data[indices, :])
# === Step 4: Train the softmax classifier ===
# Train the sparse autoencoder on the second autoencoder features.
sae2_features = autoencoder.feedforward_autoencoder(sae2_opt_theta,
hidden_size_l2,
hidden_size_l1,
sae1_features)
sae_softmax_theta = 0.005 * np.random.randn(hidden_size_l2 * num_classes, 1)
# Train the softmax classifier. The classifier takes in input of
# dimension `hidden_size_l2` corresponding to the hidden layer size of
# the 2nd layer.
softmax_model = softmax.train(hidden_size_l2,
num_classes,
1e-4,
sae2_features,
train_labels,
maxfun=maxfun)
sae_softmax_opt_theta = softmax_model['opt_theta'].ravel('F')
# === Step 5: Finetune softmax model ===
#
# Implemented in stackedae.cost()
# Initialize the stack using the parameters learned
stack = [util.Empty(), util.Empty()]
W11, W21, b11, b21 = autoencoder.unflatten(sae1_opt_theta, input_size,
hidden_size_l1)
W12, W22, b12, b22 = autoencoder.unflatten(sae2_opt_theta, hidden_size_l1,
hidden_size_l2)
stack[0].w = W11
#!/usr/bin/env python
import sys
sys.path.append('..')
import numpy as np
from library import mnist
from library import softmax
if __name__ == '__main__':
inputSize = 28 * 28
numOfClasses = 10
lamb = 1e-4
trainImages = mnist.load_images('../data/train-images-idx3-ubyte')
trainLabels = mnist.load_labels('../data/train-labels-idx1-ubyte')
softmaxModel = softmax.train(inputSize, numOfClasses, lamb, trainImages, trainLabels, maxfun=100)
testImages = mnist.load_images('../data/t10k-images-idx3-ubyte')
testLabels = mnist.load_labels('../data/t10k-labels-idx1-ubyte')
pred = softmax.predict(softmaxModel, testImages)
acc = (testLabels == pred).mean()
print 'Accuracy: %0.3f' % (acc * 100)
# A = np.transpose(pooledFeaturesTrain, (2, 3, 0, 1)) # A = A[:, :, :, 1] # A = A.reshape([9, 400]) # util.display_network(A) # Train Softmax Classifier softmaxLambda = 1e-4; numOfClasses = 4; inputSize = np.size(pooledFeaturesTrain) / numTrainImages softmaxX = np.transpose(pooledFeaturesTrain, (0, 2, 3, 1)) softmaxX = softmaxX.reshape([inputSize, numTrainImages]) softmaxY = np.int_(trainLabels) - 1 softmaxModel = softmax.train(inputSize, numOfClasses, softmaxLambda, softmaxX, softmaxY, maxfun=400) # Test inputSize = np.size(pooledFeaturesTest) / numTestImages softmaxX = np.transpose(pooledFeaturesTest, (0, 2, 3, 1)) softmaxX = softmaxX.reshape([inputSize, numTestImages]) softmaxY = np.int_(testLabels) - 1 softmaxY = softmaxY.reshape([1, numTestImages]) pred = softmax.predict(softmaxModel, softmaxX) acc = (softmaxY == pred).mean() print 'Accuracy: %0.3f' % (acc * 100)
if DISPLAY:
W11, W21, b11, b21 = autoencoder.unflatten(sae1_opt_theta, input_size, hidden_size_l1)
W12, W22, b12, b22 = autoencoder.unflatten(sae2_opt_theta, hidden_size_l1, hidden_size_l2)
# TODO(zellyn): figure out how to display a 2-level network
# display_network(log(W11' ./ (1-W11')) * W12');
# === Step 4: Train the softmax classifier ===
# Train the sparse autoencoder on the second autoencoder features.
sae2_features = autoencoder.feedforward_autoencoder(sae2_opt_theta, hidden_size_l2, hidden_size_l1, sae1_features)
sae_softmax_theta = 0.005 * np.random.randn(hidden_size_l2 * num_classes, 1)
# Train the softmax classifier. The classifier takes in input of
# dimension `hidden_size_l2` corresponding to the hidden layer size of
# the 2nd layer.
softmax_model = softmax.train(hidden_size_l2, num_classes, 1e-4, sae2_features, train_labels, maxfun=maxfun)
sae_softmax_opt_theta = softmax_model["opt_theta"].ravel("F")
# === Step 5: Finetune softmax model ===
#
# Implemented in stackedae.cost()
# Initialize the stack using the parameters learned
stack = [util.Empty(), util.Empty()]
W11, W21, b11, b21 = autoencoder.unflatten(sae1_opt_theta, input_size, hidden_size_l1)
W12, W22, b12, b22 = autoencoder.unflatten(sae2_opt_theta, hidden_size_l1, hidden_size_l2)
stack[0].w = W11
stack[0].b = b11
stack[1].w = W12
stack[1].b = b12
fn = lambda theta: autoencoder.sparse_autoencoder_loss(
theta, input_size, hidden_size,lamb, sparsity_param, beta, unlabeled_data)
# Find `opt_theta` by running the sparse autoencoder on unlabeled
# training images.
opt_theta, loss, d = (
scipy.optimize.fmin_l_bfgs_b(fn, theta, maxfun=maxfun, iprint=1, m=20))
# Visualize weights
W1, W2, b1, b2 = autoencoder.unflatten(opt_theta, input_size, hidden_size)
util.display_network(W1.T)
# === Step 3: Extract Features from the Supervised Dataset ===
train_features = autoencoder.feedforward_autoencoder(
opt_theta, hidden_size, input_size, train_data)
test_features = autoencoder.feedforward_autoencoder(
opt_theta, hidden_size, input_size, test_data)
# === Step 4: Train the softmax classifier ===
lamb = 1e-4
num_classes = len(set(train_labels))
softmax_model = softmax.train(hidden_size, num_classes, lamb,
train_features, train_labels, maxfun=100)
# === Step 5: Testing ===
#
# Compute Predictions on the test set (testFeatures) using
# `softmax.predict`.
pred = softmax.predict(softmax_model, test_features)
acc = (test_labels == pred).mean()
print 'Accuracy: %0.3f' % (acc * 100)
opts.batchsize = 100 opts.momentun = 0 opts.numepochs = 50 # The cnn setup & training # starttime = datetime.datetime.now() stack = dbn.setup(stack, trainData, opts) stack = dbn.train(stack, trainData, opts) if DISPLAY: util.display_network(stack.layers[1].W.T) util.display_network(stack.layers[2].W.T) trainFeature = dbn.feedforward(stack, trainData) testFeature = dbn.feedforward(stack, testData) lamb = 1e-4 maxfun = 400 softmaxModel = softmax.train(trainFeature.shape[0], numOfClasses, lamb, trainFeature, trainLabels, maxfun=maxfun) pred = softmax.predict(softmaxModel, testFeature) acc = (testLabels == pred).mean() print 'Accuracy: %0.3f' % (acc * 100) # endtime = datetime.datetime.now() # print 'training stackedcnn last time : %s s' % (endtime-starttime)
iprint=1,
m=20))
# Visualize weights
W1, W2, b1, b2 = autoencoder.unflatten(opt_theta, input_size, hidden_size)
util.display_network(W1.T)
# === Step 3: Extract Features from the Supervised Dataset ===
train_features = autoencoder.feedforward_autoencoder(opt_theta, hidden_size,
input_size, train_data)
test_features = autoencoder.feedforward_autoencoder(opt_theta, hidden_size,
input_size, test_data)
# === Step 4: Train the softmax classifier ===
lamb = 1e-4
num_classes = len(set(train_labels))
softmax_model = softmax.train(hidden_size,
num_classes,
lamb,
train_features,
train_labels,
maxfun=100)
# === Step 5: Testing ===
#
# Compute Predictions on the test set (testFeatures) using
# `softmax.predict`.
pred = softmax.predict(softmax_model, test_features)
acc = (test_labels == pred).mean()
print 'Accuracy: %0.3f' % (acc * 100)
fn = lambda theta: autoencoder.sparseAutoencoderCost(theta, hiddenSizeL1, hiddenSizeL2, lamb, sparsityParam, beta, sae1Features)
sae2OptTheta, loss, d = scipy.optimize.fmin_l_bfgs_b(fn, sae2Theta, maxfun=maxfun, iprint=1)
if DISPLAY:
W11, W21, b11, b21 = autoencoder.unflatten(sae1OptTheta, inputSize, hiddenSizeL1)
W12, W22, b12, b22 = autoencoder.unflatten(sae2OptTheta, hiddenSizeL1, hiddenSizeL2)
# figure out how to display a 2-level network
# util.display_network( np.log(W11.T / (1-W11.T)).dot(W12.T) )
sae2Features = autoencoder.feedforwardAutoencoder(sae2OptTheta, hiddenSizeL2, hiddenSizeL1, sae1Features)
# Train the softmax classifier.
saeSoftmaxTheta = 0.005 * np.random.randn(hiddenSizeL2 * numOfClasses, 1)
softmaxModel = softmax.train(hiddenSizeL2, numOfClasses, 1e-4, sae2Features, trainLabels, maxfun=maxfun)
saeSoftmaxOptTheta = softmaxModel['optTheta'].ravel('F')
stack = [util.Empty(), util.Empty()]
W11, W21, b11, b21 = autoencoder.unflatten(sae1OptTheta, inputSize, hiddenSizeL1)
W12, W22, b12, b22 = autoencoder.unflatten(sae2OptTheta, hiddenSizeL1, hiddenSizeL2)
stack[0].w = W11
stack[0].b = b11
stack[1].w = W12
stack[1].b = b12
np.savez("result.npz", W11 = W11, W12 = W12, b11 = b11, b12 = b12, saeSoftmaxOptTheta = saeSoftmaxOptTheta)
stackParams, netconfig = stackedae.stack2params(stack)
stackedaeTheta = np.append(saeSoftmaxOptTheta, stackParams).ravel('F')
