def main(testing=False):
# STEP 0: Parameters
patchsize = 8
num_patches = testing and 10 or 10000
visible_size = patchsize * patchsize
hidden_size = testing and 3 or 25
target_activation = 0.01
lamb = 0.0001
beta = 3
# STEP 1: Get data
patches = sampleIMAGES(patchsize, num_patches)
util.display_network(patches[:, np.random.randint(0, num_patches, 200)])
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
# STEP 2: Implement sparseAutoencoderLoss
def sal(theta):
return autoencoder.sparse_autoencoder_loss(theta, visible_size,
hidden_size, lamb,
target_activation, beta,
patches)
loss, grad = sal(theta)
# STEP 3: Gradient Checking
if testing:
numgrad = util.compute_numerical_gradient(lambda x: sal(x)[0], theta)
# Eyeball the gradients
print np.hstack([numgrad, grad])
diff = linalg.norm(numgrad - grad) / linalg.norm(numgrad + grad)
print "Normed difference: %f" % diff
# STEP 4: Run sparse_autoencoder_loss with L-BFGS
# Initialize random theta
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
print "Starting..."
x, f, d = scipy.optimize.fmin_l_bfgs_b(sal,
theta,
maxfun=400,
iprint=25,
m=20)
print "Done"
print x
print f
print d
W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size)
print "W1.shape=%s" % (W1.shape, )
util.display_network(W1.T)
def main(testing=False):
# STEP 0: Parameters
patchsize = 8
num_patches = testing and 10 or 10000
visible_size = patchsize * patchsize
hidden_size = testing and 3 or 25
target_activation = 0.01
lamb = 0.0001
beta = 3
# STEP 1: Get data
patches = sampleIMAGES(patchsize, num_patches)
util.display_network(patches[:,np.random.randint(0, num_patches, 200)])
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
# STEP 2: Implement sparseAutoencoderLoss
def sal(theta):
return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb,
target_activation, beta, patches)
loss, grad = sal(theta)
# STEP 3: Gradient Checking
if testing:
numgrad = util.compute_numerical_gradient(lambda x: sal(x)[0], theta)
# Eyeball the gradients
print np.hstack([numgrad, grad])
diff = linalg.norm(numgrad-grad) / linalg.norm(numgrad+grad)
print "Normed difference: %f" % diff
# STEP 4: Run sparse_autoencoder_loss with L-BFGS
# Initialize random theta
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
print "Starting..."
x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=25, m=20)
print "Done"
print x
print f
print d
W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size)
print "W1.shape=%s" % (W1.shape,)
util.display_network(W1.T)
def main(testing=True):
images = mnist.load_images('../data/train-images-idx3-ubyte') # 784 x 60000
labels = mnist.load_labels('../data/train-labels-idx1-ubyte') # 60000 x 1
util.display_network(images[:,0:100]) # Show the first 100 images
visible_size = 28*28
hidden_size = 196
sparsity_param = 0.1
lamb = 3e-3
beta = 3
patches = images[:,0:10000]
theta = autoencoder.initialize_parameters(hidden_size, visible_size)
def sal(theta):
return autoencoder.sparse_autoencoder_loss(theta, visible_size, hidden_size, lamb,
sparsity_param, beta, patches)
x, f, d = scipy.optimize.fmin_l_bfgs_b(sal, theta, maxfun=400, iprint=1, m=20)
W1, W2, b1, b2 = autoencoder.unflatten(x, visible_size, hidden_size)
util.display_network(W1.T)
if DEBUG:
input_size = 64
# only 100 datapoints
train_data = train_data[:, :100]
train_labels = train_labels[:100]
# only top input_size most-varying input elements (pixels)
indices = train_data.var(1).argsort()[-input_size:]
train_data = np.asfortranarray(train_data[indices, :])
# === Step 2: Train the first sparse autoencoder ===
#
# Train the first sparse autoencoder on the unlabelled STL training
# images.
# Randomly initialize the parameters
sae1_theta = autoencoder.initialize_parameters(hidden_size_l1, input_size)
# Train the first layer sparse autoencoder. This layer has a hidden
# size of `hidden_size_l1`.
# sae1_opt_theta, loss = minFunc( @(p) sparseAutoencoderLoss(p, ...
# inputSize, hiddenSizeL1, ...
# lambda, sparsityParam, ...
# beta, trainData), ...
# sae1Theta, options);
fn = lambda theta: autoencoder.sparse_autoencoder_loss(
theta, input_size, hidden_size_l1, lamb, sparsity_param, beta, train_data)
sae1_opt_theta, loss, d = (scipy.optimize.fmin_l_bfgs_b(fn,
sae1_theta,
maxfun=maxfun,
if DEBUG:
input_size = 64
# only 100 datapoints
train_data = train_data[:, :100]
train_labels = train_labels[:100]
# only top input_size most-varying input elements (pixels)
indices = train_data.var(1).argsort()[-input_size:]
train_data = np.asfortranarray(train_data[indices, :])
# === Step 2: Train the first sparse autoencoder ===
#
# Train the first sparse autoencoder on the unlabelled STL training
# images.
# Randomly initialize the parameters
sae1_theta = autoencoder.initialize_parameters(hidden_size_l1, input_size)
# Train the first layer sparse autoencoder. This layer has a hidden
# size of `hidden_size_l1`.
# sae1_opt_theta, loss = minFunc( @(p) sparseAutoencoderLoss(p, ...
# inputSize, hiddenSizeL1, ...
# lambda, sparsityParam, ...
# beta, trainData), ...
# sae1Theta, options);
fn = lambda theta: autoencoder.sparse_autoencoder_loss(
theta, input_size, hidden_size_l1, lamb, sparsity_param, beta, train_data
)
sae1_opt_theta, loss, d = scipy.optimize.fmin_l_bfgs_b(fn, sae1_theta, maxfun=maxfun, iprint=1)
train_labels = labels[:num_train] test_data = labeled_data[:, num_train:] test_labels = labels[num_train:] # Output some statistics print '# examples in unlabeled set: %d' % unlabeled_data.shape[1] print '# examples in supervised training set: %d' % train_data.shape[1] print '# examples in supervised testing set: %d' % test_data.shape[1] # === Step 2: Train the sparse autoencoder === # # This trains the sparse autoencoder on the unlabeled training images. # Randomly initialize the parameters theta = autoencoder.initialize_parameters(hidden_size, input_size) # The single-parameter function to minimize fn = lambda theta: autoencoder.sparse_autoencoder_loss( theta, input_size, hidden_size,lamb, sparsity_param, beta, unlabeled_data) # Find `opt_theta` by running the sparse autoencoder on unlabeled # training images. opt_theta, loss, d = ( scipy.optimize.fmin_l_bfgs_b(fn, theta, maxfun=maxfun, iprint=1, m=20)) # Visualize weights W1, W2, b1, b2 = autoencoder.unflatten(opt_theta, input_size, hidden_size) util.display_network(W1.T) # === Step 3: Extract Features from the Supervised Dataset === train_features = autoencoder.feedforward_autoencoder(
train_labels = labels[:num_train]
test_data = labeled_data[:, num_train:]
test_labels = labels[num_train:]
# Output some statistics
print '# examples in unlabeled set: %d' % unlabeled_data.shape[1]
print '# examples in supervised training set: %d' % train_data.shape[1]
print '# examples in supervised testing set: %d' % test_data.shape[1]
# === Step 2: Train the sparse autoencoder ===
#
# This trains the sparse autoencoder on the unlabeled training images.
# Randomly initialize the parameters
theta = autoencoder.initialize_parameters(hidden_size, input_size)
# The single-parameter function to minimize
fn = lambda theta: autoencoder.sparse_autoencoder_loss(
theta, input_size, hidden_size, lamb, sparsity_param, beta, unlabeled_data)
# Find `opt_theta` by running the sparse autoencoder on unlabeled
# training images.
opt_theta, loss, d = (scipy.optimize.fmin_l_bfgs_b(fn,
theta,
maxfun=maxfun,
iprint=1,
m=20))
# Visualize weights
W1, W2, b1, b2 = autoencoder.unflatten(opt_theta, input_size, hidden_size)
util.display_network(W1.T)
