def classifier(hidden_units, n_unlabeled_inputs, n_labeled_inputs):
"""
Train a semi-supervised classifier. We begin with pretraining,
creating an autoencoder which uses ``n_unlabeled_inputs`` from the
MNIST training data. This is then converted into a classifier
which is fine-tuned using the ``n_labeled_inputs``.
For comparison a classifier is also created which does not make
use of the unlabeled data.
"""
training_data, test_inputs, actual_test_results = \
mnist_loader.load_data_nn()
print "\nUsing pretraining and %s items of unlabeled data" %\
n_unlabeled_inputs
net_ae = train_autoencoder(hidden_units,
training_data[:n_unlabeled_inputs])
net_c = Network([784, hidden_units, 10])
net_c.biases = net_ae.biases[:1] + [np.random.randn(10, 1) / np.sqrt(10)]
net_c.weights = net_ae.weights[:1]+\
[np.random.randn(10, hidden_units)/np.sqrt(10)]
net_c.SGD(training_data[-n_labeled_inputs:], 300, 10, 0.01, 0.05)
print "Result on test data: %s / %s" % (net_c.evaluate(
test_inputs, actual_test_results), len(test_inputs))
print "Training a network with %s items of training data" % n_labeled_inputs
net = Network([784, hidden_units, 10])
net.SGD(training_data[-n_labeled_inputs:], 300, 10, 0.01, 0.05)
print "Result on test data: %s / %s" % (net.evaluate(
test_inputs, actual_test_results), len(test_inputs))
return net_c
def add_classifier_layer(net, num_outputs):
"""
Return the Network ``net``, but with an extra layer containing
``num_outputs`` neurons appended."""
net_classifier = Network(net.sizes+[num_outputs])
net_classifier.weights[:-1] = net.weights
net_classifier.biases[:-1] = net.biases
return net_classifier
def __init__(self, layers):
"""
The list ``layers`` specifies the sizes of the nested
autoencoders. For example, if ``layers`` is [50, 20, 10] then
the deep autoencoder will be a neural network with layers of
size [50, 20, 10, 20, 50]."""
self.layers = layers
Network.__init__(self, layers + layers[-2::-1])
def unroll(deep_autoencoder):
"""
Return a Network that contains the compression stage of the
``deep_autoencoder``."""
net = Network(deep_autoencoder.layers)
net.weights = deep_autoencoder.weights[:len(deep_autoencoder.layers)-1]
net.biases = deep_autoencoder.biases[:len(deep_autoencoder.layers)-1]
return net
def __init__(self, layers):
"""
The list ``layers`` specifies the sizes of the nested
autoencoders. For example, if ``layers`` is [50, 20, 10] then
the deep autoencoder will be a neural network with layers of
size [50, 20, 10, 20, 50]."""
self.layers = layers
Network.__init__(self, layers+layers[-2::-1])
def SGD_final_layer(
self, training_data, epochs, mini_batch_size, eta, lmbda):
"""
Run SGD on the final layer of the Network ``self``. Note that
``training_data`` is the input to the whole Network, not the
encoded training data input to the final layer.
"""
encoded_training_data = [
(self.feedforward(x, start=0, end=self.num_layers-2), y)
for x, y in training_data]
net = Network(self.sizes[-2:])
net.biases[0] = self.biases[-1]
net.weights[0] = self.weights[-1]
net.SGD(encoded_training_data, epochs, mini_batch_size, eta, lmbda)
self.biases[-1] = net.biases[0]
self.weights[-1] = net.weights[0]
def train_nested_autoencoder(
self, j, encoded_training_data, epochs, mini_batch_size, eta, lmbda):
"""
Train the nested autoencoder that starts at layer ``j`` in the
deep autoencoder. Note that ``encoded_training_data`` is a
list with entries of the form ``(x, x)``, where the ``x`` are
encoded training inputs for layer ``j``."""
net = Network([self.layers[j], self.layers[j+1], self.layers[j]])
net.biases[0] = self.biases[j]
net.biases[1] = self.biases[-j-1]
net.weights[0] = self.weights[j]
net.weights[1] = self.weights[-j-1]
net.SGD(encoded_training_data, epochs, mini_batch_size, eta, lmbda)
self.biases[j] = net.biases[0]
self.biases[-j-1] = net.biases[1]
self.weights[j] = net.weights[0]
self.weights[-j-1] = net.weights[1]
def classifier(hidden_units, n_unlabeled_inputs, n_labeled_inputs):
"""
Train a semi-supervised classifier. We begin with pretraining,
creating an autoencoder which uses ``n_unlabeled_inputs`` from the
MNIST training data. This is then converted into a classifier
which is fine-tuned using the ``n_labeled_inputs``.
For comparison a classifier is also created which does not make
use of the unlabeled data.
"""
training_data, test_inputs, actual_test_results = \
mnist_loader.load_data_nn()
print "\nUsing pretraining and %s items of unlabeled data" %\
n_unlabeled_inputs
net_ae = train_autoencoder(hidden_units, training_data[:n_unlabeled_inputs])
net_c = Network([784, hidden_units, 10])
net_c.biases = net_ae.biases[:1]+[np.random.randn(10, 1)/np.sqrt(10)]
net_c.weights = net_ae.weights[:1]+\
[np.random.randn(10, hidden_units)/np.sqrt(10)]
net_c.SGD(training_data[-n_labeled_inputs:], 300, 10, 0.01, 0.05)
print "Result on test data: %s / %s" % (
net_c.evaluate(test_inputs, actual_test_results), len(test_inputs))
print "Training a network with %s items of training data" % n_labeled_inputs
net = Network([784, hidden_units, 10])
net.SGD(training_data[-n_labeled_inputs:], 300, 10, 0.01, 0.05)
print "Result on test data: %s / %s" % (
net.evaluate(test_inputs, actual_test_results), len(test_inputs))
return net_c
def train_nested_autoencoder(self, j, encoded_training_data, epochs,
mini_batch_size, eta, lmbda):
"""
Train the nested autoencoder that starts at layer ``j`` in the
deep autoencoder. Note that ``encoded_training_data`` is a
list with entries of the form ``(x, x)``, where the ``x`` are
encoded training inputs for layer ``j``."""
net = Network([self.layers[j], self.layers[j + 1], self.layers[j]])
net.biases[0] = self.biases[j]
net.biases[1] = self.biases[-j - 1]
net.weights[0] = self.weights[j]
net.weights[1] = self.weights[-j - 1]
net.sgd(encoded_training_data, epochs, mini_batch_size, eta, lmbda)
self.biases[j] = net.biases[0]
self.biases[-j - 1] = net.biases[1]
self.weights[j] = net.weights[0]
self.weights[-j - 1] = net.weights[1]
def train_autoencoder(hidden_units, training_data):
"Return a trained autoencoder."
autoencoder_training_data = [(x, x) for x, _ in training_data]
net = Network([784, hidden_units, 784])
net.SGD(autoencoder_training_data, 6, 10, 0.01, 0.05)
return net
#### Parameters
# Size of the training sets. May range from 1000 to 12,500. Lower
# will be faster, higher will give more accuracy.
SIZE = 5000
# Number of hidden units in the autoencoder
HIDDEN = 30
print("\nGenerating training data")
training_data, _, _ = mnist_loader.load_data_nn()
td_1 = [(x, x) for x, _ in training_data[0:SIZE]]
td_2 = [(x, x) for x, _ in training_data[12500:12500 + SIZE]]
td_3 = [x for x, _ in training_data[25000:25000 + SIZE]]
test = [x for x, _ in training_data[37500:37500 + SIZE]]
print("\nFinding first autoencoder")
ae_1 = Network([784, HIDDEN, 784])
ae_1.SGD(td_1, 4, 10, 0.01, 0.05)
print("\nFinding second autoencoder")
ae_2 = Network([784, HIDDEN, 784])
ae_2.SGD(td_1, 4, 10, 0.01, 0.05)
print("\nGenerating encoded training data")
encoded_td_1 = [
sigmoid_vec(np.dot(ae_1.weights[0], x) + ae_1.biases[0]) for x in td_3
]
encoded_td_2 = [
sigmoid_vec(np.dot(ae_2.weights[0], x) + ae_2.biases[0]) for x in td_3
]
encoded_training_data = zip(encoded_td_1, encoded_td_2)
print("\nFinding mapping between theories")
ae_1 = Network([784, HIDDEN, 784])
ae_1.SGD(td_1, 4, 10, 0.01, 0.05)
print "\nFinding second autoencoder"
ae_2 = Network([784, HIDDEN, 784])
ae_2.SGD(td_1, 4, 10, 0.01, 0.05)
print "\nGenerating encoded training data"
encoded_td_1 = [sigmoid_vec(np.dot(ae_1.weights[0], x)+ae_1.biases[0])
for x in td_3]
encoded_td_2 = [sigmoid_vec(np.dot(ae_2.weights[0], x)+ae_2.biases[0])
for x in td_3]
encoded_training_data = zip(encoded_td_1, encoded_td_2)
print "\nFinding mapping between theories"
net = Network([HIDDEN, HIDDEN])
net.SGD(encoded_training_data, 6, 10, 0.01, 0.05)
print """\nBaseline for comparison: decompress with the first autoencoder"""
print """and compress with the second autoencoder"""
encoded_test_1 = [sigmoid_vec(np.dot(ae_1.weights[0], x)+ae_1.biases[0])
for x in test]
encoded_test_2 = [sigmoid_vec(np.dot(ae_2.weights[0], x)+ae_2.biases[0])
for x in test]
test_data = zip(encoded_test_1, encoded_test_2)
net_baseline = Network([HIDDEN, 784, HIDDEN])
net_baseline.biases[0] = ae_1.biases[1]
net_baseline.weights[0] = ae_1.weights[1]
net_baseline.biases[1] = ae_2.biases[0]
net_baseline.weights[1] = ae_2.weights[0]
error_baseline = sum(np.linalg.norm(net_baseline.feedforward(x)-y, 1)
