def double_fc_dropout(p0, p1, p2, repetitions):
expanded_training_data, _, _ = crater_deep_network.load_data_shared(
"../data/mnist_expanded.pkl.gz")
nets = []
for j in range(repetitions):
print("\n\nTraining using a dropout network with parameters ", p0, p1,
p2)
print("Training with expanded data, run num %s" % j)
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=1000, activation_fn=ReLU, p_dropout=p0),
FullyConnectedLayer(
n_in=1000, n_out=1000, activation_fn=ReLU, p_dropout=p1),
SoftmaxLayer(n_in=1000, n_out=10, p_dropout=p2)
], mini_batch_size)
net.SGD(expanded_training_data, 40, mini_batch_size, 0.03,
validation_data, test_data)
nets.append(net)
return nets
def expanded_data_double_fc(n=100):
"""n is the number of neurons in both fully-connected layers. We'll
try n=100, 300, and 1000.
"""
expanded_training_data, _, _ = crater_deep_network.load_data_shared(
"../data/mnist_expanded.pkl.gz")
for j in range(3):
print(
"Training with expanded data, %s neurons in two FC layers, run num %s"
% (n, j))
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=n, activation_fn=ReLU),
FullyConnectedLayer(n_in=n, n_out=n, activation_fn=ReLU),
SoftmaxLayer(n_in=n, n_out=10)
], mini_batch_size)
net.SGD(expanded_training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
def dbl_conv_relu():
for lmbda in [0.0, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
for j in range(3):
print("Conv + Conv + FC num %s, relu, with regularization %s" %
(j, lmbda))
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=lmbda)
def omit_FC():
for j in range(3):
print("Conv only, no FC")
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
SoftmaxLayer(n_in=20 * 12 * 12, n_out=10)
], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data,
test_data)
return net
def shallow(n=3, epochs=60):
nets = []
for j in range(n):
print("A shallow net with 100 hidden neurons")
net = Network([
FullyConnectedLayer(n_in=784, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, epochs, mini_batch_size, 0.1, validation_data,
test_data)
nets.append(net)
return nets
def basic_conv(n=3, epochs=60):
for j in range(n):
print("Conv + FC architecture")
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
FullyConnectedLayer(n_in=20 * 12 * 12, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, epochs, mini_batch_size, 0.1, validation_data,
test_data)
return net
def dbl_conv(activation_fn=sigmoid):
for j in range(3):
print("Conv + Conv + FC architecture")
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=activation_fn),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=activation_fn),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=100, activation_fn=activation_fn),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data,
test_data)
return net