def double_fc_dropout(p0, p1, p2, repetitions):
expanded_training_data, _, _ = network3.load_data_shared(
"../data/mnist_expanded.pkl.gz")
nets = []
for j in range(repetitions):
print("\n\nTraining using a dropout network with parameters ", p0, p1, p2)
print("Training with expanded data, run num %s" % j)
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=1000, activation_fn=ReLU, p_dropout=p0),
FullyConnectedLayer(
n_in=1000, n_out=1000, activation_fn=ReLU, p_dropout=p1),
SoftmaxLayer(n_in=1000, n_out=10, p_dropout=p2)], mini_batch_size)
net.SGD(expanded_training_data, 40, mini_batch_size, 0.03,
validation_data, test_data)
nets.append(net)
return nets
def conv_main():
train, validation, test = load_data_shared('data/mnist_expanded.pkl.gz')
mini_batch_size = 10
epochs = 30
net = Network([
ConvPoolLayer(input_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=relu),
ConvPoolLayer(input_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=relu),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=1000, activation_fn=relu, p_dropout=0.5),
FullyConnectedLayer(
n_in=1000, n_out=1000, activation_fn=relu, p_dropout=0.5),
SoftmaxLayer(n_in=1000, n_out=10)
], mini_batch_size)
net.SGD(train,
epochs=epochs,
mini_batch_size=mini_batch_size,
eta=0.03,
validation_data=validation,
test_data=test,
lmbda=0.1)
def regularized_dbl_conv():
for lmbda in [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
for j in range(3):
print("Conv + Conv + FC num %s, with reg %s" % (j, lmbda))
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2)),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data, lmbda=lmbda)
def dbl_conv(activation_fn=sigmoid):
for j in range(3):
print("Conv + Conv + FC architecture")
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=activation_fn),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=activation_fn),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=100, activation_fn=activation_fn),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)
return net
def omit_FC():
for j in range(3):
print("Conv only, no FC")
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
SoftmaxLayer(n_in=20 * 12 * 12, n_out=10)], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)
return net
def basic_conv(n=3, epochs=60):
for j in range(n):
print("Conv + FC architecture")
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
FullyConnectedLayer(n_in=20 * 12 * 12, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(
training_data, epochs, mini_batch_size, 0.1, validation_data, test_data)
return net
def expanded_data_double_fc(n=100):
"""n is the number of neurons in both fully-connected layers. We'll
try n=100, 300, and 1000.
"""
expanded_training_data, _, _ = network3.load_data_shared(
"../data/mnist_expanded.pkl.gz")
for j in range(3):
print("Training with expanded data, %s neurons in two FC layers, run num %s" % (n, j))
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=n, activation_fn=ReLU),
FullyConnectedLayer(n_in=n, n_out=n, activation_fn=ReLU),
SoftmaxLayer(n_in=n, n_out=10)], mini_batch_size)
net.SGD(expanded_training_data, 60, mini_batch_size, 0.03,
validation_data, test_data, lmbda=0.1)