def expanded_data():
expanded_training_data, _, _ = network3.load_data_shared(
"../data/mnist_expanded.pkl.gz")
for j in range(3):
print "Training with expanded data, run num %s" % j
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100,
activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(expanded_training_data,
20,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
def dbl_conv_relu():
for lmbda in [0.0, 0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0]:
for j in range(3):
print("Conv + Conv + FC num %s, relu, with regularization %s") % (
j, lmbda)
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=lmbda)
def expanded_data_double_fc(n=100):
"""n is the number of neurons in both fully-connected layers. We'll
try n=100, 300, and 1000.
"""
expanded_training_data, _, _ = network3.load_data_shared(
"../data/mnist_expanded.pkl.gz")
for j in range(3):
print(
"Training with expanded data, %s neurons in two FC layers, run num %s"
) % (n, j)
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=n, activation_fn=ReLU),
FullyConnectedLayer(n_in=n, n_out=n, activation_fn=ReLU),
SoftmaxLayer(n_in=n, n_out=10)
], mini_batch_size)
net.SGD(expanded_training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
def double_fc_dropout(p0, p1, p2, repetitions):
expanded_training_data, _, _ = network3.load_data_shared(
"../data/mnist_expanded.pkl.gz")
nets = []
for j in range(repetitions):
print("\n\nTraining using a dropout network with parameters "
), p0, p1, p2
print("Training with expanded data, run num %s") % j
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(
n_in=40 * 4 * 4, n_out=1000, activation_fn=ReLU, p_dropout=p0),
FullyConnectedLayer(
n_in=1000, n_out=1000, activation_fn=ReLU, p_dropout=p1),
SoftmaxLayer(n_in=1000, n_out=10, p_dropout=p2)
], mini_batch_size)
net.SGD(expanded_training_data, 40, mini_batch_size, 0.03,
validation_data, test_data)
nets.append(net)
return nets
def elu():
net = None
for j in range(RUNS):
print "num %s, leaky relu, with regularization %s" % (j, 0.0001)
net = Network([
ConvPoolLayer(image_shape=(MB_SIZE, 1, IMAGE_SIZE, IMAGE_SIZE),
filter_shape=(5, 1, 12, 12),
poolsize=(3, 3),
activation_fn=ELU),
ConvPoolLayer(image_shape=(MB_SIZE, 5, 30, 30),
filter_shape=(10, 5, 3, 3),
poolsize=(2, 2),
activation_fn=ELU),
FullyConnectedLayer(
n_in=10 * 14 * 14, n_out=200, activation_fn=ELU),
FullyConnectedLayer(n_in=200, n_out=200, activation_fn=ELU),
FullyConnectedLayer(n_in=200, n_out=100, activation_fn=ELU),
SoftmaxLayer(n_in=100, n_out=2)
], MB_SIZE)
net.SGD("ELU",
training_data,
EPOCHS,
MB_SIZE,
ETA,
validation_data,
test_data,
lmbda=0.0001)
return net
def test_4():
"""全连接 + 卷积混合层 + 卷积混合层 + 全连接 + softmax
激活函数:修正线性单元
代价函数：L2规范化
测试准确率：99.18%
"""
name = sys._getframe().f_code.co_name
print(name + "\n")
training_data, validation_data, test_data = network3.load_data_shared()
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
def shallow():
for j in range(3):
print "A shallow net with 100 hidden neurons"
net = Network([
FullyConnectedLayer(n_in=784, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data,
test_data)
def omit_FC():
for j in range(3):
print "Conv only, no FC"
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
SoftmaxLayer(n_in=20*12*12, n_out=10)], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)
return net
def test_drop(mini_batch_size):
size = 100
f_s = 7
padding = 1
conv_stride = 1
pool_stride = 1
pool_size = 4
n_f1 = 20
n_f2 = 40
n_f3 = 80
co1 = ((size - f_s + 2 * padding) / conv_stride) + 1
po1 = ((co1 - pool_size) / pool_stride) + 1
print(po1)
co2 = ((po1 - f_s + 2 * padding) / conv_stride) + 1
po2 = ((co2 - pool_size) / pool_stride) + 1
print(po2)
co3 = ((po2 - f_s + 2 * padding) / conv_stride) + 1
po3 = ((co3 - pool_size) / pool_stride) + 1
print(po3)
layer1 = ConvPoolLayer(input_shape=(mini_batch_size, 1, size, size),
filter_shape=(n_f1, 1, f_s, f_s),
poolsize=(4, 4),
activation_fn=ReLU)
layer2 = ConvPoolLayer(input_shape=(mini_batch_size, n_f1, po1, po1),
filter_shape=(n_f2, n_f1, f_s, f_s),
poolsize=(4, 4),
activation_fn=ReLU)
layer3 = ConvPoolLayer(input_shape=(mini_batch_size, n_f2, po2, po2),
filter_shape=(n_f3, n_f2, f_s, f_s),
poolsize=(4, 4),
activation_fn=ReLU)
layer4 = FullyConnectedLayer(n_in=n_f3 * po3 * po3,
n_out=1000,
activation_fn=ReLU,
p_dropout=0.0)
layer5 = FullyConnectedLayer(n_in=1000,
n_out=500,
activation_fn=ReLU,
p_dropout=0.0)
layer6 = SoftmaxLayer(n_in=500, n_out=2, p_dropout=0.0)
net = Network([layer1, layer2, layer3, layer4, layer5, layer6],
mini_batch_size)
net.SGD(training_data,
10,
mini_batch_size,
0.001,
validation_data,
test_data,
lmbda=0.0)
def basic_conv(n=3, epochs=60):
for j in range(n):
print "Conv + FC architecture"
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
FullyConnectedLayer(n_in=20*12*12, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(
training_data, epochs, mini_batch_size, 0.1, validation_data, test_data)
return net
def shallow(n=3, epochs=60):
nets = []
for j in range(n):
print("A shallow net with 100 hidden neurons")
net = Network([
FullyConnectedLayer(n_in=784, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, epochs, mini_batch_size, 0.1, validation_data,
test_data)
nets.append(net)
return nets
def test_conv(mini_batch_size):
nets = []
net = Network(
[
# Layer 0
# 1 Input image of size = 100 x 100
# 20 Filters of size = 5 x 5
# poolsize = 2 x 2 ( stride length 2)
# output of layer 0 = 20 feature Images of size 48 x 48
ConvPoolLayer(input_shape=(mini_batch_size, 1, 224, 224),
filter_shape=(64, 1, 3, 3),
poolsize=(2, 2),
activation_fn=ReLU),
# Layer 1
# 20 Input images of size = 48 x 48
# 40 Filters of size = 5 x 5
# poolsize = 2 x 2 ( stride length = 2)
# output of layer 1 = 40 feature Images of size 22 x 22
ConvPoolLayer(input_shape=(mini_batch_size, 64, 112, 112),
filter_shape=(128, 64, 3, 3),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(input_shape=(mini_batch_size, 128, 56, 56),
filter_shape=(256, 128, 3, 3),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(input_shape=(mini_batch_size, 256, 28, 28),
filter_shape=(512, 256, 3, 3),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(input_shape=(mini_batch_size, 512, 14, 14),
filter_shape=(512, 512, 3, 3),
poolsize=(2, 2),
activation_fn=ReLU),
# Layer 2
FullyConnectedLayer(n_in=512 * 7 * 7,
n_out=4096,
activation_fn=ReLU,
p_dropout=0.0),
FullyConnectedLayer(
n_in=4096, n_out=1000, activation_fn=ReLU, p_dropout=0.0),
SoftmaxLayer(n_in=1000, n_out=2, p_dropout=0.0)
],
mini_batch_size)
# End of Network Architecture
net.SGD(training_data, 10, mini_batch_size, 0.01, validation_data,
test_data)
nets.append(net)
return nets
def basic_softmax_NN():
mini_batch_size = 10
train_data, val_data, test_data = go_parser.parse_games(1000,
test_percent=0.2,
val_percent=0.2,
onehot=False)
net = Network(
[
# FullyConnectedLayer(n_in=361, n_out=200),
SoftmaxLayer(n_in=361, n_out=361)
],
mini_batch_size)
net.SGD(shared(train_data), 50, mini_batch_size, 0.1, shared(val_data),
shared(test_data))
def test_0():
"""全连接 + 全连接 + softmax, 测试准确率97:80%
"""
name = sys._getframe().f_code.co_name
print(name + "\n")
training_data, validation_data, test_data = network3.load_data_shared()
mini_batch_size = 10
net = Network([
FullyConnectedLayer(n_in=784, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data,
test_data)
def test_1():
"""全连接 + 卷积混合层 + softmax，测试准确率98.48%
"""
name = sys._getframe().f_code.co_name
print(name + "\n")
training_data, validation_data, test_data = network3.load_data_shared()
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
SoftmaxLayer(n_in=20 * 12 * 12, n_out=10)
], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data,
test_data)
def dbl_conv(activation_fn=sigmoid):
for j in range(3):
print "Conv + Conv + FC architecture"
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=activation_fn),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=activation_fn),
FullyConnectedLayer(
n_in=40*4*4, n_out=100, activation_fn=activation_fn),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)
return net
def test_ding(mini_batch_size):
size = 32
f_s = 5
padding = 1
conv_stride = 1
pool_stride = 1
pool_size = 3
n_f1 = 20
n_f2 = 40
co1 = ((size - f_s + 2 * padding) / conv_stride) + 1
po1 = ((co1 - pool_size) / pool_stride) + 1
#print(po1)
co2 = ((po1 - f_s + 2 * padding) / conv_stride) + 1
po2 = ((co2 - pool_size) / pool_stride) + 1
#print(po2)
layer1 = ConvPoolLayer(input_shape=(mini_batch_size, 1, size, size),
filter_shape=(n_f1, 1, f_s, f_s),
poolsize=(pool_size, pool_size),
activation_fn=ReLU)
#layer2 = ConvPoolLayer(input_shape=(mini_batch_size, n_f1, po1 , po1),
# filter_shape=(n_f2, n_f1, f_s, f_s),
# poolsize=(pool_size, pool_size),
# activation_fn=ReLU)
layer3 = FullyConnectedLayer(n_in=n_f1 * po1 * po1,
n_out=500,
activation_fn=ReLU,
p_dropout=0.0)
layer4 = SoftmaxLayer(n_in=500, n_out=2, p_dropout=0.0)
net = Network([layer1, layer3, layer4], mini_batch_size)
net.SGD(training_data,
50,
mini_batch_size,
0.3,
validation_data,
training_data,
lmbda=0.0)
def test_small(mini_batch_size):
size = 100
f_s = 5
padding = 1
conv_stride = 1
pool_stride = 1
pool_size = 3
n_f1 = 30
n_f2 = 60
co1 = ((size - f_s + 2 * padding) / conv_stride) + 1
po1 = ((co1 - pool_size) / pool_stride) + 1
co2 = ((po1 - f_s + 2 * padding) / conv_stride) + 1
po2 = ((co2 - pool_size) / pool_stride) + 1
net = Network([
ConvPoolLayer(input_shape=(mini_batch_size, 1, size, size),
filter_shape=(n_f1, 1, f_s, f_s),
poolsize=(pool_size, pool_size),
activation_fn=ReLU),
ConvPoolLayer(input_shape=(mini_batch_size, n_f1, po1, po1),
filter_shape=(n_f2, n_f1, f_s, f_s),
poolsize=(pool_size, pool_size),
activation_fn=ReLU),
FullyConnectedLayer(n_in=n_f2 * po2 * po2,
n_out=100,
activation_fn=ReLU,
p_dropout=0.0),
SoftmaxLayer(n_in=100, n_out=2, p_dropout=0.0)
], mini_batch_size)
net.SGD(training_data,
100,
mini_batch_size,
0.01,
validation_data,
test_data,
lmbda=0.0)
f = open('../data/network.cnn', 'wb')
cPickle.dump(net, f, protocol=2)
f.close()
def test_6():
"""全连接 + 卷积混合层 + 卷积混合层 + 全连接 + softmax
激活函数:修正线性单元
代价函数：L2规范化
训练数据：使用扩展数据集，将数据集多扩张8倍
测试准确率：99.45% (60 epochs), 99.58% (600 epochs)
"""
name = sys._getframe().f_code.co_name
print(name + "\n")
# 扩展数据集多扩展8倍
src_path = '../../minst-data/data/mnist.pkl.gz'
dst_path = '../../minst-data/data/mnist_expanded_8.pkl.gz'
study_note.mnistTest().expand_mnist(src_path=src_path,
dst_path=dst_path,
expand_count=8)
training_data, validation_data, test_data = \
network3.load_data_shared(dst_path)
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
def test_digit(mini_batch_size):
size = 100
f_s = 10
padding = 1
conv_stride = 1
pool_stride = 1
pool_size = 5
n_f1 = 25
n_f2 = 50
co1 = ((size - f_s + 2 * padding) / conv_stride) + 1
po1 = ((co1 - pool_size) / pool_stride) + 1
co2 = ((po1 - f_s + 2 * padding) / conv_stride) + 1
po2 = ((co2 - pool_size) / pool_stride) + 1
net = Network(
[
ConvPoolLayer(input_shape=(mini_batch_size, 1, size, size),
filter_shape=(n_f1, 1, f_s, f_s),
poolsize=(pool_size, pool_size),
activation_fn=ReLU),
ConvPoolLayer(input_shape=(mini_batch_size, n_f1, po1, po1),
filter_shape=(n_f2, n_f1, f_s, f_s),
poolsize=(pool_size, pool_size),
activation_fn=ReLU),
FullyConnectedLayer(n_in=n_f2 * po2 * po2,
n_out=500,
activation_fn=ReLU,
p_dropout=0.0),
#FullyConnectedLayer(n_in=1000, n_out=1000, activation_fn=ReLU, p_dropout=0.0),
SoftmaxLayer(n_in=500, n_out=2, p_dropout=0.0)
],
mini_batch_size)
net.SGD(training_data,
100,
mini_batch_size,
0.01,
validation_data,
test_data,
lmbda=0.0)
def test_3():
"""全连接 + 卷积混合层 + 卷积混合层 + 全连接 + softmax，测试准确率99.09%
"""
name = sys._getframe().f_code.co_name
print(name + "\n")
training_data, validation_data, test_data = network3.load_data_shared()
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2)),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data,
test_data)
def test_7():
"""全连接 + 卷积混合层 + 卷积混合层 + 全连接 + 全连接 + softmax
激活函数:修正线性单元
代价函数：L2规范化
训练数据：使用扩展数据集
测试准确率：99.49%
"""
name = sys._getframe().f_code.co_name
print(name + "\n")
# 扩展数据集
expand_mnist.expand_mnist_data()
dst_path = "../../minst-data/data/mnist_expanded.pkl.gz"
training_data, validation_data, test_data = \
network3.load_data_shared(dst_path)
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
FullyConnectedLayer(n_in=100, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
def test_basic(mini_batch_size):
nets = []
net = Network(
[
ConvPoolLayer(input_shape=(mini_batch_size, 1, 224, 224),
filter_shape=(64, 1, 3, 3),
poolsize=(2, 2),
activation_fn=ReLU),
# Layer 2
FullyConnectedLayer(
n_in=64 * 112 * 112, n_out=40, activation_fn=ReLU),
# Activation function
SoftmaxLayer(n_in=40, n_out=2)
],
mini_batch_size)
# End of Network Architecture
net.SGD(training_data, 10, mini_batch_size, 0.001, validation_data,
test_data)
nets.append(net)
return nets
import network3 from network3 import Network from network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer training_data, validation_data, test_data = network3.load_data_shared() mini_batch_size = 10 net = Network([FullyConnectedLayer(n_in=784, n_out=100), SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size) net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(input_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
start_time = time.time()
net.SGD(training_data,
120,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
end_time = time.time()
print(f'Total time elapsed: {end_time - start_time} seconds')
"""RESULTS:
100% of data
------------
mini_batch_size = 10
net.SGD(training_data, 50, mini_batch_size, 0.03, validation_data, test_data, lmbda=0.1)
Best validation accuracy of 99.16% obtained at iteration 139999 (epoch 27)
Corresponding test accuracy of 99.19%
Total time elapsed: 3364.949262857437 seconds
"../data/quickdraw_expanded.pkl.gz")
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2)),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 3, 3),
poolsize=(2, 2)),
FullyConnectedLayer(
n_in=40 * 5 * 5, n_out=500, activation_fn=sigmoid, p_dropout=0.1),
FullyConnectedLayer(
n_in=500, n_out=500, activation_fn=sigmoid, p_dropout=0.1),
SoftmaxLayer(n_in=500, n_out=10, p_dropout=0.1)
], mini_batch_size)
net.SGD(expanded_training_data, 70, mini_batch_size, 0.01, validation_data,
test_data)
####Plot the weights of first layer
weights0 = np.load("/home/syrine92/Syrine_Belakaria/weights0.npy")
i = -1
fig, ax = plt.subplots(nrows=4, ncols=5)
for row in ax:
for col in row:
col.axes.get_xaxis().set_visible(False)
col.axes.get_yaxis().set_visible(False)
i = i + 1
col.imshow(weights0[i][0], cmap='Greys')
fig.savefig('/home/syrine92/Syrine_Belakaria/weights0.png')
####Plot the weights of second layer
weights1 = np.load("/home/syrine92/Syrine_Belakaria/weights1.npy")
i = -1
fig, ax = plt.subplots(nrows=5, ncols=8)
poolsize=(2, 2)),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2)),
FullyConnectedLayer(n_in=40*4*4, n_out=100),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(training_data, 60, mini_batch_size, 0.1, validation_data, test_data)
'''
# chapter 6 - rectified linear units and some l2 regularization (lmbda=0.1) => even better accuracy
from network3 import ReLU
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40 * 4 * 4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)
], mini_batch_size)
net.SGD(training_data,
60,
mini_batch_size,
0.03,
validation_data,
test_data,
lmbda=0.1)
regularization_factor = 0.1
topology = [
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)]
net = Network(topology, mini_batch_size)
result.append(net.SGD(training_data, epochs, mini_batch_size, learning_rate, validation_data, test_data, lmbda=regularization_factor))
dump_file(result, "result_pickle"+str(filter_size))
filter_size=1
mini_batch_size = 10
epochs = 30
learning_rate = 0.03
regularization_factor = 0.1
topology = [
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, filter_size, filter_size),
poolsize=(2, 2),
filter_shape=(50, 6, 9, 9),
poolsize=(1, 1),
activation_fn=ReLU),
FullyConnectedLayer(
n_in=50 * 5 * 5, n_out=1000, activation_fn=ReLU,
p_dropout=0.5),
FullyConnectedLayer(
n_in=1000, n_out=500, activation_fn=ReLU, p_dropout=0.5),
SoftmaxLayer(n_in=500, n_out=10, p_dropout=0.5)
], mini_batch_size)
print "=========Currently Calculating Voter number %s=========" % vote
k, p = net.SGD(expanded_training_data,
120,
mini_batch_size,
0.001,
validation_data,
test_data,
lmbda=0)
#vote_box = np.concatenate((vote_box, k))
#vote_prob_box = np.concatenate((vote_prob_box, np.array(p).reshape(-1,)))
### Polishing
k, p = net.SGD(training_data,
5,
mini_batch_size,
0.0005,
validation_data,
test_data,
lmbda=0)
vote_box = np.concatenate((vote_box, k))
vote_prob_box = np.concatenate((vote_prob_box, np.array(p).reshape(-1, )))
import network3
from network3 import Network
from network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer
from network3 import ReLU
training_data, validation_data, test_data = network3.load_data_shared()
expanded_training_data, _, _ = network3.load_data_shared("mnist_expanded.pkl.gz")
mini_batch_size = 10
net = Network([
ConvPoolLayer(image_shape=(mini_batch_size, 1, 28, 28),
filter_shape=(20, 1, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
ConvPoolLayer(image_shape=(mini_batch_size, 20, 12, 12),
filter_shape=(40, 20, 5, 5),
poolsize=(2, 2),
activation_fn=ReLU),
FullyConnectedLayer(n_in=40*4*4, n_out=100, activation_fn=ReLU),
SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)
net.SGD(expanded_training_data, 30, mini_batch_size, 0.03,
validation_data, test_data, lmbda=0.1)
