from network.layers.conv_to_fully_connected import ConvToFullyConnected
from network.layers.fully_connected import FullyConnected
from network.model import Model
from network.optimizer import GDMomentumOptimizer
if __name__ == '__main__':
"""
"""
freeze_support()
num_iteration = 20
data = dataset.cifar10_dataset.load()
layers = [
ConvToFullyConnected(),
FullyConnected(size=1000, activation=activation.tanh),
FullyConnected(size=10, activation=None, last_layer=True)
]
# -------------------------------------------------------
# Train with BP
# -------------------------------------------------------
model = Model(
layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-3, mu=0.9),
)
print("\nRun training:\n------------------------------------")
'Normal(sigma=1, mu=0)',
'Normal(sigma=1/sqrt(fan_out), mu=0)',
]
statistics = []
for initializer in initializers:
layers = [
MaxPool(size=2, stride=2),
Convolution((8, 3, 3, 3), stride=1, padding=1, dropout_rate=0, activation=activation.tanh, weight_initializer=initializer[0]),
MaxPool(size=2, stride=2),
Convolution((16, 8, 3, 3), stride=1, padding=1, dropout_rate=0, activation=activation.tanh, weight_initializer=initializer[1]),
MaxPool(size=2, stride=2),
Convolution((32, 16, 3, 3), stride=1, padding=1, dropout_rate=0, activation=activation.tanh, weight_initializer=initializer[2]),
MaxPool(size=2, stride=2),
ConvToFullyConnected(),
FullyConnected(size=64, activation=activation.tanh),
FullyConnected(size=10, activation=None, last_layer=True)
]
model = Model(
layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-3, mu=0.9),
)
print("\n\n------------------------------------")
print("Initialize: {}".format(initializer))
print("\nRun training:\n------------------------------------")
'#6666ff', '#ff6666', '#66ff66', '#0000ff', '#ff0000', '#00ff00',
'#000099', '#990000', '#009900'
]
lines = ['-', '-', '-', '--', '--', '--', ':', ':', ':']
layer_sizes = [[800] * 2, [400] * 10, [240] * 50]
iterations = [4] * 3
data = dataset.mnist_dataset.load('dataset/mnist')
statistics = []
labels = []
# Hyperbolic Tangens
for layer_size, num_passes in zip(layer_sizes, iterations):
layers = [ConvToFullyConnected()]
for size in layer_size:
layers.append(FullyConnected(size=size,
activation=activation.tanh))
layers.append(FullyConnected(size=10, activation=None,
last_layer=True))
model = Model(layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-2, mu=0.9))
print("\nRun training:\n------------------------------------")
stats = model.train(data_set=data,
method='dfa',
num_passes=num_passes,
batch_size=64)
loss, accuracy = model.cost(*data.test_set())
labels = [
'Fill(0)',
'Fill(0.001)',
'Fill(1)',
'Uniform(low=-1, high=1)',
'Uniform(low=-1/sqrt(fan_out), high=1/sqrt(fan_out))',
'Uniform(low=-1/fan_out, high=1/fan_out)',
'Normal(sigma=1, mu=0)',
'Normal(sigma=1/sqrt(fan_out), mu=0)',
]
statistics = []
for initializer in initializers:
layers = [ConvToFullyConnected()]
for i in range(num_hidden_layers):
layers += [FullyConnected(size=num_hidden_units, activation=activation.tanh, weight_initializer=initializer)]
layers += [FullyConnected(size=10, activation=None, last_layer=True)]
model = Model(
layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-3, mu=0.9)
)
print("\n\n------------------------------------")
print("Initialize: {}".format(initializer))
print("\nRun training:\n------------------------------------")
stats = model.train(data_set=data, method='dfa', num_passes=num_passes, batch_size=50)
colors = [('blue', 'blue'), ('red', 'red'), ('green', 'green'),
('orange', 'orange'), ('black', 'black')]
depths = [10, 15, 20, 50, 100]
iterations = ([4] * 5)
# colors = [('blue', 'blue')]
# depths = [20]
# iterations = [20]
data = dataset.mnist_dataset.load('dataset/mnist')
statistics = []
for depth, num_passes in zip(depths, iterations):
layers = [ConvToFullyConnected()] + \
[FullyConnected(size=240, activation=activation.tanh) for _ in range(depth)] + \
[FullyConnected(size=10, activation=None, last_layer=True)]
""" DFA """
model = Model(
layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-3, mu=0.9),
)
print("\nRun training:\n------------------------------------")
stats_dfa = model.train(data_set=data,
method='dfa',
num_passes=num_passes,
batch_size=64)
from network.weight_initializer import RandomNormal
if __name__ == '__main__':
freeze_support()
colors = [('blue', 'blue'), ('red', 'red'), ('green', 'green'),
('black', 'black')]
depths = [10, 20, 50, 100]
iterations = ([4] * 4)
data = dataset.mnist_dataset.load('dataset/mnist')
statistics = []
for depth, num_passes in zip(depths, iterations):
layers = [ConvToFullyConnected()] + \
[FullyConnected(size=240, activation=activation.leaky_relu,
weight_initializer=RandomNormal(sigma=np.sqrt(2.0/240))) for _ in range(depth)] + \
[FullyConnected(size=10, activation=None, last_layer=True)]
""" BP """
model = Model(layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-2, mu=0.9))
print("\nRun training:\n------------------------------------")
stats_bp = model.train(data_set=data,
method='bp',
num_passes=num_passes,
batch_size=64)
loss, accuracy = model.cost(*data.test_set())
initializers = ['Normal(1/sqrt(fan_in), 0)', 'Normal(1/sqrt(fan_out), 0)']
train_methods = ['dfa', 'bp']
statistics = []
labels = []
for sizes, initializer in zip([fan_in, fan_out], initializers):
for train_method in train_methods:
layers = [ConvToFullyConnected()]
for i in range(len(sizes)):
layers.append(
FullyConnected(
size=fan_out[i],
activation=activation.tanh,
weight_initializer=weight_initializer.RandomNormal(
1 / np.sqrt(sizes[i])))),
layers.append(
FullyConnected(size=10, activation=None, last_layer=True))
model = Model(
layers=layers,
num_classes=10,
optimizer=GDMomentumOptimizer(lr=1e-3, mu=0.9),
regularization=0.001,
# lr_decay=0.5,
# lr_decay_interval=100
)
