def test_draw_network(self, mock_pydot):
mock_pydot_obj = mock_pydotplus()
mock_pydot.return_value = mock_pydot_obj
util_graph.draw_network(self.G, "test.pdf", verbose=3)
print(mock_pydot_obj.called)
print()
self.assertEqual(mock_pydot_obj.called, 3)
self.assertEqual(mock_pydot_obj.filename, "test.pdf")
def lenet5 ( dataset= None, verbose = 1 ):
"""
This function is a demo example of lenet5 from the infamous paper by Yann LeCun.
This is an example code. You should study this code rather than merely run it.
Warning:
This is not the exact implementation but a modern re-incarnation.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'false',
# false, polyak, nesterov
"momentum_params" : (0.5, 0.95, 30),
# (mom_start, momentum_end, momentum_end_epoch)
"regularization" : (0.00, 0.00),
# l1_coeff, l2_coeff, decisiveness (optional)
"optimizer_type" : 'sgd',
# sgd, adagrad, rmsprop, adam
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
# intitialize the network
net = network( borrow = True,
verbose = verbose )
# or you can add modules after you create the net.
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = False )
# add first convolutional layer
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 20,
filter_size = (5,5),
pool_size = (2,2),
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "conv_pool",
origin = "conv_pool_1",
id = "conv_pool_2",
num_neurons = 50,
filter_size = (3,3),
pool_size = (2,2),
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "conv_pool_2",
id = "dot_product_1",
num_neurons = 800,
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 800,
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "nll",
datastream_origin = 'data',
verbose = verbose
)
# objective provided by classifier layer
# nll-negative log likelihood,
# cce-categorical cross entropy,
# bce-binary cross entropy,
# hinge-hinge loss
learning_rates = (0, 0.1, 0.01, 0.001, 0.0001)
# (annealing, initial_learning_rate, ... )
# net.pretty_print() # this will print out the network.
# visualization of the network.
net.pretty_print()
draw_network(net.graph, filename = 'lenet.png')
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
net.train( epochs = (10, 10, 10, 10),
validate_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)
def mlp ( dataset, verbose = 1 ):
"""
This method is a tutorial on building a two layer multi-layer neural network. The built
network is mnist->800->800->10 .It optimizes with polyak momentum and rmsprop.
Args:
dataset: an already created dataset.
"""
optimizer_params = {
"momentum_type" : 'polyak',
"momentum_params" : (0.9, 0.95, 30),
"regularization" : (0.0001, 0.0001),
"optimizer_type" : 'adagrad',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
net = network( borrow = True,
verbose = verbose )
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = True )
net.add_layer ( type = "dot_product",
origin = "input",
id = "dot_product_1",
num_neurons = 800,
activation = 'relu',
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 800,
activation = 'relu',
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "nll",
origin = "softmax",
verbose = verbose
)
learning_rates = (0.05, 0.01, 0.001)
net.cook( optimizer = 'main',
objective_layer = 'nll',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
draw_network(net.graph, filename = 'mlp.png')
net.train( epochs = (20, 20),
validate_after_epochs = 1,
training_accuracy = True,
show_progress = True,
early_terminate = True,
learning_rates = learning_rates,
verbose = verbose)
net.test(verbose = verbose)
def update_current_network(self, dataset, root='.', verbose=2):
"""
This method setup the increment net.
Args:
root: location to save outputs.
dataset: Increment dataset.
Notes:
This method creates two networks with shared parameters. One network is used to update
the parameters using the dataset and the other network is used to update the parameters
for the mentoring via GAN.
The parameters are not shared with the mentor network they are newly created copies, but
the two networks created in this method do share parameters.
"""
# Collecting parameters as copies from the base mentor network.
# New copies of params so that I wont overwrite the
# pointer params in the phantom sampler network object.
self._update_num_classes(dataset)
params = self.base.get_params(verbose=verbose)
params = shared_params(params)
# For classifier layer, recreating...
old_w = params['softmax'][0].eval()
old_b = params['softmax'][1].eval()
new_w = numpy.asarray(0.01 * rng.standard_normal( size=(old_w.shape[0], \
self.num_classes - old_w.shape[1])),
dtype=theano.config.floatX)
new_w_values = numpy.concatenate((old_w, new_w), axis=1)
new_b = numpy.asarray(0.01 * rng.standard_normal( size = \
(self.num_classes - old_w.shape[1])),
dtype=theano.config.floatX)
new_b_values = numpy.concatenate((old_b, new_b), axis=0)
new_w = theano.shared(value=new_w_values,
name='inc-weights',
borrow=True)
new_b = theano.shared(value=new_b_values, name='inc-bias', borrow=True)
params["softmax"] = [new_w, new_b]
# This net is initialized with the same parameters as the phantom / base network
# but with more nodes in the last softmax layer to accommodate for more labels.
self.current = self._mlp(dataset=dataset,
params=params,
cook=False,
root=root,
id="-current-" + str(self.increment),
verbose=verbose)
# AGain will have increment number of GANs.
objective_layers = list()
for inc in xrange(len(self.gans.keys())):
self.current.add_layer(
type="tensor",
id="gan-input-" + str(inc),
input=self.phantom_labeler.inference_layers["input-" +
str(inc)].output,
input_shape=self.phantom_labeler.dropout_layers[
"input-" + str(inc)].output_shape,
verbose=verbose)
self.current.add_layer(type="unflatten",
id="gan-input-unflattened-" + str(inc),
origin="gan-input-" + str(inc),
shape=(32, 32, 3),
verbose=verbose)
self.current.add_layer(
type="conv_pool",
id="c1-gan-" + str(inc),
origin="gan-input-unflattened-" + str(inc),
num_neurons=20,
filter_size=(5, 5),
pool_size=(2, 2),
activation='relu',
regularize=True,
batch_norm=True,
input_params=self.current.dropout_layers['c1'].params,
verbose=verbose)
self.current.add_layer(
type="conv_pool",
id="c2-gan-" + str(inc),
origin="c1-gan-" + str(inc),
num_neurons=50,
filter_shape=(3, 3),
pool_size=(2, 2),
batch_norm=True,
regularize=True,
activation='relu',
input_params=self.current.dropout_layers['c2'].params,
verbose=verbose)
self.current.add_layer(
type="dot_product",
origin="c2-gan-" + str(inc),
id="fc1-gan-" + str(inc),
num_neurons=800,
activation='relu',
batch_norm=True,
dropout_rate=0.5,
regularize=True,
input_params=self.current.dropout_layers['fc1'].params,
verbose=verbose)
self.current.add_layer(
type="dot_product",
origin="fc1-gan-" + str(inc),
id="fc2-gan-" + str(inc),
num_neurons=800,
activation='relu',
batch_norm=True,
dropout_rate=0.5,
regularize=True,
input_params=self.current.dropout_layers['fc2'].params,
verbose=verbose)
self.current.add_layer ( type = "classifier",
id = "softmax-gan-" + str(inc),
origin = "fc2-gan-" + str(inc),
num_classes = self.num_classes,
activation = ('softmax', self.temperature),
input_params = self.current.dropout_layers \
['softmax'].params,
regularize = True,
verbose = verbose )
##########
# This will make the mentor values available to the current network so that we
# can caluclate errors
##########
self.current.add_layer(type = "random",
id = "zero-targets-" + str(inc),
num_neurons = (self.mini_batch_size, \
self.num_classes - old_w.shape[1] ),
distribution = 'binomial',
p = 0,
verbose = verbose)
# input_shape = [self.phantom_labeler.layers['softmax-base-temperature'].output_shape,
# self.hallucinated.layers['zero-targets'].output_shape]
# importing a layer from the mentor network.
self.current.add_layer (type = "tensor",
id = "merge-import-" + str(inc),
input = self.phantom_labeler.inference_layers \
["phantom-" + str(inc)].output,
input_shape = self.phantom_labeler.inference_layers \
["phantom-" + str(inc)].output_shape,
verbose = verbose )
self.current.add_layer (type = "merge",
layer_type = "concatenate",
id = "phantom-targets-" + str(inc),
origin = ( "merge-import-" + str(inc),\
"zero-targets-" + str(inc)),
verbose = verbose
)
##########
# objective layers
##########
# This is error between the temperature softmax layer and the mentor target.
# This provides the incremental update.
self.current.add_layer(
type="merge",
id="obj-phantom-" + str(inc),
layer_type="error",
error="rmse",
origin=("softmax-gan-" + str(inc),
"phantom-targets-" + str(inc)),
)
objective_layers.append("obj-phantom-" + str(inc))
# This is the regular classifier objective for the incremental net.
self.current.add_layer(type="objective",
id="obj-current",
origin="softmax",
verbose=verbose)
objective_layers.append("obj-current")
# self.current.pretty_print()
draw_network(self.current.graph, filename='hallucinated.png')
self.current.cook(optimizer='optim',
objective_layers=objective_layers,
datastream='data',
classifier_layer='softmax',
verbose=verbose)
def update_phantom_labeler(self, temperature=3, verbose=2):
"""
This method sets up the phantom labeler network which is basically the same
network that takes the GAN as input and produces softmaxes.
"""
if verbose >= 2:
print(".. Creating the labeler network")
self.phantom_labeler = network()
if not temperature is None:
self.temperature = temperature
# Every layer should have increment number of layers. But the same
# parameters are shared. So it is in effect just one layer.
for inc in xrange(len(self.gans.keys())):
self.phantom_labeler.add_layer(
type="tensor",
id="input-" + str(inc),
input=self.gans[inc].dropout_layers['G(z)'].output,
input_shape=(self.mini_batch_size, 3072),
verbose=verbose)
self.phantom_labeler.add_layer(type="unflatten",
id="input-unflattened-" + str(inc),
origin="input-" + str(inc),
shape=(32, 32, 3),
verbose=verbose)
self.phantom_labeler.add_layer(
type="conv_pool",
id="c1-" + str(inc),
origin="input-unflattened-" + str(inc),
num_neurons=20,
filter_size=(5, 5),
pool_size=(2, 2),
activation='relu',
regularize=True,
batch_norm=True,
input_params=self.base.dropout_layers['c1'].params,
verbose=verbose)
self.phantom_labeler.add_layer(
type="conv_pool",
id="c2-" + str(inc),
origin="c1-" + str(inc),
num_neurons=50,
filter_shape=(3, 3),
pool_size=(2, 2),
batch_norm=True,
regularize=True,
activation='relu',
input_params=self.base.dropout_layers['c2'].params,
verbose=verbose)
self.phantom_labeler.add_layer(
type="dot_product",
origin="c2-" + str(inc),
id="fc1-" + str(inc),
num_neurons=800,
activation='relu',
batch_norm=True,
dropout_rate=0.5,
regularize=True,
input_params=self.base.dropout_layers['fc1'].params,
verbose=verbose)
self.phantom_labeler.add_layer(
type="dot_product",
origin="fc1-" + str(inc),
id="fc2-" + str(inc),
num_neurons=800,
activation='relu',
input_params=self.base.dropout_layers['fc2'].params,
batch_norm=True,
dropout_rate=0.5,
regularize=True,
verbose=verbose)
self.phantom_labeler.add_layer(
type="classifier",
id="softmax-" + str(inc),
origin="fc2-" + str(inc),
num_classes=self.base.dropout_layers['softmax'].
output_shape[1],
activation='softmax',
input_params=self.base.dropout_layers['softmax'].params,
regularize=True,
verbose=verbose)
self.phantom_labeler.add_layer(
type="classifier",
id="phantom-" + str(inc),
origin="fc2-" + str(inc),
num_classes=self.base.dropout_layers['softmax'].
output_shape[1],
activation=('softmax', self.temperature),
input_params=self.base.dropout_layers['softmax'].params,
regularize=True,
verbose=verbose)
#self.phantom_labeler.pretty_print()
draw_network(self.phantom_labeler.graph,
filename='phantom_labeler.png')
def shallow_gan(dataset=None, verbose=1):
"""
This function is a demo example of a generative adversarial network.
This is an example code. You should study this code rather than merely run it.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type": 'polyak',
"momentum_params": (0.65, 0.9, 50),
"regularization": (0.000, 0.000),
"optimizer_type": 'rmsprop',
"id": "main"
}
dataset_params = {"dataset": dataset, "type": 'xy', "id": 'data'}
visualizer_params = {
"root": '.',
"frequency": 1,
"sample_size": 225,
"rgb_filters": False,
"debug_functions": False,
"debug_layers": True,
"id": 'main'
}
# intitialize the network
net = gan(borrow=True, verbose=verbose)
net.add_module(type='datastream', params=dataset_params, verbose=verbose)
net.add_module(type='visualizer',
params=visualizer_params,
verbose=verbose)
#z - latent space created by random layer
net.add_layer(type='random',
id='z',
num_neurons=(100, 32),
distribution='normal',
mu=0,
sigma=1,
verbose=verbose)
#x - inputs come from dataset 1 X 784
net.add_layer(
type="input",
id="x",
verbose=verbose,
datastream_origin='data', # if you didnt add a dataset module, now is
# the time.
mean_subtract=False)
net.add_layer(
type="dot_product",
origin="z",
id="G(z)",
num_neurons=784,
activation='tanh',
verbose=verbose) # This layer is the one that creates the images.
#D(x) - Contains params theta_d creates features 1 X 800.
net.add_layer(type="dot_product",
id="D(x)",
origin="x",
num_neurons=800,
activation='relu',
regularize=True,
verbose=verbose)
net.add_layer(type="dot_product",
id="D(G(z))",
origin="G(z)",
input_params=net.dropout_layers["D(x)"].params,
num_neurons=800,
activation='relu',
regularize=True,
verbose=verbose)
#C(D(x)) - This is the opposite of C(D(G(z))), real
net.add_layer(type="dot_product",
id="real",
origin="D(x)",
num_neurons=1,
activation='sigmoid',
verbose=verbose)
#C(D(G(z))) fake - the classifier for fake/real that always predicts fake
net.add_layer(
type="dot_product",
id="fake",
origin="D(G(z))",
num_neurons=1,
activation='sigmoid',
input_params=net.dropout_layers["real"].
params, # Again share their parameters
verbose=verbose)
#C(D(x)) - This is the opposite of C(D(G(z))), real
net.add_layer(type="classifier",
id="softmax",
origin="D(x)",
num_classes=10,
activation='softmax',
verbose=verbose)
# objective layers
# discriminator objective
net.add_layer (type = "tensor",
input = - 0.5 * T.mean(T.log(net.layers['real'].output)) - \
0.5 * T.mean(T.log(1-net.layers['fake'].output)),
input_shape = (1,),
id = "discriminator_task"
)
net.add_layer(type="objective",
id="discriminator_obj",
origin="discriminator_task",
layer_type='value',
objective=net.dropout_layers['discriminator_task'].output,
datastream_origin='data',
verbose=verbose)
#generator objective
net.add_layer(type="tensor",
input=-0.5 * T.mean(T.log(net.layers['fake'].output)),
input_shape=(1, ),
id="objective_task")
net.add_layer(type="objective",
id="generator_obj",
layer_type='value',
origin="objective_task",
objective=net.dropout_layers['objective_task'].output,
datastream_origin='data',
verbose=verbose)
#softmax objective.
net.add_layer(type="objective",
id="classifier_obj",
origin="softmax",
objective="nll",
layer_type='discriminator',
datastream_origin='data',
verbose=verbose)
from yann.utils.graph import draw_network
draw_network(net.graph, filename='gan.png')
net.pretty_print()
net.cook(objective_layers=[
"classifier_obj", "discriminator_obj", "generator_obj"
],
optimizer_params=optimizer_params,
discriminator_layers=["D(x)"],
generator_layers=["G(z)"],
classifier_layers=["D(x)", "softmax"],
softmax_layer="softmax",
game_layers=("fake", "real"),
verbose=verbose)
learning_rates = (0.05, 0.01)
net.train(epochs=(20),
k=2,
pre_train_discriminator=3,
validate_after_epochs=1,
visualize_after_epochs=1,
training_accuracy=True,
show_progress=True,
early_terminate=True,
verbose=verbose)
return net
def log_reg ( dataset, verbose ):
"""
This function is a demo example of multi-layer neural networks from the infamous paper by
Yann LeCun. This is an example code. You should study this code rather than merely run it.
"""
optimizer_params = {
"momentum_type" : 'false',
# false, polyak, nesterov
"momentum_params" : (0.5, 0.95, 30),
# (mom_start, momentum_end, momentum_end_epoch)
"regularization" : (0.000, 0.000),
# l1_coeff, l2_coeff, decisiveness (optional)
"optimizer_type" : 'sgd',
# sgd, adagrad, rmsprop, adam
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data',
}
# intitialize the network
net = network( verbose = verbose )
# or you can add modules after you create the net.
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data',
# if you didnt add a dataset module, now is the time.
mean_subtract = False )
net.add_layer ( type = "classifier",
id = "softmax",
origin = "input",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "nll",
origin = "softmax",
objective = 'nll',
verbose = verbose
)
# objective provided by classifier layer
# nll-negative log likelihood,
# cce-categorical cross entropy,
# bce-binary cross entropy,
# hinge-hinge loss
learning_rates = (1, 0.01, 0.001)
# (initial_learning_rate, annealing, ft_learnint_rate)
net.cook( optimizer = 'main',
objective_layer = 'nll',
datastream = 'data',
classifier = 'softmax',
learning_rates = learning_rates,
verbose = verbose
)
# visualization of the network.
draw_network(net.graph, filename = 'log_reg.pdf')
net.train( epochs = (10, 5),
validate_after_epochs = 1,
training_accuracy = True,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test( show_progress = True,
verbose = verbose)
def lenet5 ( dataset= None, verbose = 1 ):
"""
This function is a demo example of lenet5 from the infamous paper by Yann LeCun.
This is an example code. You should study this code rather than merely run it.
Warning:
This is not the exact implementation but a modern re-incarnation.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'polyak',
"momentum_params" : (0.65, 0.97, 30),
"optimizer_type" : 'adagrad',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
visualizer_params = {
"root" : 'lenet5',
"frequency" : 1,
"sample_size": 144,
"rgb_filters": True,
"debug_functions" : False,
"debug_layers": False, # Since we are on steroids this time, print everything.
"id" : 'main'
}
# intitialize the network
net = network( borrow = True,
verbose = verbose )
# or you can add modules after you create the net.
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_module ( type = 'visualizer',
params = visualizer_params,
verbose = verbose
)
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = False )
# add first convolutional layer
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 20,
filter_size = (5,5),
pool_size = (2,2),
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "conv_pool",
origin = "conv_pool_1",
id = "conv_pool_2",
num_neurons = 50,
filter_size = (3,3),
pool_size = (2,2),
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "conv_pool_2",
id = "dot_product_1",
num_neurons = 1250,
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 1250,
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
regularize = True,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "nll",
datastream_origin = 'data',
regularization = (0.0001, 0.0001),
verbose = verbose
)
learning_rates = (0.05, 0.01, 0.001)
net.pretty_print()
draw_network(net.graph, filename = 'lenet.png')
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier_layer = 'softmax',
verbose = verbose
)
net.train( epochs = (40, 40),
validate_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)
def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
"""
This is a version with nesterov momentum and rmsprop instead of the typical sgd.
This also has maxout activations for convolutional layers, dropouts on the last
convolutional layer and the other dropout layers and this also applies batch norm
to all the layers. The difference though is that we use the ``batch_norm`` layer to apply
batch norm that applies batch norm after the activation fo the previous layer.
So we just spice things up and add a bit of steroids to
:func:`lenet5`. This also introduces a visualizer module usage.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'nesterov',
"momentum_params" : (0.75, 0.95, 30),
"optimizer_type" : 'rmsprop',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
visualizer_params = {
"root" : 'lenet_bn_after',
"frequency" : 1,
"sample_size": 32,
"rgb_filters": True,
"debug_functions" : False,
"debug_layers": False, # Since we are on steroids this time, print everything.
"id" : 'main'
}
net = network( borrow = True,
verbose = verbose )
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_module ( type = 'visualizer',
params = visualizer_params,
verbose = verbose )
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
origin = 'data' )
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 40,
filter_size = (5,5),
pool_size = (2,2),
activation = ('maxout', 'maxout', 2),
batch_norm = False,
regularize = True,
verbose = verbose
)
net.add_layer ( type = 'batch_norm',
origin = 'conv_pool_1',
id = 'batch_norm_after_cp_1',
)
net.add_layer ( type = "convolution",
origin = "batch_norm_after_cp_1",
id = "conv_pool_2",
num_neurons = 100,
filter_size = (3,3),
pool_size = (2,2),
activation = ('maxout', 'maxout', 2),
batch_norm = False,
regularize = True,
verbose = verbose
)
net.add_layer ( type = 'batchnorm',
origin = 'conv_pool_2',
id = 'batch_norm_after_cp_2',
)
net.add_layer ( type = "dot_product",
origin = "batch_norm_after_cp_2",
id = "dot_product_1",
num_neurons = 1250,
activation = 'relu',
dropout_rate = 0.5,
batch_norm = False,
regularize = True,
verbose = verbose
)
net.add_layer ( type = 'batchnorm',
origin = 'dot_product_1',
id = 'batch_norm_after_dp_1',
)
net.add_layer ( type = "dot_product",
origin = "batch_norm_after_dp_1",
id = "dot_product_2",
num_neurons = 1250,
activation = 'relu',
dropout_rate = 0.5,
regularize = True,
batch_norm = False,
verbose = verbose
)
net.add_layer ( type = 'batch_norm',
origin = 'dot_product_2',
id = 'batch_norm_after_dp_2',
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "batch_norm_after_dp_2",
num_classes = 10,
regularize = True,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "nll",
regularization = (0.0001, 0.0001),
datastream_origin = 'data',
verbose = verbose
)
learning_rates = (0.05, 0.001, 0.0001)
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
draw_network(net.graph, filename = 'lenet.png')
net.pretty_print()
net.train( epochs = (40, 40),
validate_after_epochs = 1,
visualize_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)