def setUp(self):
self.net = network.network()
self.params = {
'1': np.random.rand(5),
'2': np.random.rand(5),
'3': np.random.rand(5)
}
def log_reg(dataset):
"""
This function is a demo example of logistic regression.
"""
# Create the yann network class with empty layers.
net = network()
# Setup the datastream module and add it to network.
dataset_params = {"dataset": dataset, "svm": False, "n_classes": 10}
net.add_module(type='datastream', params=dataset_params)
# Create an input layer that feeds from the datastream modele.
net.add_layer(type="input", datastream_origin='data')
# Create a logistic regression layer.
# Creates a softmax layer.
net.add_layer(type="classifier", num_classes=10)
# Create an objective layer.
# Default is negative log likelihood.
# What ever the objective is, is always minimized.
net.add_layer(type="objective")
# Cook the network.
net.cook()
# See how the network looks like.
net.pretty_print()
# Train the network.
net.train()
# Test for acccuracy.
net.test()
def log_reg(dataset):
"""
This function is a demo example of logistic regression.
"""
dataset_params = {"dataset": dataset, "svm": False, "n_classes": 10}
net = network()
net.add_module(type='datastream', params=dataset_params)
net.add_layer(type="input", datastream_origin='data')
net.add_layer(type="classifier", num_classes=10)
net.add_layer(type="objective")
net.cook()
net.pretty_print()
net.train()
net.test()
import os
os.environ[
'THEANO_FLAGS'] = "force_device=True, device=gpu, lib.cnmem=0.75, floatX=float32"
import theano
print("Theano Default Device: ")
print(theano.config.device)
import sys # For CLI args
from yann.network import network
net = network()
dataset_params = {"dataset": sys.argv[1], "id": 'mnist', "n_classes": 10}
net.add_layer(type="input", id="input", dataset_init_args=dataset_params)
net.add_layer(type="dot_product",
origin="input",
id="dot_product_1",
num_neurons=800,
regularize=True,
activation='relu')
net.add_layer(type="dot_product",
origin="dot_product_1",
id="dot_product_2",
num_neurons=800,
regularize=True,
activation='relu')
net.add_layer(
type="classifier",
def lenet5 ( dataset= None, verbose = 1 ):
"""
This function is a demo example of lenet5 from the infamous paper by Yann LeCun.
This is an example code. You should study this code rather than merely run it.
Warning:
This is not the exact implementation but a modern re-incarnation.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'false',
# false, polyak, nesterov
"momentum_params" : (0.5, 0.95, 30),
# (mom_start, momentum_end, momentum_end_epoch)
"regularization" : (0.00, 0.00),
# l1_coeff, l2_coeff, decisiveness (optional)
"optimizer_type" : 'sgd',
# sgd, adagrad, rmsprop, adam
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
# intitialize the network
net = network( borrow = True,
verbose = verbose )
# or you can add modules after you create the net.
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = False )
# add first convolutional layer
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 20,
filter_size = (5,5),
pool_size = (2,2),
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "conv_pool",
origin = "conv_pool_1",
id = "conv_pool_2",
num_neurons = 50,
filter_size = (3,3),
pool_size = (2,2),
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "conv_pool_2",
id = "dot_product_1",
num_neurons = 800,
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 800,
activation = 'tanh',
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "nll",
datastream_origin = 'data',
verbose = verbose
)
# objective provided by classifier layer
# nll-negative log likelihood,
# cce-categorical cross entropy,
# bce-binary cross entropy,
# hinge-hinge loss
learning_rates = (0, 0.1, 0.01, 0.001, 0.0001)
# (annealing, initial_learning_rate, ... )
# net.pretty_print() # this will print out the network.
# visualization of the network.
net.pretty_print()
draw_network(net.graph, filename = 'lenet.png')
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
net.train( epochs = (10, 10, 10, 10),
validate_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)
def lenet_on_steroids ( dataset= None, verbose = 1 ):
"""
This function is a demo example of lenet5 from the infamous paper by Yann LeCun.
This is an example code. You should study this code rather than merely run it.
This is a version with nesterov momentum and rmsprop instead of the typical sgd.
This also has maxout activations for convolutional layers, dropouts on the last
convolutional layer and the other dropout layers and this also applies batch norm
to all the layers. So we just spice things up and add a bit of steroids to
:func:`lenet5`. This also introduces a visualizer module usage.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'nesterov',
"momentum_params" : (0.5, 0.95, 30),
"regularization" : (0.0001, 0.0001),
"optimizer_type" : 'rmsprop',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
visualizer_params = {
"root" : '.',
"frequency" : 1,
"sample_size": 32,
"rgb_filters": False,
"debug_functions" : False,
"debug_layers": False, # Since we are on steroids this time, print everything.
"id" : 'main'
}
net = network( borrow = True,
verbose = verbose )
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_module ( type = 'visualizer',
params = visualizer_params,
verbose = verbose )
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = False )
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 20,
filter_size = (5,5),
pool_size = (2,2),
activation = ('maxout', 'maxout', 2),
batch_norm = True,
verbose = verbose
)
net.add_layer ( type = "conv_pool",
origin = "conv_pool_1",
id = "conv_pool_2",
num_neurons = 50,
filter_size = (3,3),
pool_size = (2,2),
activation = ('maxout', 'maxout', 2),
batch_norm = True,
dropout_rate = 0.5,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "conv_pool_2",
id = "dot_product_1",
num_neurons = 800,
activation = 'relu',
batch_norm = True,
dropout_rate = 0.5,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 800,
activation = 'relu',
batch_norm = True,
dropout_rate = 0.5,
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "cce",
datastream_origin = 'data',
verbose = verbose
)
learning_rates = (0.001, 0.01, 0.001, 0.0001, 0.00001)
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
net.train( epochs = (20, 20, 10, 5),
validate_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)
def lenet5(dataset=None, verbose=1):
"""
This function is a demo example of lenet5 from the infamous paper by Yann LeCun.
This is an example code. You should study this code rather than merely run it.
"""
optimizer_params = {
"momentum_type": 'false',
# false, polyak, nesterov
"momentum_params": (0.5, 0.95, 30),
# (mom_start, momentum_end, momentum_end_epoch)
"regularization": (0.00, 0.00),
# l1_coeff, l2_coeff, decisiveness (optional)
"optimizer_type": 'sgd',
# sgd, adagrad, rmsprop, adam
"id": "main"
}
dataset_params = {
"dataset": dataset,
"svm": False,
"n_classes": 10,
"id": 'mnist'
}
# intitialize the network
net = network(borrow=True, verbose=verbose)
# or you can add modules after you create the net.
net.add_module(type='optimizer', params=optimizer_params, verbose=verbose)
net.add_module(type='datastream', params=dataset_params, verbose=verbose)
# add an input layer
net.add_layer(
type="input",
id="input",
verbose=verbose,
datastream_origin='mnist', # if you didnt add a dataset module, now is
# the time.
mean_subtract=False)
# add first convolutional layer
net.add_layer(type="conv_pool",
origin="input",
id="conv_pool_1",
num_neurons=20,
filter_size=(5, 5),
pool_size=(2, 2),
activation='relu',
verbose=verbose)
net.add_layer(type="conv_pool",
origin="conv_pool_1",
id="conv_pool_2",
num_neurons=50,
filter_size=(3, 3),
pool_size=(2, 2),
activation='relu',
verbose=verbose)
net.add_layer(type="dot_product",
origin="conv_pool_2",
id="dot_product_1",
num_neurons=800,
activation='relu',
verbose=verbose)
net.add_layer(type="dot_product",
origin="dot_product_1",
id="dot_product_2",
num_neurons=800,
activation='relu',
verbose=verbose)
net.add_layer(type="classifier",
id="softmax",
origin="dot_product_2",
num_classes=10,
activation='softmax',
verbose=verbose)
net.add_layer(type="objective",
id="obj",
origin="softmax",
objective="nll",
datastream_origin='mnist',
verbose=verbose)
# objective provided by classifier layer
# nll-negative log likelihood,
# cce-categorical cross entropy,
# bce-binary cross entropy,
# hinge-hinge loss
learning_rates = (0.01, 0.05, 0.001)
# (initial_learning_rate, annealing, ft_learnint_rate)
# net.pretty_print() # this will print out the network.
net.cook(optimizer='main',
objective_layer='obj',
datastream='mnist',
classifier='softmax',
learning_rates=learning_rates,
verbose=verbose)
net.train(epochs=(20, 20),
ft_learning_rate=0.001,
validate_after_epochs=1,
training_accuracy=True,
show_progress=True,
early_terminate=True,
verbose=verbose)
net.test(verbose=verbose)
def shallow_autoencoder(dataset=None, verbose=1):
"""
This function is a demo example of a sparse shallow autoencoder.
This is an example code. You should study this code rather than merely run it.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
dataset_params = {
"dataset": dataset,
"type": 'x',
"id": 'data'
} # MNIST dataset.
visualizer_params = {
"root": '.',
"frequency": 1,
"sample_size": 32,
"rgb_filters": False,
"debug_functions": False,
"debug_layers": True,
"id": 'main'
} # A visualizer that saves as everything
# intitialize the network
optimizer_params = {
"momentum_type": 'polyak',
"momentum_params": (0.5, 0.95, 20),
"regularization": (0.0001, 0.0001),
"optimizer_type": 'adagrad',
"id": "main"
} # A typical optimizer.
net = network(borrow=True, verbose=verbose)
net.add_module(type='datastream', params=dataset_params, verbose=verbose)
net.add_module(type='visualizer',
params=visualizer_params,
verbose=verbose)
net.add_module(type='optimizer', params=optimizer_params, verbose=verbose)
# add an input layer
net.add_layer(
type="input",
id="input",
verbose=verbose,
origin='data', # if you didnt add a dataset module, now is
# the time.
mean_subtract=False)
net.add_layer(
type="flatten", origin="input", id="flatten",
verbose=verbose) # Setup for a dot product layer. This is also done
# by default if not provided.
net.add_layer(type="dot_product",
origin="flatten",
id="encoder",
num_neurons=64,
activation='tanh',
verbose=verbose)
net.add_layer(
type="dot_product",
origin="encoder",
id="decoder",
num_neurons=784,
activation='tanh',
input_params=[net.dropout_layers['encoder'].w.T, None],
# Use the same weights but transposed for decoder.
learnable=False,
# because we don't want to learn the weights of somehting already used in
# an optimizer, when reusing the weights, always use learnable as False
verbose=verbose)
# We still need to learn the newly created biases in the decoder layer, so add them to the
# Learnable parameters list before cooking
net.active_params.append(net.dropout_layers['decoder'].b)
net.add_layer(type="unflatten",
origin="decoder",
id="unflatten",
shape=(28, 28, 1),
verbose=verbose) # Unflatten produces a full image.
net.add_layer(type="merge",
origin=("input", "unflatten"),
id="merge",
layer_type="error",
error="rmse",
verbose=verbose) # Create an error that is pixelwise.
net.add_layer(
type="objective",
id="obj",
origin="merge", # this is useless anyway.
layer_type='value',
objective=net.layers['merge'].output,
datastream_origin='data',
verbose=verbose)
learning_rates = (0.001, 0.1, 0.001)
net.cook(objective_layers=['obj'],
datastream='data',
learning_rates=learning_rates,
verbose=verbose)
# from yann.utils.graph import draw_network
# draw_network(net.graph, filename = 'autoencoder.png')
net.pretty_print()
net.train(epochs=(10, 10),
validate_after_epochs=1,
training_accuracy=True,
show_progress=True,
early_terminate=True,
verbose=verbose)
def convolutional_autoencoder(dataset=None, verbose=1):
"""
This function is a demo example of a deep convolutional autoencoder.
This is an example code. You should study this code rather than merely run it.
This is also an example for using the deconvolutional layer or the transposed fractional stride
convolutional layers.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
dataset_params = {
"dataset": dataset,
"type": 'x',
"id": 'data'
} # MNIST dataset.
visualizer_params = {
"root": '.',
"frequency": 1,
"sample_size": 32,
"rgb_filters": False,
"debug_functions": False,
"debug_layers": True,
"id": 'main'
} # Visualizer as usual prints everything.
# intitialize the network
optimizer_params = {
"momentum_type": 'nesterov',
"momentum_params": (0.65, 0.95, 30),
"regularization": (0.0001, 0.0001),
"optimizer_type": 'rmsprop',
"id": "main"
} # A differnt optimizer but one nevertheless.
net = network(borrow=True, verbose=verbose)
net.add_module(type='datastream', params=dataset_params, verbose=verbose)
net.add_module(type='visualizer',
params=visualizer_params,
verbose=verbose)
net.add_module(type='optimizer', params=optimizer_params, verbose=verbose)
# add an input layer
net.add_layer(
type="input",
id="input",
verbose=verbose,
origin='data', # if you didnt add a dataset module, now is
# the time.
mean_subtract=True)
net.add_layer(
type="conv_pool",
origin="input",
id="conv",
num_neurons=20,
filter_size=(5, 5),
pool_size=(1, 1),
activation='tanh',
regularize=True,
#stride = (2,2),
verbose=verbose) # We add a convolutional layer
net.add_layer(
type="flatten", origin="conv", id="flatten",
verbose=verbose) # prepare for dot product layer, but can be done
# by default also.
net.add_layer(type="dot_product",
origin="flatten",
id="hidden-encoder",
num_neurons=1200,
activation='tanh',
dropout_rate=0.5,
regularize=True,
verbose=verbose)
net.add_layer(type="dot_product",
origin="hidden-encoder",
id="encoder",
num_neurons=128,
activation='tanh',
dropout_rate=0.5,
regularize=True,
verbose=verbose) # this is the codeword layer effectively.
net.add_layer(
type="dot_product",
origin="encoder",
id="decoder",
num_neurons=1200,
activation='tanh',
input_params=[net.dropout_layers['encoder'].w.T, None],
# Use the same weights but transposed for decoder.
learnable=False,
# because we don't want to learn the weights of somehting already used in
# an optimizer, when reusing the weights, always use learnable as False
dropout_rate=0.5,
verbose=verbose) # Symmetric layers ...
net.add_layer(
type="dot_product",
origin="decoder",
id="hidden-decoder",
num_neurons=net.layers['flatten'].output_shape[1],
activation='tanh',
input_params=[net.dropout_layers['hidden-encoder'].w.T, None],
# Use the same weights but transposed for decoder.
learnable=False,
# because we don't want to learn the weights of somehting already used in
# an optimizer, when reusing the weights, always use learnable as False
dropout_rate=0.5,
verbose=verbose
) # Notice how we used the ``output_shape`` variables to
# assign parameter size.
net.add_layer(type="unflatten",
origin="hidden-decoder",
id="unflatten",
shape=(net.layers['conv'].output_shape[2],
net.layers['conv'].output_shape[3], 20),
verbose=verbose) # Prepare for deconvolutional layers..
net.add_layer(
type="deconv",
origin="unflatten",
id="deconv",
num_neurons=20,
filter_size=(5, 5),
pool_size=(1, 1),
output_shape=(28, 28, 1),
activation='tanh',
input_params=[net.dropout_layers['conv'].w, None],
learnable=False,
#stride = (2,2),
verbose=verbose)
# We still need to learn the newly created biases in the decoder layer, so add them to the
# Learnable parameters list before cooking
net.active_params.append(net.dropout_layers['hidden-decoder'].b)
net.active_params.append(net.dropout_layers['decoder'].b)
net.active_params.append(net.dropout_layers['deconv'].b)
net.add_layer(type="merge",
origin=("input", "deconv"),
id="merge",
layer_type="error",
error="rmse",
verbose=verbose)
net.add_layer(
type="objective",
id="obj",
origin="merge", # this is useless anyway.
layer_type='value',
objective=net.layers['merge'].output,
datastream_origin='data',
verbose=verbose)
learning_rates = (0.04, 0.0001, 0.00001)
net.cook(objective_layers=['obj'],
datastream='data',
learning_rates=learning_rates,
verbose=verbose)
# from yann.utils.graph import draw_network
# draw_network(net.graph, filename = 'autoencoder.png')
net.pretty_print()
net.train(epochs=(10, 10),
validate_after_epochs=1,
training_accuracy=True,
show_progress=True,
early_terminate=True,
verbose=verbose)
def update_phantom_labeler(self, temperature=3, verbose=2):
"""
This method sets up the phantom labeler network which is basically the same
network that takes the GAN as input and produces softmaxes.
"""
if verbose >= 2:
print(".. Creating the labeler network")
self.phantom_labeler = network()
if not temperature is None:
self.temperature = temperature
# Every layer should have increment number of layers. But the same
# parameters are shared. So it is in effect just one layer.
for inc in xrange(len(self.gans.keys())):
self.phantom_labeler.add_layer(
type="tensor",
id="input-" + str(inc),
input=self.gans[inc].dropout_layers['G(z)'].output,
input_shape=(self.mini_batch_size, 3072),
verbose=verbose)
self.phantom_labeler.add_layer(type="unflatten",
id="input-unflattened-" + str(inc),
origin="input-" + str(inc),
shape=(32, 32, 3),
verbose=verbose)
self.phantom_labeler.add_layer(
type="conv_pool",
id="c1-" + str(inc),
origin="input-unflattened-" + str(inc),
num_neurons=20,
filter_size=(5, 5),
pool_size=(2, 2),
activation='relu',
regularize=True,
batch_norm=True,
input_params=self.base.dropout_layers['c1'].params,
verbose=verbose)
self.phantom_labeler.add_layer(
type="conv_pool",
id="c2-" + str(inc),
origin="c1-" + str(inc),
num_neurons=50,
filter_shape=(3, 3),
pool_size=(2, 2),
batch_norm=True,
regularize=True,
activation='relu',
input_params=self.base.dropout_layers['c2'].params,
verbose=verbose)
self.phantom_labeler.add_layer(
type="dot_product",
origin="c2-" + str(inc),
id="fc1-" + str(inc),
num_neurons=800,
activation='relu',
batch_norm=True,
dropout_rate=0.5,
regularize=True,
input_params=self.base.dropout_layers['fc1'].params,
verbose=verbose)
self.phantom_labeler.add_layer(
type="dot_product",
origin="fc1-" + str(inc),
id="fc2-" + str(inc),
num_neurons=800,
activation='relu',
input_params=self.base.dropout_layers['fc2'].params,
batch_norm=True,
dropout_rate=0.5,
regularize=True,
verbose=verbose)
self.phantom_labeler.add_layer(
type="classifier",
id="softmax-" + str(inc),
origin="fc2-" + str(inc),
num_classes=self.base.dropout_layers['softmax'].
output_shape[1],
activation='softmax',
input_params=self.base.dropout_layers['softmax'].params,
regularize=True,
verbose=verbose)
self.phantom_labeler.add_layer(
type="classifier",
id="phantom-" + str(inc),
origin="fc2-" + str(inc),
num_classes=self.base.dropout_layers['softmax'].
output_shape[1],
activation=('softmax', self.temperature),
input_params=self.base.dropout_layers['softmax'].params,
regularize=True,
verbose=verbose)
#self.phantom_labeler.pretty_print()
draw_network(self.phantom_labeler.graph,
filename='phantom_labeler.png')
def _mlp(self,
id,
dataset=None,
root='.',
params=None,
num_classes=None,
cook=True,
verbose=1):
"""
This method is initializes and trains an MLP on some dataset.
Args:
root: save location for data
params: Initialize network with params.
cook: <True> If False, won't cook.
increment: which increment of MLP should be trained.
id: For directory setup.
dataset: an already created dataset.
"""
if verbose >= 2:
print(".. Creating the MLP network")
if dataset is None:
dataset = self.base_dataset
if num_classes is None:
num_classes = self.num_classes
input_params = None
optimizer_params = {
"momentum_type": 'false',
"momentum_params": (0.65, 0.9, 30),
"regularization": (0.0001, 0.0001),
"optimizer_type": 'adam',
"id": "optim"
}
dataset_params = {
"dataset": dataset,
"svm": False,
"n_classes": num_classes,
"id": 'data'
}
visualizer_params = {
"root": root + '/visualizer/network-' + id,
"frequency": 1,
"sample_size": 225,
"rgb_filters": True,
"debug_functions": False,
"debug_layers": False,
"id": 'visualizer'
}
resultor_params = {
"root": root + "/resultor/network-" + id,
"id": "resultor"
}
net = network(borrow=True, verbose=verbose)
net.add_module(type='optimizer',
params=optimizer_params,
verbose=verbose)
net.add_module(type='datastream',
params=dataset_params,
verbose=verbose)
net.add_module(type='visualizer',
params=visualizer_params,
verbose=verbose)
net.add_module(type='resultor',
params=resultor_params,
verbose=verbose)
net.add_layer(type="input",
id="input",
verbose=verbose,
datastream_origin='data')
if not params is None:
input_params = params['c1']
net.add_layer(type="conv_pool",
id="c1",
origin="input",
num_neurons=20,
filter_size=(5, 5),
pool_size=(2, 2),
activation='relu',
regularize=True,
batch_norm=True,
input_params=input_params,
verbose=verbose)
if not params is None:
input_params = params['c2']
net.add_layer(type="conv_pool",
id="c2",
origin="c1",
num_neurons=50,
filter_shape=(3, 3),
pool_size=(2, 2),
batch_norm=True,
regularize=True,
activation='relu',
input_params=input_params,
verbose=verbose)
if not params is None:
input_params = params['fc1']
net.add_layer(type="dot_product",
origin="c2",
id="fc1",
num_neurons=800,
activation='relu',
batch_norm=True,
regularize=True,
dropout_rate=0.5,
input_params=input_params,
verbose=verbose)
if not params is None:
input_params = params['fc2']
net.add_layer(type="dot_product",
origin="fc1",
id="fc2",
num_neurons=800,
activation='relu',
batch_norm=True,
dropout_rate=0.5,
regularize=True,
input_params=input_params,
verbose=verbose)
if not params is None:
input_params = params['softmax']
net.add_layer(type="classifier",
id="softmax",
origin="fc2",
num_classes=num_classes,
activation='softmax',
regularize=True,
input_params=input_params,
verbose=verbose)
net.add_layer(type="objective",
id="obj",
origin="softmax",
verbose=verbose)
# self.base.pretty_print()
# draw_network(gan_net.graph, filename = 'base.png')
if cook is True:
net.cook(optimizer='optim',
objective_layers=['obj'],
datastream='data',
classifier='softmax',
verbose=verbose)
return net
def log_reg ( dataset, verbose ):
"""
This function is a demo example of multi-layer neural networks from the infamous paper by
Yann LeCun. This is an example code. You should study this code rather than merely run it.
"""
optimizer_params = {
"momentum_type" : 'false',
# false, polyak, nesterov
"momentum_params" : (0.5, 0.95, 30),
# (mom_start, momentum_end, momentum_end_epoch)
"regularization" : (0.000, 0.000),
# l1_coeff, l2_coeff, decisiveness (optional)
"optimizer_type" : 'sgd',
# sgd, adagrad, rmsprop, adam
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data',
}
# intitialize the network
net = network( verbose = verbose )
# or you can add modules after you create the net.
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data',
# if you didnt add a dataset module, now is the time.
mean_subtract = False )
net.add_layer ( type = "classifier",
id = "softmax",
origin = "input",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "nll",
origin = "softmax",
objective = 'nll',
verbose = verbose
)
# objective provided by classifier layer
# nll-negative log likelihood,
# cce-categorical cross entropy,
# bce-binary cross entropy,
# hinge-hinge loss
learning_rates = (1, 0.01, 0.001)
# (initial_learning_rate, annealing, ft_learnint_rate)
net.cook( optimizer = 'main',
objective_layer = 'nll',
datastream = 'data',
classifier = 'softmax',
learning_rates = learning_rates,
verbose = verbose
)
# visualization of the network.
draw_network(net.graph, filename = 'log_reg.pdf')
net.train( epochs = (10, 5),
validate_after_epochs = 1,
training_accuracy = True,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test( show_progress = True,
verbose = verbose)
def lenet5 ( dataset= None, verbose = 1 ):
"""
This function is a demo example of lenet5 from the infamous paper by Yann LeCun.
This is an example code. You should study this code rather than merely run it.
Warning:
This is not the exact implementation but a modern re-incarnation.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'polyak',
"momentum_params" : (0.65, 0.97, 30),
"optimizer_type" : 'adagrad',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
visualizer_params = {
"root" : 'lenet5',
"frequency" : 1,
"sample_size": 144,
"rgb_filters": True,
"debug_functions" : False,
"debug_layers": False, # Since we are on steroids this time, print everything.
"id" : 'main'
}
# intitialize the network
net = network( borrow = True,
verbose = verbose )
# or you can add modules after you create the net.
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_module ( type = 'visualizer',
params = visualizer_params,
verbose = verbose
)
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = False )
# add first convolutional layer
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 20,
filter_size = (5,5),
pool_size = (2,2),
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "conv_pool",
origin = "conv_pool_1",
id = "conv_pool_2",
num_neurons = 50,
filter_size = (3,3),
pool_size = (2,2),
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "conv_pool_2",
id = "dot_product_1",
num_neurons = 1250,
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 1250,
activation = 'relu',
regularize = True,
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
regularize = True,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "nll",
datastream_origin = 'data',
regularization = (0.0001, 0.0001),
verbose = verbose
)
learning_rates = (0.05, 0.01, 0.001)
net.pretty_print()
draw_network(net.graph, filename = 'lenet.png')
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier_layer = 'softmax',
verbose = verbose
)
net.train( epochs = (40, 40),
validate_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)
def lenet_maxout_batchnorm_after_activation ( dataset= None, verbose = 1 ):
"""
This is a version with nesterov momentum and rmsprop instead of the typical sgd.
This also has maxout activations for convolutional layers, dropouts on the last
convolutional layer and the other dropout layers and this also applies batch norm
to all the layers. The difference though is that we use the ``batch_norm`` layer to apply
batch norm that applies batch norm after the activation fo the previous layer.
So we just spice things up and add a bit of steroids to
:func:`lenet5`. This also introduces a visualizer module usage.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
optimizer_params = {
"momentum_type" : 'nesterov',
"momentum_params" : (0.75, 0.95, 30),
"optimizer_type" : 'rmsprop',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
visualizer_params = {
"root" : 'lenet_bn_after',
"frequency" : 1,
"sample_size": 32,
"rgb_filters": True,
"debug_functions" : False,
"debug_layers": False, # Since we are on steroids this time, print everything.
"id" : 'main'
}
net = network( borrow = True,
verbose = verbose )
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_module ( type = 'visualizer',
params = visualizer_params,
verbose = verbose )
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
origin = 'data' )
net.add_layer ( type = "conv_pool",
origin = "input",
id = "conv_pool_1",
num_neurons = 40,
filter_size = (5,5),
pool_size = (2,2),
activation = ('maxout', 'maxout', 2),
batch_norm = False,
regularize = True,
verbose = verbose
)
net.add_layer ( type = 'batch_norm',
origin = 'conv_pool_1',
id = 'batch_norm_after_cp_1',
)
net.add_layer ( type = "convolution",
origin = "batch_norm_after_cp_1",
id = "conv_pool_2",
num_neurons = 100,
filter_size = (3,3),
pool_size = (2,2),
activation = ('maxout', 'maxout', 2),
batch_norm = False,
regularize = True,
verbose = verbose
)
net.add_layer ( type = 'batchnorm',
origin = 'conv_pool_2',
id = 'batch_norm_after_cp_2',
)
net.add_layer ( type = "dot_product",
origin = "batch_norm_after_cp_2",
id = "dot_product_1",
num_neurons = 1250,
activation = 'relu',
dropout_rate = 0.5,
batch_norm = False,
regularize = True,
verbose = verbose
)
net.add_layer ( type = 'batchnorm',
origin = 'dot_product_1',
id = 'batch_norm_after_dp_1',
)
net.add_layer ( type = "dot_product",
origin = "batch_norm_after_dp_1",
id = "dot_product_2",
num_neurons = 1250,
activation = 'relu',
dropout_rate = 0.5,
regularize = True,
batch_norm = False,
verbose = verbose
)
net.add_layer ( type = 'batch_norm',
origin = 'dot_product_2',
id = 'batch_norm_after_dp_2',
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "batch_norm_after_dp_2",
num_classes = 10,
regularize = True,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "obj",
origin = "softmax",
objective = "nll",
regularization = (0.0001, 0.0001),
datastream_origin = 'data',
verbose = verbose
)
learning_rates = (0.05, 0.001, 0.0001)
net.cook( optimizer = 'main',
objective_layer = 'obj',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
draw_network(net.graph, filename = 'lenet.png')
net.pretty_print()
net.train( epochs = (40, 40),
validate_after_epochs = 1,
visualize_after_epochs = 1,
training_accuracy = True,
learning_rates = learning_rates,
show_progress = True,
early_terminate = True,
verbose = verbose)
net.test(verbose = verbose)
def convolutional_autoencoder ( dataset= None, expNo=0, codeWords=128, verbose = 1 ):
"""
This function is a demo example of a deep convolutional autoencoder.
This is an example code. You should study this code rather than merely run it.
This is also an example for using the deconvolutional layer or the transposed fractional stride
convolutional layers.
Args:
dataset: Supply a dataset.
verbose: Similar to the rest of the dataset.
"""
exp_no = str(expNo) + '_autoencoders_cnn_tutorial_'+ str(codeWords) + dataset
root_dir = './experiments/' + exp_no
dataset_params = {
"dataset" : dataset,
"type" : 'x',
"id" : 'data'
}
print dataset_params
visualizer_params = {
"root" : root_dir,
"frequency" : 1,
"sample_size": 32,
"rgb_filters": False,
"debug_functions" : False,
"debug_layers": True,
"id" : 'main'
}
print visualizer_params
# intitialize the network
optimizer_params = {
"momentum_type" : 'nesterov',
"momentum_params" : (0.65, 0.95, 30),
"regularization" : (0.0001, 0.0001),
"optimizer_type" : 'rmsprop',
"id" : "main"
}
print optimizer_params
net = network( borrow = True,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_module ( type = 'visualizer',
params = visualizer_params,
verbose = verbose
)
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
# add an input layer
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = True )
net.add_layer(type = 'random',
id = 'rand_layer',
num_neurons = (100,net.layers['input'].output_shape[2],
net.layers['input'].output_shape[3],),
distribution = 'normal',
mu = 0,
sigma = 1,
verbose = verbose)
net.add_layer ( type = "merge",
origin = ("input","rand_layer"),
id = "merge_in",
layer_type = "sum",
verbose = verbose)
net.add_layer ( type = "conv_pool",
origin = "merge_in",
id = "conv",
num_neurons = 20,
filter_size = (5,5),
pool_size = (1,1),
activation = 'relu',
regularize = True,
#stride = (2,2),
verbose = verbose
)
net.add_layer ( type = "flatten",
origin = "conv",
id = "flatten",
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "flatten",
id = "hidden-encoder",
num_neurons = 1200,
activation = 'relu',
dropout_rate = 0.5,
regularize = True,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "hidden-encoder",
id = "encoder",
num_neurons = codeWords,
activation = 'relu',
dropout_rate = 0.5,
regularize = True,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "encoder",
id = "decoder",
num_neurons = 1200,
activation = 'relu',
input_params = [net.dropout_layers['encoder'].w.T, None],
# Use the same weights but transposed for decoder.
learnable = False,
# because we don't want to learn the weights of somehting already used in
# an optimizer, when reusing the weights, always use learnable as False
dropout_rate = 0.5,
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "decoder",
id = "hidden-decoder",
num_neurons = net.layers['flatten'].output_shape[1],
activation = 'relu',
input_params = [net.dropout_layers['hidden-encoder'].w.T, None],
# Use the same weights but transposed for decoder.
learnable = False,
# because we don't want to learn the weights of somehting already used in
# an optimizer, when reusing the weights, always use learnable as False
dropout_rate = 0.5,
verbose = verbose
)
net.add_layer ( type = "unflatten",
origin = "hidden-decoder",
id = "unflatten",
shape = (net.layers['conv'].output_shape[2],
net.layers['conv'].output_shape[3],
20),
verbose = verbose
)
net.add_layer ( type = "deconv",
origin = "unflatten",
id = "deconv",
num_neurons = 20,
filter_size = (5,5),
pool_size = (1,1),
output_shape = (28,28,1),
activation = 'relu',
input_params = [net.dropout_layers['conv'].w, None],
learnable = False,
#stride = (2,2),
verbose = verbose
)
# We still need to learn the newly created biases in the decoder layer, so add them to the
# Learnable parameters list before cooking
net.active_params.append(net.dropout_layers['hidden-decoder'].b)
net.active_params.append(net.dropout_layers['decoder'].b)
net.active_params.append(net.dropout_layers['deconv'].b)
net.add_layer ( type = "merge",
origin = ("input","deconv"),
id = "merge",
layer_type = "error",
error = "rmse",
verbose = verbose)
net.add_layer ( type = "objective",
id = "obj",
origin = "merge", # this is useless anyway.
layer_type = 'value',
objective = net.layers['merge'].output,
datastream_origin = 'data',
verbose = verbose
)
learning_rates = (0, 0.01, 0.001)
net.cook( objective_layers = ['obj'],
datastream = 'data',
learning_rates = learning_rates,
verbose = verbose
)
# from yann.utils.graph import draw_network
# draw_network(net.graph, filename = 'autoencoder.png')
net.pretty_print()
net.train( epochs = (10, 10),
validate_after_epochs = 1,
training_accuracy = True,
show_progress = True,
early_terminate = True,
verbose = verbose)
def mlp(dataset, verbose=1):
"""
This method is a tutorial on building a two layer multi-layer neural network. The built
network is mnist->800->800->10 .It optimizes with polyak momentum and rmsprop.
Args:
dataset: an already created dataset.
"""
optimizer_params = {
"momentum_type": 'polyak',
"momentum_params": (0.65, 0.9, 30),
"regularization": (0.0001, 0.0001),
"optimizer_type": 'adagrad',
"id": "main"
}
dataset_params = {
"dataset": dataset,
"svm": False,
"n_classes": 10,
"id": 'data'
}
net = network(borrow=True, verbose=verbose)
net.add_module(type='optimizer', params=optimizer_params, verbose=verbose)
net.add_module(type='datastream', params=dataset_params, verbose=verbose)
net.add_layer(type="input",
id="input",
verbose=verbose,
datastream_origin='data')
net.add_layer(type="dot_product",
origin="input",
id="dot_product_1",
num_neurons=800,
activation='relu',
verbose=verbose)
net.add_layer(type="dot_product",
origin="dot_product_1",
id="dot_product_2",
num_neurons=800,
activation='relu',
verbose=verbose)
net.add_layer(type="classifier",
id="softmax",
origin="dot_product_2",
num_classes=10,
activation='softmax',
verbose=verbose)
net.add_layer(type="objective",
id="obj",
origin="softmax",
verbose=verbose)
learning_rates = (0.05, 0.01, 0.001)
net.cook(optimizer='main',
datastream='data',
classifier='softmax',
verbose=verbose)
# from yann.utils.graph import draw_network
# draw_network(net.graph, filename = 'mlp.png')
net.pretty_print()
net.train(epochs=(20, 20),
validate_after_epochs=1,
training_accuracy=True,
show_progress=True,
early_terminate=True,
learning_rates=learning_rates,
verbose=verbose)
net.test(verbose=verbose)
def mlp ( dataset, verbose = 1 ):
"""
This method is a tutorial on building a two layer multi-layer neural network. The built
network is mnist->800->800->10 .It optimizes with polyak momentum and rmsprop.
Args:
dataset: an already created dataset.
"""
optimizer_params = {
"momentum_type" : 'polyak',
"momentum_params" : (0.9, 0.95, 30),
"regularization" : (0.0001, 0.0001),
"optimizer_type" : 'adagrad',
"id" : "main"
}
dataset_params = {
"dataset" : dataset,
"svm" : False,
"n_classes" : 10,
"id" : 'data'
}
net = network( borrow = True,
verbose = verbose )
net.add_module ( type = 'optimizer',
params = optimizer_params,
verbose = verbose )
net.add_module ( type = 'datastream',
params = dataset_params,
verbose = verbose )
net.add_layer ( type = "input",
id = "input",
verbose = verbose,
datastream_origin = 'data', # if you didnt add a dataset module, now is
# the time.
mean_subtract = True )
net.add_layer ( type = "dot_product",
origin = "input",
id = "dot_product_1",
num_neurons = 800,
activation = 'relu',
verbose = verbose
)
net.add_layer ( type = "dot_product",
origin = "dot_product_1",
id = "dot_product_2",
num_neurons = 800,
activation = 'relu',
verbose = verbose
)
net.add_layer ( type = "classifier",
id = "softmax",
origin = "dot_product_2",
num_classes = 10,
activation = 'softmax',
verbose = verbose
)
net.add_layer ( type = "objective",
id = "nll",
origin = "softmax",
verbose = verbose
)
learning_rates = (0.05, 0.01, 0.001)
net.cook( optimizer = 'main',
objective_layer = 'nll',
datastream = 'data',
classifier = 'softmax',
verbose = verbose
)
draw_network(net.graph, filename = 'mlp.png')
net.train( epochs = (20, 20),
validate_after_epochs = 1,
training_accuracy = True,
show_progress = True,
early_terminate = True,
learning_rates = learning_rates,
verbose = verbose)
net.test(verbose = verbose)
def mlp(dataset, verbose=1):
"""
This method is a tutorial on building a two layer multi-layer neural network. The built
network is mnist->800->800->10 .It optimizes with polyak momentum and rmsprop.
Args:
dataset: an already created dataset.
"""
optimizer_params = {
"momentum_type": 'false',
# false, polyak, nesterov
"momentum_params": (0.9, 0.95, 30),
# (mom_start, momentum_end, momentum_end_epoch)
"regularization": (0.0001, 0.0001),
# l1_coeff, l2_coeff, decisiveness (optional)
"optimizer_type": 'sgd',
# sgd, adagrad, rmsprop, adam
"id": "main"
}
dataset_params = {
"dataset": dataset,
"svm": False,
"n_classes": 10,
"id": 'mnist'
}
# intitialize the network
net = network(borrow=True, verbose=verbose)
# or you can add modules after you create the net.
net.add_module(type='optimizer', params=optimizer_params, verbose=verbose)
net.add_module(type='datastream', params=dataset_params, verbose=verbose)
# add an input layer
net.add_layer(
type="input",
id="input",
verbose=verbose,
datastream_origin='mnist', # if you didnt add a dataset module, now is
# the time.
mean_subtract=True)
# add first convolutional laye
net.add_layer(type="dot_product",
origin="input",
id="dot_product_1",
num_neurons=800,
activation='relu',
verbose=verbose)
net.add_layer(type="dot_product",
origin="dot_product_1",
id="dot_product_2",
num_neurons=800,
activation='relu',
verbose=verbose)
net.add_layer(type="classifier",
id="softmax",
origin="dot_product_2",
num_classes=10,
activation='softmax',
verbose=verbose)
net.add_layer(type="objective",
id="nll",
origin="softmax",
verbose=verbose)
# objective provided by classifier layer
# nll-negative log likelihood,
# cce-categorical cross entropy,
# bce-binary cross entropy,
# hinge-hinge loss
learning_rates = (0.01, 0.05, 0.001)
# (initial_learning_rate, annealing, ft_learnint_rate)
net.cook(optimizer='main',
objective_layer='nll',
datastream='mnist',
classifier='softmax',
learning_rates=learning_rates,
verbose=verbose)
net.train(epochs=(20, 20),
ft_learning_rate=0.001,
validate_after_epochs=1,
training_accuracy=True,
show_progress=True,
early_terminate=True,
verbose=verbose)
net.test(verbose=verbose)