from opendeep.data.standard_datasets.image.mnist import MNIST
from opendeep.optimization.adadelta import AdaDelta
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
from opendeep.log import config_root_logger
config_root_logger()
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = Dense(inputs=((None, 28 * 28), matrix("x")),
outputs=1000,
activation='linear')
layer1_act = Activation(inputs=((None, 1000), layer1.get_outputs()),
activation='relu')
# create the softmax classifier
layer2 = Softmax(inputs=((None, 1000), layer1_act.get_outputs()),
outputs=10,
out_as_probs=True)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer1_act, layer2])
# define the loss function
loss = Neg_LL(inputs=mlp.get_outputs(),
targets=vector("y", dtype="int64"),
one_hot=False)
# make an optimizer to train it (AdaDelta is a good default)
# optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20)
optimizer = AdaDelta(dataset=mnist, loss=loss, epochs=20)
from opendeep.monitor import Plot, Monitor, BOKEH_AVAILABLE
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
from opendeep.log import config_root_logger
config_root_logger()
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = Dense(inputs=((None, 28*28), matrix("x")),
outputs=1000,
activation='linear')
layer1_act = Activation(inputs=((None, 1000), layer1.get_outputs()), activation='relu')
# create the softmax classifier
layer2 = Softmax(inputs=((None, 1000), layer1_act.get_outputs()),
outputs=10,
out_as_probs=True)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer1_act, layer2])
# define the loss function
loss = Neg_LL(inputs=mlp.get_outputs(), targets=vector("y", dtype="int64"), one_hot=False)
#plot the loss
if BOKEH_AVAILABLE:
plot = Plot("mlp_mnist", monitor_channels=Monitor("loss", loss.get_loss()), open_browser=True)
else:
plot = None
def _build_computation_graph(self):
###################### BUILD NETWORK ##########################
# whether or not to mirror the input images before feeding them into the network
if self.flag_datalayer:
layer_1_input = mirror_images(
input=self.x,
image_shape=(self.batch_size, 3, 256, 256), # bc01 format
cropsize=227,
rand=self.rand,
flag_rand=self.rand_crop)
else:
layer_1_input = self.x # 4D tensor (going to be in bc01 format)
# Start with 5 convolutional pooling layers
log.debug("convpool layer 1...")
convpool_layer1 = ConvPoolLayer(inputs_hook=((self.batch_size, 3, 227,
227), layer_1_input),
filter_shape=(96, 3, 11, 11),
convstride=4,
padsize=0,
group=1,
poolsize=3,
poolstride=2,
bias_init=0.0,
local_response_normalization=True)
# Add this layer's parameters!
self.params += convpool_layer1.get_params()
log.debug("convpool layer 2...")
convpool_layer2 = ConvPoolLayer(inputs_hook=((
self.batch_size,
96,
27,
27,
), convpool_layer1.get_outputs()),
filter_shape=(256, 96, 5, 5),
convstride=1,
padsize=2,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.1,
local_response_normalization=True)
# Add this layer's parameters!
self.params += convpool_layer2.get_params()
log.debug("convpool layer 3...")
convpool_layer3 = ConvPoolLayer(
inputs_hook=((self.batch_size, 256, 13, 13),
convpool_layer2.get_outputs()),
filter_shape=(384, 256, 3, 3),
convstride=1,
padsize=1,
group=1,
poolsize=1,
poolstride=0,
bias_init=0.0,
local_response_normalization=False)
# Add this layer's parameters!
self.params += convpool_layer3.get_params()
log.debug("convpool layer 4...")
convpool_layer4 = ConvPoolLayer(
inputs_hook=((self.batch_size, 384, 13, 13),
convpool_layer3.get_outputs()),
filter_shape=(384, 384, 3, 3),
convstride=1,
padsize=1,
group=2,
poolsize=1,
poolstride=0,
bias_init=0.1,
local_response_normalization=False)
# Add this layer's parameters!
self.params += convpool_layer4.get_params()
log.debug("convpool layer 5...")
convpool_layer5 = ConvPoolLayer(
inputs_hook=((self.batch_size, 384, 13, 13),
convpool_layer4.get_outputs()),
filter_shape=(256, 384, 3, 3),
convstride=1,
padsize=1,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.0,
local_response_normalization=False)
# Add this layer's parameters!
self.params += convpool_layer5.get_params()
# Now onto the fully-connected layers!
fc_config = {
'activation':
'rectifier', # type of activation function to use for output
'weights_init':
'gaussian', # either 'gaussian' or 'uniform' - how to initialize weights
'weights_mean': 0.0, # mean for gaussian weights init
'weights_std':
0.005, # standard deviation for gaussian weights init
'bias_init': 0.0 # how to initialize the bias parameter
}
log.debug("fully connected layer 1 (model layer 6)...")
# we want to have dropout applied to the training version, but not the test version.
fc_layer6_input = T.flatten(convpool_layer5.get_outputs(), 2)
fc_layer6 = Dense(inputs_hook=(9216, fc_layer6_input),
output_size=4096,
noise='dropout',
noise_level=0.5,
**fc_config)
# Add this layer's parameters!
self.params += fc_layer6.get_params()
# Add the dropout noise switch
self.noise_switches += fc_layer6.get_noise_switch()
log.debug("fully connected layer 2 (model layer 7)...")
fc_layer7 = Dense(inputs_hook=(4096, fc_layer6.get_outputs()),
output_size=4096,
noise='dropout',
noise_level=0.5,
**fc_config)
# Add this layer's parameters!
self.params += fc_layer7.get_params()
# Add the dropout noise switch
self.noise_switches += fc_layer7.get_noise_switch()
# last layer is a softmax prediction output layer
softmax_config = {
'weights_init': 'gaussian',
'weights_mean': 0.0,
'weights_std': 0.005,
'bias_init': 0.0
}
log.debug("softmax classification layer (model layer 8)...")
softmax_layer8 = SoftmaxLayer(inputs_hook=(4096,
fc_layer7.get_outputs()),
output_size=1000,
**softmax_config)
# Add this layer's parameters!
self.params += softmax_layer8.get_params()
# finally the softmax output from the whole thing!
self.output = softmax_layer8.get_outputs()
self.targets = softmax_layer8.get_targets()
#####################
# Cost and monitors #
#####################
self.train_cost = softmax_layer8.negative_log_likelihood()
cost = softmax_layer8.negative_log_likelihood()
errors = softmax_layer8.errors()
train_errors = softmax_layer8.errors()
self.monitors = OrderedDict([('cost', cost), ('errors', errors),
('dropout_errors', train_errors)])
#########################
# Compile the functions #
#########################
log.debug("Compiling functions!")
t = time.time()
log.debug("f_run...")
# use the actual argmax from the classification
self.f_run = function(inputs=[self.x],
outputs=softmax_layer8.get_argmax_prediction())
log.debug("compilation took %s",
make_time_units_string(time.time() - t))
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
from opendeep.log import config_root_logger
config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating MLP!")
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = Dense(input_size=28 * 28, output_size=1000, activation='relu')
# create the softmax classifier
layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()),
output_size=10,
out_as_probs=False)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer2])
# make an optimizer to train it (AdaDelta is a good default)
# optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20)
optimizer = AdaDelta(dataset=mnist, epochs=20)
# perform training!
# optimizer.train()
mlp.train(optimizer)
# test it on some images!
test_data, test_labels = mnist.test_inputs, mnist.test_targets
test_data = test_data[:25]
test_labels = test_labels[:25]
def _build_computation_graph(self):
###################### BUILD NETWORK ##########################
# whether or not to mirror the input images before feeding them into the network
if self.flag_datalayer:
layer_1_input = mirror_images(input=self.x,
image_shape=(self.batch_size, 3, 256, 256), # bc01 format
cropsize=227,
rand=self.rand,
flag_rand=self.rand_crop)
else:
layer_1_input = self.x # 4D tensor (going to be in bc01 format)
# Start with 5 convolutional pooling layers
log.debug("convpool layer 1...")
convpool_layer1 = ConvPoolLayer(inputs_hook=((self.batch_size, 3, 227, 227), layer_1_input),
filter_shape=(96, 3, 11, 11),
convstride=4,
padsize=0,
group=1,
poolsize=3,
poolstride=2,
bias_init=0.0,
local_response_normalization=True)
# Add this layer's parameters!
self.params += convpool_layer1.get_params()
log.debug("convpool layer 2...")
convpool_layer2 = ConvPoolLayer(inputs_hook=((self.batch_size, 96, 27, 27, ), convpool_layer1.get_outputs()),
filter_shape=(256, 96, 5, 5),
convstride=1,
padsize=2,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.1,
local_response_normalization=True)
# Add this layer's parameters!
self.params += convpool_layer2.get_params()
log.debug("convpool layer 3...")
convpool_layer3 = ConvPoolLayer(inputs_hook=((self.batch_size, 256, 13, 13), convpool_layer2.get_outputs()),
filter_shape=(384, 256, 3, 3),
convstride=1,
padsize=1,
group=1,
poolsize=1,
poolstride=0,
bias_init=0.0,
local_response_normalization=False)
# Add this layer's parameters!
self.params += convpool_layer3.get_params()
log.debug("convpool layer 4...")
convpool_layer4 = ConvPoolLayer(inputs_hook=((self.batch_size, 384, 13, 13), convpool_layer3.get_outputs()),
filter_shape=(384, 384, 3, 3),
convstride=1,
padsize=1,
group=2,
poolsize=1,
poolstride=0,
bias_init=0.1,
local_response_normalization=False)
# Add this layer's parameters!
self.params += convpool_layer4.get_params()
log.debug("convpool layer 5...")
convpool_layer5 = ConvPoolLayer(inputs_hook=((self.batch_size, 384, 13, 13), convpool_layer4.get_outputs()),
filter_shape=(256, 384, 3, 3),
convstride=1,
padsize=1,
group=2,
poolsize=3,
poolstride=2,
bias_init=0.0,
local_response_normalization=False)
# Add this layer's parameters!
self.params += convpool_layer5.get_params()
# Now onto the fully-connected layers!
fc_config = {
'activation': 'rectifier', # type of activation function to use for output
'weights_init': 'gaussian', # either 'gaussian' or 'uniform' - how to initialize weights
'weights_mean': 0.0, # mean for gaussian weights init
'weights_std': 0.005, # standard deviation for gaussian weights init
'bias_init': 0.0 # how to initialize the bias parameter
}
log.debug("fully connected layer 1 (model layer 6)...")
# we want to have dropout applied to the training version, but not the test version.
fc_layer6_input = T.flatten(convpool_layer5.get_outputs(), 2)
fc_layer6 = Dense(inputs_hook=(9216, fc_layer6_input),
output_size=4096,
noise='dropout',
noise_level=0.5,
**fc_config)
# Add this layer's parameters!
self.params += fc_layer6.get_params()
# Add the dropout noise switch
self.noise_switches += fc_layer6.get_switches()
log.debug("fully connected layer 2 (model layer 7)...")
fc_layer7 = Dense(inputs_hook=(4096, fc_layer6.get_outputs()),
output_size=4096,
noise='dropout',
noise_level=0.5,
**fc_config)
# Add this layer's parameters!
self.params += fc_layer7.get_params()
# Add the dropout noise switch
self.noise_switches += fc_layer7.get_switches()
# last layer is a softmax prediction output layer
softmax_config = {
'weights_init': 'gaussian',
'weights_mean': 0.0,
'weights_std': 0.005,
'bias_init': 0.0
}
log.debug("softmax classification layer (model layer 8)...")
softmax_layer8 = SoftmaxLayer(inputs_hook=(4096, fc_layer7.get_outputs()),
output_size=1000,
**softmax_config)
# Add this layer's parameters!
self.params += softmax_layer8.get_params()
# finally the softmax output from the whole thing!
self.output = softmax_layer8.get_outputs()
self.targets = softmax_layer8.get_targets()
#####################
# Cost and monitors #
#####################
self.train_cost = softmax_layer8.negative_log_likelihood()
cost = softmax_layer8.negative_log_likelihood()
errors = softmax_layer8.errors()
train_errors = softmax_layer8.errors()
self.monitors = OrderedDict([('cost', cost), ('errors', errors), ('dropout_errors', train_errors)])
#########################
# Compile the functions #
#########################
log.debug("Compiling functions!")
t = time.time()
log.debug("f_run...")
# use the actual argmax from the classification
self.f_run = function(inputs=[self.x], outputs=softmax_layer8.get_argmax_prediction())
log.debug("compilation took %s", make_time_units_string(time.time() - t))
if __name__ == '__main__':
# set up the logging environment to display outputs (optional)
# although this is recommended over print statements everywhere
import logging
from opendeep.log import config_root_logger
config_root_logger()
log = logging.getLogger(__name__)
log.info("Creating MLP!")
# grab the MNIST dataset
mnist = MNIST()
# create the basic layer
layer1 = Dense(input_size=28*28, output_size=1000, activation='relu')
# create the softmax classifier
layer2 = SoftmaxLayer(inputs_hook=(1000, layer1.get_outputs()), output_size=10, out_as_probs=False)
# create the mlp from the two layers
mlp = Prototype(layers=[layer1, layer2])
# make an optimizer to train it (AdaDelta is a good default)
# optimizer = AdaDelta(model=mlp, dataset=mnist, n_epoch=20)
optimizer = AdaDelta(dataset=mnist, epochs=20)
# perform training!
# optimizer.train()
mlp.train(optimizer)
# test it on some images!
test_data, test_labels = mnist.test_inputs, mnist.test_targets
test_data = test_data[:25]
test_labels = test_labels[:25]
# use the run function!
preds = mlp.run(test_data)