def main():
# First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
mlp = Prototype()
mlp.add(
Dense(input_size=28 * 28,
output_size=1000,
activation='rectifier',
noise='dropout'))
mlp.add(SoftmaxLayer(output_size=10))
# Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
# Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
# dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
# care about. By default, this method returns an empty dictionary - it was the model creator's job to
# include potential monitor values.
mlp_monitors = mlp.get_monitors()
mlp_channel = MonitorsChannel(name="error")
for name, expression in mlp_monitors.items():
mlp_channel.add(
Monitor(name=name,
expression=expression,
train=True,
valid=True,
test=True))
# create some monitors for statistics about the hidden and output weights!
# let's look at the mean, variance, and standard deviation of the weights matrices.
weights_channel = MonitorsChannel(name="weights")
hiddens_1 = mlp[0].get_params()[0]
hiddens1_mean = T.mean(hiddens_1)
weights_channel.add(
Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))
hiddens_2 = mlp[1].get_params()[0]
hiddens2_mean = T.mean(hiddens_2)
weights_channel.add(
Monitor(name="out_mean", expression=hiddens2_mean, train=True))
# create our plot object to do live plotting!
plot = Plot(bokeh_doc_name="Monitor Tutorial",
monitor_channels=[mlp_channel, weights_channel],
open_browser=True)
# use SGD optimizer
optimizer = SGD(model=mlp,
dataset=MNIST(concat_train_valid=False),
epochs=500,
save_freq=100,
batch_size=600,
learning_rate=.01,
lr_decay=False,
momentum=.9,
nesterov_momentum=True)
# train, with the plot!
optimizer.train(plot=plot)
def main():
# First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
mlp = Prototype()
mlp.add(Dense(input_size=28*28, output_size=1000, activation='rectifier', noise='dropout'))
mlp.add(SoftmaxLayer(output_size=10))
# Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
# Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
# dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
# care about. By default, this method returns an empty dictionary - it was the model creator's job to
# include potential monitor values.
mlp_monitors = mlp.get_monitors()
mlp_channel = MonitorsChannel(name="error")
for name, expression in mlp_monitors.items():
mlp_channel.add(Monitor(name=name, expression=expression, train=True, valid=True, test=True))
# create some monitors for statistics about the hidden and output weights!
# let's look at the mean, variance, and standard deviation of the weights matrices.
weights_channel = MonitorsChannel(name="weights")
hiddens_1 = mlp[0].get_params()[0]
hiddens1_mean = T.mean(hiddens_1)
weights_channel.add(Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))
hiddens_2 = mlp[1].get_params()[0]
hiddens2_mean = T.mean(hiddens_2)
weights_channel.add(Monitor(name="out_mean", expression=hiddens2_mean, train=True))
# create our plot object to do live plotting!
plot = Plot(bokeh_doc_name="Monitor Tutorial", monitor_channels=[mlp_channel, weights_channel], open_browser=True)
# use SGD optimizer
optimizer = SGD(model=mlp,
dataset=MNIST(concat_train_valid=False),
epochs=500,
save_freq=100,
batch_size=600,
learning_rate=.01,
lr_decay=False,
momentum=.9,
nesterov_momentum=True)
# train, with the plot!
optimizer.train(plot=plot)
def run_mlp():
# test the new way to automatically fill in inputs for models
mlp = Prototype()
x = ((None, 784), matrix("x"))
mlp.add(Dense(inputs=x, outputs=1000, activation='rectifier'))
mlp.add(Dense, outputs=1500, activation='tanh')
mlp.add(Softmax, outputs=10, out_as_probs=False)
# define our loss to optimize for the model (and the target variable)
# targets from MNIST are int64 numbers 0-9
y = lvector('y')
loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=y, one_hot=False)
mnist = MNIST()
optimizer = AdaDelta(model=mlp, loss=loss, dataset=mnist, epochs=10)
optimizer.train()
test_data, test_labels = mnist.test_inputs, mnist.test_targets
test_data = test_data[:25]
test_labels = test_labels[:25]
# use the run function!
yhat = mlp.run(test_data)
print('-------')
print('Prediction: %s' % str(yhat))
print('Actual: %s' % str(test_labels.astype('int32')))
def run_mlp():
# test the new way to automatically fill in inputs for models
mlp = Prototype()
x = ((None, 784), matrix("x"))
mlp.add(Dense(inputs=x, outputs=1000, activation='rectifier'))
mlp.add(Dense, outputs=1500, activation='tanh')
mlp.add(Softmax, outputs=10, out_as_probs=False)
# define our loss to optimize for the model (and the target variable)
# targets from MNIST are int64 numbers 0-9
y = lvector('y')
loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=y, one_hot=False)
mnist = MNIST()
optimizer = AdaDelta(model=mlp, loss=loss, dataset=mnist, epochs=10)
optimizer.train()
test_data, test_labels = mnist.test_inputs, mnist.test_targets
test_data = test_data[:25]
test_labels = test_labels[:25]
# use the run function!
yhat = mlp.run(test_data)
print('-------')
print('Prediction: %s' % str(yhat))
print('Actual: %s' % str(test_labels.astype('int32')))
def sequential_add_layers():
# This method is to demonstrate adding layers one-by-one to a Prototype container.
# As you can see, inputs_hook are created automatically by Prototype so we don't need to specify!
mlp = Prototype()
mlp.add(Dense(input_size=28*28, output_size=1000, activation='rectifier', noise='dropout', noise_level=0.5))
mlp.add(Dense(output_size=512, activation='rectifier', noise='dropout', noise_level=0.5))
mlp.add(SoftmaxLayer(output_size=10))
return mlp
# some debugging output to see what is going on under the hood
config_root_logger()
#########
# Model #
#########
# build a Prototype container to easily add layers and make a cohesive model!
lenet = Prototype()
# need to define a variable for the inputs to this model, as well as the shape
# (batch, channel, row, col)
images = T.tensor4('xs')
images_shape = (None, 1, 28, 28)
# conv/pool/droput layer 1
lenet.add(
Conv2D(inputs=(images_shape, images),
n_filters=20,
filter_size=(5, 5),
border_mode='full',
activation='relu'))
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# conv/pool/droput layer 2
lenet.add(Conv2D,
n_filters=50,
filter_size=(5, 5),
border_mode='full',
activation='relu')
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# reshape convolution output to be 2D to feed into fully-connected layers
# Prototype container keeps its layers in the `models` attribute list: grab the latest model output
dense_input = lenet.models[-1].get_outputs().flatten(2)
if __name__ == '__main__':
# some debugging output to see what is going on under the hood
config_root_logger()
#########
# Model #
#########
# build a Prototype container to easily add layers and make a cohesive model!
lenet = Prototype()
# need to define a variable for the inputs to this model, as well as the shape
# (batch, channel, row, col)
images = T.tensor4('xs')
images_shape = (None, 1, 28, 28)
# conv/pool/droput layer 1
lenet.add(
Conv2D(inputs=(images_shape, images),
n_filters=20, filter_size=(5, 5), border_mode='full', activation='relu')
)
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# conv/pool/droput layer 2
lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), border_mode='full', activation='relu')
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# reshape convolution output to be 2D to feed into fully-connected layers
# Prototype container keeps its layers in the `models` attribute list: grab the latest model output
dense_input = lenet.models[-1].get_outputs().flatten(2)
dense_in_shape = lenet.models[-1].output_size[:1] + (np.prod(lenet.models[-1].output_size[1:]), )
# now make the dense (fully-connected) layers!
lenet.add(Dense(inputs=(dense_in_shape, dense_input), outputs=500, activation='tanh'))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# softmax classification layer!
def testLeNet(self):
try:
# quick and dirty way to create a model from arbitrary layers
lenet = Prototype(outdir=None)
# our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
x = ((None, 1, 28, 28), ftensor4('x'))
# our first convolutional layer
lenet.add(
Conv2D(inputs=x, n_filters=20, filter_size=(5, 5),
outdir=None))
# our first pooling layer, automatically hooking inputs to the previous convolutional outputs
lenet.add(Pool2D, size=(2, 2))
# our second convolutional layer
lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), outdir=None)
# our second pooling layer
lenet.add(Pool2D, size=(2, 2))
# now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
lenet.add(Flatten, ndim=2)
# one dense hidden layer
lenet.add(Dense, outputs=500, activation='tanh', outdir=None)
# hook a softmax classification layer, outputting the probabilities.
lenet.add(Softmax, outputs=10, out_as_probs=True, outdir=None)
# Grab the MNIST dataset
data = MNIST(path="../../../datasets/{!s}".format(mnist_name),
concat_train_valid=False,
flatten=False)
# define our loss to optimize for the model (and the target variable)
# targets from MNIST are int64 numbers 0-9
y = lvector('y')
loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
# monitor
error_monitor = Monitor(
name='error',
expression=mean(neq(lenet.models[-1].y_pred, y)),
valid=True,
test=True,
out_service=FileService('outputs/lenet_error.txt'))
# optimize our model to minimize loss given the dataset using SGD
optimizer = SGD(model=lenet,
dataset=data,
loss=loss,
epochs=10,
batch_size=128,
learning_rate=.1,
momentum=False)
print("Training LeNet...")
optimizer.train(monitor_channels=error_monitor)
def test_subset(filename, expected, conf=0.001):
with open(filename, 'r') as f:
errs = [float(err) for err in f]
for i, (err, exp) in enumerate(zip(errs, expected)):
if i == 0:
c = conf * 10
else:
c = conf
self.assertTrue(
exp - c < round(err, 4) < exp + c,
"Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}"
.format(errs, expected, err, exp))
test_subset('outputs/lenet_error_train.txt', [
.0753, .0239, .0159, .0113, .0088, .0064, .0050, .0037, .0026,
.0019
])
test_subset('outputs/lenet_error_valid.txt', [
.0283, .0209, .0170, .0151, .0139, .0129, .0121, .0118, .0112,
.0113
])
test_subset('outputs/lenet_error_test.txt', [
.0319, .0213, .0167, .0134, .0122, .0119, .0116, .0107, .0104,
.0105
])
shutil.rmtree('outputs/')
finally:
if 'lenet' in locals():
del lenet
if 'data' in locals():
del data
if 'y' in locals():
del y
if 'x' in locals():
del x
if 'loss' in locals():
del loss
if 'optimizer' in locals():
del optimizer
print "Getting data..."
# data = MNIST()
df = pd.read_csv("train.csv")
features = np.array(df.ix[:,1:])
targets = np.array(df["label"])
features_train, features_test, targets_train, targets_test = train_test_split(features,targets, test_size=0.4, random_state=4)
data = Dataset(features_train, train_targets=targets_train,
test_inputs=features_test, test_targets=targets_test)
print "Creating model..."
in_shape = (None, 28*28)
in_var = matrix('xs')
mlp = Prototype()
mlp.add(Dense(inputs=(in_shape, in_var), outputs=512, activation='relu'))
mlp.add(Noise, noise='dropout', noise_level=0.5)
mlp.add(Dense, outputs=512, activation='relu')
mlp.add(Noise, noise='dropout', noise_level=0.5)
mlp.add(Softmax, outputs=10, out_as_probs=False)
print "Training..."
target_var = lvector('ys')
loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=target_var, one_hot=False)
optimizer = AdaDelta(model=mlp, loss=loss, dataset=data, epochs=10)
optimizer.train()
print "Predicting..."
predictions = mlp.run(data.test_inputs)
if __name__ == '__main__':
# some debugging output to see what is going on under the hood
config_root_logger()
#########
# Model #
#########
# build a Prototype container to easily add layers and make a cohesive model!
mlp = Prototype()
# need to define a variable for the inputs to this model, as well as the shape
# we are doing minibatch training (where we don't know the minibatch size), and the image is a (784,) array.
x = T.matrix('xs')
x_shape = (None, 28*28)
# add our first dense (fully-connected) layer!
mlp.add(Dense(inputs=(x_shape, x), outputs=500, activation='tanh'))
# noise is used to regularize the layer from overfitting to data (helps generalization)
# when adding subsequent layers, we can simply provide the class type and any other kwargs
# (omitting the `inputs` kwarg) and it will route the previous layer's outputs as the current
# layer's inputs.
mlp.add(Noise, noise='dropout', noise_level=0.5)
# add our classification layer
mlp.add(Softmax, outputs=10, out_as_probs=False)
################
# Optimization #
################
# Now that our model is complete, let's define the loss function to optimize
# first need a target variable
labels = T.lvector('ys')
# negative log-likelihood for classification cost
def build_lenet():
# quick and dirty way to create a model from arbitrary layers
lenet = Prototype()
# our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
x = ((None, 1, 28, 28), tensor4('x'))
# our first convolutional layer
lenet.add(
Conv2D(inputs=x, n_filters=20, filter_size=(5, 5))
)
# our first pooling layer, automatically hooking inputs to the previous convolutional outputs
lenet.add(
Pool2D, size=(2, 2)
)
# our second convolutional layer
lenet.add(
Conv2D, n_filters=50, filter_size=(5, 5)
)
# our second pooling layer
lenet.add(
Pool2D, size=(2, 2)
)
# now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
lenet.add(
Flatten, ndim=2
)
# one dense hidden layer
lenet.add(
Dense, outputs=500, activation='tanh'
)
# hook a softmax classification layer, outputting the probabilities.
lenet.add(
Softmax, outputs=10, out_as_probs=True
)
return lenet
if __name__ == '__main__':
# some debugging output to see what is going on under the hood
config_root_logger()
#########
# Model #
#########
# build a Prototype container to easily add layers and make a cohesive model!
mlp = Prototype()
# need to define a variable for the inputs to this model, as well as the shape
# we are doing minibatch training (where we don't know the minibatch size), and the image is a (784,) array.
x = T.matrix('xs')
x_shape = (None, 28 * 28)
# add our first dense (fully-connected) layer!
mlp.add(Dense(inputs=(x_shape, x), outputs=500, activation='tanh'))
# noise is used to regularize the layer from overfitting to data (helps generalization)
# when adding subsequent layers, we can simply provide the class type and any other kwargs
# (omitting the `inputs` kwarg) and it will route the previous layer's outputs as the current
# layer's inputs.
mlp.add(Noise, noise='dropout', noise_level=0.5)
# add our classification layer
mlp.add(Softmax, outputs=10, out_as_probs=False)
################
# Optimization #
################
# Now that our model is complete, let's define the loss function to optimize
# first need a target variable
labels = T.lvector('ys')
# negative log-likelihood for classification cost
def build_lenet():
# quick and dirty way to create a model from arbitrary layers
lenet = Prototype()
# our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
x = ((None, 1, 28, 28), tensor4('x'))
# our first convolutional layer
lenet.add(Conv2D(inputs=x, n_filters=20, filter_size=(5, 5)))
# our first pooling layer, automatically hooking inputs to the previous convolutional outputs
lenet.add(Pool2D, size=(2, 2))
# our second convolutional layer
lenet.add(Conv2D, n_filters=50, filter_size=(5, 5))
# our second pooling layer
lenet.add(Pool2D, size=(2, 2))
# now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
lenet.add(Flatten, ndim=2)
# one dense hidden layer
lenet.add(Dense, outputs=500, activation='tanh')
# hook a softmax classification layer, outputting the probabilities.
lenet.add(Softmax, outputs=10, out_as_probs=True)
return lenet
def build_lenet():
# quick and dirty way to create a model from arbitrary layers
lenet = Prototype()
# our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
x = ((None, 1, 28, 28), tensor4('x'))
# our first convolutional layer
lenet.add(
Conv2D(inputs=x, n_filters=20, filter_size=(5, 5))
)
# our first pooling layer, automatically hooking inputs to the previous convolutional outputs
lenet.add(
Pool2D, size=(2, 2)
)
# our second convolutional layer
lenet.add(
Conv2D, n_filters=50, filter_size=(5, 5)
)
# our second pooling layer
lenet.add(
Pool2D, size=(2, 2)
)
# now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
dense_input = lenet.models[-1].get_outputs().flatten(2)
# redefine the size appropriately for flattening (since we are doing a Theano modification)
dense_input_shape = (None, np.prod(lenet.models[-1].output_size[1:]))
# pass this flattened matrix as the input to a Dense layer!
lenet.add(
Dense(
inputs=[(dense_input_shape, dense_input)],
outputs=500,
activation='tanh'
)
)
# automatically hook a softmax classification layer, outputting the probabilities.
lenet.add(
Softmax, outputs=10, out_as_probs=True
)
return lenet
from opendeep.models.utils import Noise, Pool2D
from opendeep.optimization.loss import Neg_LL
from opendeep.optimization import AdaDelta
from opendeep.monitor import Monitor
from opendeep.data.stream import ModifyStream
from opendeep.data import ImageDataset
import filter_test
#config_root_logger()
lenet = Prototype()
images = T.tensor4('xs')
images_shape = (None, 1, 480, 640)
lenet.add(Conv2D(inputs=(images_shape, images), n_filters=6, filter_size=(5, 5), border_mode="full", activation="relu"))
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise="dropout", noise_level=0.5)
lenet.add(Conv2D, n_filters=20, filter_size=(5, 5), border_mode="full", activation="relu")
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise="dropout", noise_level=0.5)
dense_input = lenet.models[-1].get_outputs().flatten(2)
dense_in_shape = lenet.models[-1].output_size[:1] + (np.prod(lenet.models[-1].output_size[1:]), )
lenet.add(Dense(inputs=(dense_in_shape, dense_input), outputs=500, activation="relu"))
lenet.add(Noise, noise="dropout", noise_level=0.5)
lenet.add(Dense, outputs=2)
def testLeNet(self):
try:
# quick and dirty way to create a model from arbitrary layers
lenet = Prototype(outdir=None)
# our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
x = ((None, 1, 28, 28), ftensor4('x'))
# our first convolutional layer
lenet.add(
Conv2D(inputs=x, n_filters=20, filter_size=(5, 5), outdir=None)
)
# our first pooling layer, automatically hooking inputs to the previous convolutional outputs
lenet.add(
Pool2D, size=(2, 2)
)
# our second convolutional layer
lenet.add(
Conv2D, n_filters=50, filter_size=(5, 5), outdir=None
)
# our second pooling layer
lenet.add(
Pool2D, size=(2, 2)
)
# now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
lenet.add(
Flatten, ndim=2
)
# one dense hidden layer
lenet.add(
Dense, outputs=500, activation='tanh', outdir=None
)
# hook a softmax classification layer, outputting the probabilities.
lenet.add(
Softmax, outputs=10, out_as_probs=True, outdir=None
)
# Grab the MNIST dataset
data = MNIST(path="../../../datasets/{!s}".format(mnist_name), concat_train_valid=False, flatten=False)
# define our loss to optimize for the model (and the target variable)
# targets from MNIST are int64 numbers 0-9
y = lvector('y')
loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
# monitor
error_monitor = Monitor(name='error', expression=mean(neq(lenet.models[-1].y_pred, y)), valid=True,
test=True, out_service=FileService('outputs/lenet_error.txt'))
# optimize our model to minimize loss given the dataset using SGD
optimizer = SGD(model=lenet,
dataset=data,
loss=loss,
epochs=10,
batch_size=128,
learning_rate=.1,
momentum=False)
print("Training LeNet...")
optimizer.train(monitor_channels=error_monitor)
def test_subset(filename, expected, conf=0.001):
with open(filename, 'r') as f:
errs = [float(err) for err in f]
for i, (err, exp) in enumerate(zip(errs, expected)):
if i == 0:
c = conf*10
else:
c = conf
self.assertTrue(exp-c < round(err, 4) < exp+c,
"Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}".format(
errs, expected, err, exp)
)
test_subset('outputs/lenet_error_train.txt',
[.0753,
.0239,
.0159,
.0113,
.0088,
.0064,
.0050,
.0037,
.0026,
.0019]
)
test_subset('outputs/lenet_error_valid.txt',
[.0283,
.0209,
.0170,
.0151,
.0139,
.0129,
.0121,
.0118,
.0112,
.0113]
)
test_subset('outputs/lenet_error_test.txt',
[.0319,
.0213,
.0167,
.0134,
.0122,
.0119,
.0116,
.0107,
.0104,
.0105]
)
shutil.rmtree('outputs/')
finally:
if 'lenet' in locals():
del lenet
if 'data' in locals():
del data
if 'y' in locals():
del y
if 'x' in locals():
del x
if 'loss' in locals():
del loss
if 'optimizer' in locals():
del optimizer
