def sequential_add_layers():
# This method is to demonstrate adding layers one-by-one to a Prototype container.
# As you can see, inputs_hook are created automatically by Prototype so we don't need to specify!
mlp = Prototype()
mlp.add(Dense(input_size=28*28, output_size=1000, activation='rectifier', noise='dropout', noise_level=0.5))
mlp.add(Dense(output_size=512, activation='rectifier', noise='dropout', noise_level=0.5))
mlp.add(SoftmaxLayer(output_size=10))
return mlp
def run_mlp():
# test the new way to automatically fill in inputs for models
mlp = Prototype()
x = ((None, 784), matrix("x"))
mlp.add(Dense(inputs=x, outputs=1000, activation='rectifier'))
mlp.add(Dense, outputs=1500, activation='tanh')
mlp.add(Softmax, outputs=10, out_as_probs=False)
# define our loss to optimize for the model (and the target variable)
# targets from MNIST are int64 numbers 0-9
y = lvector('y')
loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=y, one_hot=False)
mnist = MNIST()
optimizer = AdaDelta(model=mlp, loss=loss, dataset=mnist, epochs=10)
optimizer.train()
test_data, test_labels = mnist.test_inputs, mnist.test_targets
test_data = test_data[:25]
test_labels = test_labels[:25]
# use the run function!
yhat = mlp.run(test_data)
print('-------')
print('Prediction: %s' % str(yhat))
print('Actual: %s' % str(test_labels.astype('int32')))
def testAutoEncoder(self):
try:
s = (None, 3)
x = matrix('xs')
e = Dense(inputs=(s, x), outputs=int(s[1]*2), activation='sigmoid')
W = e.get_param("W")
d = Dense(inputs=e, outputs=s[1], params={'W': W.T}, activation='sigmoid')
ae = Prototype([e, d])
x2 = matrix('xs1')
W2 = d.get_param("W")
e2 = Dense(inputs=(s, x2), outputs=int(s[1]*2), params={"W": W2.T, "b": e.get_param('b')}, activation='sigmoid')
W3 = e2.get_param("W")
d2 = Dense(inputs=e2, outputs=s[1], params={"W": W3.T, 'b': d.get_param('b')}, activation='sigmoid')
ae2 = Prototype([e2, d2])
aerun1 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
ae2run1 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
self.assertTrue(np.array_equal(aerun1, ae2run1))
data = np.ones((10,3), dtype='float32')*.1
data = np.vstack([data, np.ones((10,3), dtype='float32')*.2])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.3])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.4])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.5])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.6])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.7])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.8])
data = np.vstack([data, np.ones((10,3), dtype='float32')*.9])
data = np.vstack([data, np.ones((10,3), dtype='float32')*0])
dataset = NumpyDataset(data)
sgd = SGD(dataset=dataset, model=ae, loss=BinaryCrossentropy(inputs=ae.get_outputs(), targets=x), epochs=5)
sgd.train()
aerun2 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
ae2run2 = ae2.run(np.array([[.1,.5,.9]], dtype='float32'))
self.assertFalse(np.array_equal(aerun2, aerun1))
self.assertFalse(np.array_equal(ae2run2, ae2run1))
self.assertTrue(np.array_equal(aerun2, ae2run2))
sgd2 = SGD(dataset=dataset, model=ae2, loss=BinaryCrossentropy(inputs=ae2.get_outputs(), targets=x2), epochs=5)
sgd2.train()
aerun3 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
ae2run3 = ae2.run(np.array([[.1,.5,.9]], dtype='float32'))
self.assertFalse(np.array_equal(aerun3, aerun2))
self.assertFalse(np.array_equal(ae2run3, ae2run2))
self.assertTrue(np.array_equal(aerun3, ae2run3))
finally:
del x, e, d, ae, x2, e2, d2, ae2
def main():
# First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
mlp = Prototype()
mlp.add(
Dense(input_size=28 * 28,
output_size=1000,
activation='rectifier',
noise='dropout'))
mlp.add(SoftmaxLayer(output_size=10))
# Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
# Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
# dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
# care about. By default, this method returns an empty dictionary - it was the model creator's job to
# include potential monitor values.
mlp_monitors = mlp.get_monitors()
mlp_channel = MonitorsChannel(name="error")
for name, expression in mlp_monitors.items():
mlp_channel.add(
Monitor(name=name,
expression=expression,
train=True,
valid=True,
test=True))
# create some monitors for statistics about the hidden and output weights!
# let's look at the mean, variance, and standard deviation of the weights matrices.
weights_channel = MonitorsChannel(name="weights")
hiddens_1 = mlp[0].get_params()[0]
hiddens1_mean = T.mean(hiddens_1)
weights_channel.add(
Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))
hiddens_2 = mlp[1].get_params()[0]
hiddens2_mean = T.mean(hiddens_2)
weights_channel.add(
Monitor(name="out_mean", expression=hiddens2_mean, train=True))
# create our plot object to do live plotting!
plot = Plot(bokeh_doc_name="Monitor Tutorial",
monitor_channels=[mlp_channel, weights_channel],
open_browser=True)
# use SGD optimizer
optimizer = SGD(model=mlp,
dataset=MNIST(concat_train_valid=False),
epochs=500,
save_freq=100,
batch_size=600,
learning_rate=.01,
lr_decay=False,
momentum=.9,
nesterov_momentum=True)
# train, with the plot!
optimizer.train(plot=plot)
def create_mlp():
# define the model layers
relu_layer1 = Dense(input_size=784, output_size=1000, activation='rectifier')
relu_layer2 = Dense(inputs_hook=(1000, relu_layer1.get_outputs()), output_size=1000, activation='rectifier')
class_layer3 = SoftmaxLayer(inputs_hook=(1000, relu_layer2.get_outputs()), output_size=10, out_as_probs=False)
# add the layers as a Prototype
mlp = Prototype(layers=[relu_layer1, relu_layer2, class_layer3])
mnist = MNIST()
optimizer = AdaDelta(model=mlp, dataset=mnist, epochs=20)
optimizer.train()
test_data, test_labels = mnist.test_inputs[:25], mnist.test_targets[:25]
# use the run function!
preds = mlp.run(test_data)
log.info('-------')
log.info("predicted: %s",str(preds))
log.info("actual: %s",str(test_labels.astype('int32')))
def create_mlp():
# define the model layers
relu_layer1 = Dense(input_size=784,
output_size=1000,
activation='rectifier')
relu_layer2 = Dense(inputs_hook=(1000, relu_layer1.get_outputs()),
output_size=1000,
activation='rectifier')
class_layer3 = SoftmaxLayer(inputs_hook=(1000, relu_layer2.get_outputs()),
output_size=10,
out_as_probs=False)
# add the layers as a Prototype
mlp = Prototype(layers=[relu_layer1, relu_layer2, class_layer3])
mnist = MNIST()
optimizer = AdaDelta(model=mlp, dataset=mnist, epochs=20)
optimizer.train()
test_data, test_labels = mnist.test_inputs[:25], mnist.test_targets[:25]
# use the run function!
preds = mlp.run(test_data)
log.info('-------')
log.info("predicted: %s", str(preds))
log.info("actual: %s", str(test_labels.astype('int32')))
def add_list_layers():
# You can also add lists of layers at a time (or as initialization) to a Prototype! This lets you specify
# more complex interactions between layers!
hidden1 = Dense(input_size=28*28,
output_size=512,
activation='rectifier',
noise='dropout')
hidden2 = Dense(inputs_hook=(512, hidden1.get_outputs()),
output_size=512,
activation='rectifier',
noise='dropout')
class_layer = SoftmaxLayer(inputs_hook=(512, hidden2.get_outputs()),
output_size=10)
mlp = Prototype([hidden1, hidden2, class_layer])
return mlp
lenet.add(Conv2D,
n_filters=50,
filter_size=(5, 5),
border_mode='full',
activation='relu')
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# reshape convolution output to be 2D to feed into fully-connected layers
# Prototype container keeps its layers in the `models` attribute list: grab the latest model output
dense_input = lenet.models[-1].get_outputs().flatten(2)
dense_in_shape = lenet.models[-1].output_size[:1] + (np.prod(
lenet.models[-1].output_size[1:]), )
# now make the dense (fully-connected) layers!
lenet.add(
Dense(inputs=(dense_in_shape, dense_input),
outputs=500,
activation='tanh'))
lenet.add(Noise, noise='dropout', noise_level=0.5)
# softmax classification layer!
lenet.add(Softmax, outputs=10, out_as_probs=False)
################
# Optimization #
################
# Now that our model is complete, let's define the loss function to optimize
# first need a target variable
labels = T.lvector('ys')
# negative log-likelihood for classification cost
loss = Neg_LL(inputs=lenet.models[-1].p_y_given_x,
targets=labels,
one_hot=False)
if __name__ == '__main__':
# some debugging output to see what is going on under the hood
config_root_logger()
#########
# Model #
#########
# build a Prototype container to easily add layers and make a cohesive model!
mlp = Prototype()
# need to define a variable for the inputs to this model, as well as the shape
# we are doing minibatch training (where we don't know the minibatch size), and the image is a (784,) array.
x = T.matrix('xs')
x_shape = (None, 28 * 28)
# add our first dense (fully-connected) layer!
mlp.add(Dense(inputs=(x_shape, x), outputs=500, activation='tanh'))
# noise is used to regularize the layer from overfitting to data (helps generalization)
# when adding subsequent layers, we can simply provide the class type and any other kwargs
# (omitting the `inputs` kwarg) and it will route the previous layer's outputs as the current
# layer's inputs.
mlp.add(Noise, noise='dropout', noise_level=0.5)
# add our classification layer
mlp.add(Softmax, outputs=10, out_as_probs=False)
################
# Optimization #
################
# Now that our model is complete, let's define the loss function to optimize
# first need a target variable
labels = T.lvector('ys')
# negative log-likelihood for classification cost
def testAutoEncoder(self):
try:
s = (None, 3)
x = matrix('xs')
e = Dense(inputs=(s, x),
outputs=int(s[1] * 2),
activation='sigmoid')
W = e.get_param("W")
d = Dense(inputs=e,
outputs=s[1],
params={'W': W.T},
activation='sigmoid')
ae = Prototype([e, d])
x2 = matrix('xs1')
W2 = d.get_param("W")
e2 = Dense(inputs=(s, x2),
outputs=int(s[1] * 2),
params={
"W": W2.T,
"b": e.get_param('b')
},
activation='sigmoid')
W3 = e2.get_param("W")
d2 = Dense(inputs=e2,
outputs=s[1],
params={
"W": W3.T,
'b': d.get_param('b')
},
activation='sigmoid')
ae2 = Prototype([e2, d2])
aerun1 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
ae2run1 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
self.assertTrue(np.array_equal(aerun1, ae2run1))
data = np.ones((10, 3), dtype='float32') * .1
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .2])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .3])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .4])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .5])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .6])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .7])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .8])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * .9])
data = np.vstack([data, np.ones((10, 3), dtype='float32') * 0])
dataset = NumpyDataset(data)
sgd = SGD(dataset=dataset,
model=ae,
loss=BinaryCrossentropy(inputs=ae.get_outputs(),
targets=x),
epochs=5)
sgd.train()
aerun2 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
ae2run2 = ae2.run(np.array([[.1, .5, .9]], dtype='float32'))
self.assertFalse(np.array_equal(aerun2, aerun1))
self.assertFalse(np.array_equal(ae2run2, ae2run1))
self.assertTrue(np.array_equal(aerun2, ae2run2))
sgd2 = SGD(dataset=dataset,
model=ae2,
loss=BinaryCrossentropy(inputs=ae2.get_outputs(),
targets=x2),
epochs=5)
sgd2.train()
aerun3 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
ae2run3 = ae2.run(np.array([[.1, .5, .9]], dtype='float32'))
self.assertFalse(np.array_equal(aerun3, aerun2))
self.assertFalse(np.array_equal(ae2run3, ae2run2))
self.assertTrue(np.array_equal(aerun3, ae2run3))
finally:
del x, e, d, ae, x2, e2, d2, ae2
images = T.tensor4('xs')
images_shape = (None, 1, 480, 640)
lenet.add(Conv2D(inputs=(images_shape, images), n_filters=6, filter_size=(5, 5), border_mode="full", activation="relu"))
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise="dropout", noise_level=0.5)
lenet.add(Conv2D, n_filters=20, filter_size=(5, 5), border_mode="full", activation="relu")
lenet.add(Pool2D, size=(2, 2))
lenet.add(Noise, noise="dropout", noise_level=0.5)
dense_input = lenet.models[-1].get_outputs().flatten(2)
dense_in_shape = lenet.models[-1].output_size[:1] + (np.prod(lenet.models[-1].output_size[1:]), )
lenet.add(Dense(inputs=(dense_in_shape, dense_input), outputs=500, activation="relu"))
lenet.add(Noise, noise="dropout", noise_level=0.5)
lenet.add(Dense, outputs=2)
labels = T.lvector('ys')
loss = Neg_LL(inputs=lenet.models[-1].get_outputs(), targets=labels, one_hot=False)
#accuracy = Monitor(name="Accuracy", expression=1-(T.mean(T.neq(lenet.models[-1].y_pred, labels))),
# valid=True, test=True)
def greyscale_image(img):
img = img.transpose(2, 1, 0)
arr = np.average(img, 0).astype(int)
def __init__(self, inputs):
super(VGG, self).__init__(inputs)
# vgg conv layers
self.conv1_1 = Conv2D(inputs=inputs,
n_filters=64,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv1_2 = Conv2D(inputs=self.conv1_1,
n_filters=64,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.pool1 = Pool2D(inputs=self.conv1_2, size=(2, 2), stride=(2, 2))
self.conv2_1 = Conv2D(inputs=self.pool1,
n_filters=128,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv2_2 = Conv2D(inputs=self.conv2_1,
n_filters=128,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.pool2 = Pool2D(inputs=self.conv2_2, size=(2, 2), stride=(2, 2))
self.conv3_1 = Conv2D(inputs=self.pool2,
n_filters=256,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv3_2 = Conv2D(inputs=self.conv3_1,
n_filters=256,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv3_3 = Conv2D(inputs=self.conv3_2,
n_filters=256,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.pool3 = Pool2D(inputs=self.conv3_3, size=(2, 2), stride=(2, 2))
self.conv4_1 = Conv2D(inputs=self.pool3,
n_filters=512,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv4_2 = Conv2D(inputs=self.conv4_1,
n_filters=512,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv4_3 = Conv2D(inputs=self.conv4_2,
n_filters=512,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.pool4 = Pool2D(inputs=self.conv4_3, size=(2, 2), stride=(2, 2))
self.conv5_1 = Conv2D(inputs=self.pool4,
n_filters=512,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv5_2 = Conv2D(inputs=self.conv5_1,
n_filters=512,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.conv5_3 = Conv2D(inputs=self.conv5_2,
n_filters=512,
filter_size=(3, 3),
activation='relu',
border_mode='full')
self.pool5 = Pool2D(inputs=self.conv4_3, size=(2, 2), stride=(2, 2))
# Disclaimer: Full VGG-19 with FC layers doesn't fit in GTX780's 3GB memory.
fc6_in = self.pool5.get_outputs().flatten(2)
dims_prod = None if any([
size is None for size in self.pool5.output_size[1:]
]) else np.prod(self.pool5.output_size[1:])
fc6_in_shape = (self.pool5.output_size[0], dims_prod)
self.fc6 = Dense(inputs=(fc6_in_shape, fc6_in),
outputs=4096,
activation='relu')
fc6_drop = Noise(inputs=self.fc6, noise='dropout', noise_level=0.5)
self.fc7 = Dense(inputs=fc6_drop, outputs=406, activation='relu')
fc7_drop = Noise(inputs=self.fc7, noise='dropout', noise_level=0.5)
self.fc8 = Softmax(inputs=fc7_drop, outputs=1000)
# uncomment if we are only doing convolutional output
# self.outputs = self.conv5_3.get_outputs()
# self.output_size = self.conv5_3.output_size
self.output_size = self.fc8.output_size
self.outputs = self.fc8.get_outputs()
self.switches = []
self.switches = fc6_drop.get_switches() + fc7_drop.get_switches()
self.params = {}
self.params.update(p_dict("conv1_1_", self.conv1_1))
self.params.update(p_dict("conv1_2_", self.conv1_2))
self.params.update(p_dict("conv2_1_", self.conv2_1))
self.params.update(p_dict("conv2_2_", self.conv2_2))
self.params.update(p_dict("conv3_1_", self.conv3_1))
self.params.update(p_dict("conv3_2_", self.conv3_2))
self.params.update(p_dict("conv3_3_", self.conv3_3))
self.params.update(p_dict("conv4_1_", self.conv4_1))
self.params.update(p_dict("conv4_2_", self.conv4_2))
self.params.update(p_dict("conv4_3_", self.conv4_3))
self.params.update(p_dict("conv5_1_", self.conv5_1))
self.params.update(p_dict("conv5_2_", self.conv5_2))
self.params.update(p_dict("conv5_3_", self.conv5_3))
self.params.update(p_dict("fc6_", self.fc6))
self.params.update(p_dict("fc7_", self.fc7))
self.params.update(p_dict("fc8_", self.fc8))