def main():
    # First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
    mlp = Prototype()
    mlp.add(
        Dense(input_size=28 * 28,
              output_size=1000,
              activation='rectifier',
              noise='dropout'))
    mlp.add(SoftmaxLayer(output_size=10))

    # Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
    # Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
    # dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
    # care about. By default, this method returns an empty dictionary - it was the model creator's job to
    # include potential monitor values.
    mlp_monitors = mlp.get_monitors()
    mlp_channel = MonitorsChannel(name="error")
    for name, expression in mlp_monitors.items():
        mlp_channel.add(
            Monitor(name=name,
                    expression=expression,
                    train=True,
                    valid=True,
                    test=True))

    # create some monitors for statistics about the hidden and output weights!
    # let's look at the mean, variance, and standard deviation of the weights matrices.
    weights_channel = MonitorsChannel(name="weights")
    hiddens_1 = mlp[0].get_params()[0]
    hiddens1_mean = T.mean(hiddens_1)
    weights_channel.add(
        Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))

    hiddens_2 = mlp[1].get_params()[0]
    hiddens2_mean = T.mean(hiddens_2)
    weights_channel.add(
        Monitor(name="out_mean", expression=hiddens2_mean, train=True))

    # create our plot object to do live plotting!
    plot = Plot(bokeh_doc_name="Monitor Tutorial",
                monitor_channels=[mlp_channel, weights_channel],
                open_browser=True)

    # use SGD optimizer
    optimizer = SGD(model=mlp,
                    dataset=MNIST(concat_train_valid=False),
                    epochs=500,
                    save_freq=100,
                    batch_size=600,
                    learning_rate=.01,
                    lr_decay=False,
                    momentum=.9,
                    nesterov_momentum=True)

    # train, with the plot!
    optimizer.train(plot=plot)
def main():
    # First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
    mlp = Prototype()
    mlp.add(Dense(input_size=28*28, output_size=1000, activation='rectifier', noise='dropout'))
    mlp.add(SoftmaxLayer(output_size=10))

    # Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
    # Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
    # dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
    # care about. By default, this method returns an empty dictionary - it was the model creator's job to
    # include potential monitor values.
    mlp_monitors = mlp.get_monitors()
    mlp_channel = MonitorsChannel(name="error")
    for name, expression in mlp_monitors.items():
        mlp_channel.add(Monitor(name=name, expression=expression, train=True, valid=True, test=True))

    # create some monitors for statistics about the hidden and output weights!
    # let's look at the mean, variance, and standard deviation of the weights matrices.
    weights_channel = MonitorsChannel(name="weights")
    hiddens_1 = mlp[0].get_params()[0]
    hiddens1_mean = T.mean(hiddens_1)
    weights_channel.add(Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))

    hiddens_2 = mlp[1].get_params()[0]
    hiddens2_mean = T.mean(hiddens_2)
    weights_channel.add(Monitor(name="out_mean", expression=hiddens2_mean, train=True))

    # create our plot object to do live plotting!
    plot = Plot(bokeh_doc_name="Monitor Tutorial", monitor_channels=[mlp_channel, weights_channel], open_browser=True)

    # use SGD optimizer
    optimizer = SGD(model=mlp,
                    dataset=MNIST(concat_train_valid=False),
                    epochs=500,
                    save_freq=100,
                    batch_size=600,
                    learning_rate=.01,
                    lr_decay=False,
                    momentum=.9,
                    nesterov_momentum=True)

    # train, with the plot!
    optimizer.train(plot=plot)
Example #3
0
    def testAutoEncoder(self):
        try:
            s = (None, 3)
            x = matrix('xs')
            e = Dense(inputs=(s, x), outputs=int(s[1]*2), activation='sigmoid')
            W = e.get_param("W")
            d = Dense(inputs=e, outputs=s[1], params={'W': W.T}, activation='sigmoid')
            ae = Prototype([e, d])

            x2 = matrix('xs1')
            W2 = d.get_param("W")
            e2 = Dense(inputs=(s, x2), outputs=int(s[1]*2), params={"W": W2.T, "b": e.get_param('b')}, activation='sigmoid')
            W3 = e2.get_param("W")
            d2 = Dense(inputs=e2, outputs=s[1], params={"W": W3.T, 'b': d.get_param('b')}, activation='sigmoid')
            ae2 = Prototype([e2, d2])

            aerun1 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
            ae2run1 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
            self.assertTrue(np.array_equal(aerun1, ae2run1))

            data = np.ones((10,3), dtype='float32')*.1
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.2])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.3])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.4])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.5])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.6])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.7])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.8])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*.9])
            data = np.vstack([data, np.ones((10,3), dtype='float32')*0])
            dataset = NumpyDataset(data)
            sgd = SGD(dataset=dataset, model=ae, loss=BinaryCrossentropy(inputs=ae.get_outputs(), targets=x), epochs=5)
            sgd.train()

            aerun2 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
            ae2run2 = ae2.run(np.array([[.1,.5,.9]], dtype='float32'))

            self.assertFalse(np.array_equal(aerun2, aerun1))
            self.assertFalse(np.array_equal(ae2run2, ae2run1))
            self.assertTrue(np.array_equal(aerun2, ae2run2))

            sgd2 = SGD(dataset=dataset, model=ae2, loss=BinaryCrossentropy(inputs=ae2.get_outputs(), targets=x2), epochs=5)
            sgd2.train()

            aerun3 = ae.run(np.array([[.1,.5,.9]], dtype='float32'))
            ae2run3 = ae2.run(np.array([[.1,.5,.9]], dtype='float32'))

            self.assertFalse(np.array_equal(aerun3, aerun2))
            self.assertFalse(np.array_equal(ae2run3, ae2run2))
            self.assertTrue(np.array_equal(aerun3, ae2run3))


        finally:
            del x, e, d, ae, x2, e2, d2, ae2
Example #4
0
                         output_size=512,
                         activation='rectifier',
                         noise='dropout')

    hidden2 = Dense(inputs_hook=(512, hidden1.get_outputs()),
                         output_size=512,
                         activation='rectifier',
                         noise='dropout')

    class_layer = SoftmaxLayer(inputs_hook=(512, hidden2.get_outputs()),
                               output_size=10)

    mlp = Prototype([hidden1, hidden2, class_layer])
    return mlp


if __name__ == '__main__':
    mlp = sequential_add_layers()
    data = MNIST(concat_train_valid=True)
    print data.train_inputs.shape
    print data.valid_inputs.shape
    print data.test_inputs.shape
    optimizer = SGD(model=mlp,
                    dataset=data,
                    epochs=500,
                    batch_size=600,
                    learning_rate=.01,
                    momentum=.9,
                    nesterov_momentum=True)
    optimizer.train()
    def testLeNet(self):
        try:
            # quick and dirty way to create a model from arbitrary layers
            lenet = Prototype(outdir=None)
            # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
            x = ((None, 1, 28, 28), ftensor4('x'))
            # our first convolutional layer
            lenet.add(
                Conv2D(inputs=x, n_filters=20, filter_size=(5, 5),
                       outdir=None))
            # our first pooling layer, automatically hooking inputs to the previous convolutional outputs
            lenet.add(Pool2D, size=(2, 2))
            # our second convolutional layer
            lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), outdir=None)
            # our second pooling layer
            lenet.add(Pool2D, size=(2, 2))
            # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
            lenet.add(Flatten, ndim=2)
            # one dense hidden layer
            lenet.add(Dense, outputs=500, activation='tanh', outdir=None)
            # hook a softmax classification layer, outputting the probabilities.
            lenet.add(Softmax, outputs=10, out_as_probs=True, outdir=None)

            # Grab the MNIST dataset
            data = MNIST(path="../../../datasets/{!s}".format(mnist_name),
                         concat_train_valid=False,
                         flatten=False)
            # define our loss to optimize for the model (and the target variable)
            # targets from MNIST are int64 numbers 0-9
            y = lvector('y')
            loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
            # monitor
            error_monitor = Monitor(
                name='error',
                expression=mean(neq(lenet.models[-1].y_pred, y)),
                valid=True,
                test=True,
                out_service=FileService('outputs/lenet_error.txt'))
            # optimize our model to minimize loss given the dataset using SGD
            optimizer = SGD(model=lenet,
                            dataset=data,
                            loss=loss,
                            epochs=10,
                            batch_size=128,
                            learning_rate=.1,
                            momentum=False)
            print("Training LeNet...")
            optimizer.train(monitor_channels=error_monitor)

            def test_subset(filename, expected, conf=0.001):
                with open(filename, 'r') as f:
                    errs = [float(err) for err in f]
                for i, (err, exp) in enumerate(zip(errs, expected)):
                    if i == 0:
                        c = conf * 10
                    else:
                        c = conf
                    self.assertTrue(
                        exp - c < round(err, 4) < exp + c,
                        "Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}"
                        .format(errs, expected, err, exp))

            test_subset('outputs/lenet_error_train.txt', [
                .0753, .0239, .0159, .0113, .0088, .0064, .0050, .0037, .0026,
                .0019
            ])
            test_subset('outputs/lenet_error_valid.txt', [
                .0283, .0209, .0170, .0151, .0139, .0129, .0121, .0118, .0112,
                .0113
            ])
            test_subset('outputs/lenet_error_test.txt', [
                .0319, .0213, .0167, .0134, .0122, .0119, .0116, .0107, .0104,
                .0105
            ])
            shutil.rmtree('outputs/')

        finally:
            if 'lenet' in locals():
                del lenet
            if 'data' in locals():
                del data
            if 'y' in locals():
                del y
            if 'x' in locals():
                del x
            if 'loss' in locals():
                del loss
            if 'optimizer' in locals():
                del optimizer
Example #6
0
    return lenet


if __name__ == '__main__':
    # Grab the MNIST dataset
    data = MNIST(concat_train_valid=False)

    # we need to convert the (784,) flat example from MNIST to (1, 28, 28) for a 2D greyscale image
    process_mnist = lambda img: np.reshape(img, (1, 28, 28))

    # we can do this by using ModifyStreams over the inputs!
    data.train_inputs = ModifyStream(data.train_inputs, process_mnist)
    data.valid_inputs = ModifyStream(data.valid_inputs, process_mnist)
    data.test_inputs = ModifyStream(data.test_inputs, process_mnist)

    # now build the actual model
    lenet = build_lenet()
    # define our loss to optimize for the model (and the target variable)
    # targets from MNIST are int64 numbers 0-9
    y = lvector('y')
    loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
    error_monitor = Monitor(name='error', expression=mean(neq(lenet.models[-1].y_pred, y)), valid=True, test=True)
    # optimize our model to minimize loss given the dataset using SGD
    optimizer = SGD(model=lenet,
                    dataset=data,
                    loss=loss,
                    epochs=200,
                    batch_size=500,
                    learning_rate=.1,
                    momentum=False)
    optimizer.train(monitor_channels=error_monitor)
Example #7
0
    # Grab the MNIST dataset
    data = MNIST(concat_train_valid=False)

    # we need to convert the (784,) flat example from MNIST to (1, 28, 28) for a 2D greyscale image
    process_mnist = lambda img: np.reshape(img, (1, 28, 28))

    # we can do this by using ModifyStreams over the inputs!
    data.train_inputs = ModifyStream(data.train_inputs, process_mnist)
    data.valid_inputs = ModifyStream(data.valid_inputs, process_mnist)
    data.test_inputs = ModifyStream(data.test_inputs, process_mnist)

    # now build the actual model
    lenet = build_lenet()
    # define our loss to optimize for the model (and the target variable)
    # targets from MNIST are int64 numbers 0-9
    y = lvector('y')
    loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
    error_monitor = Monitor(name='error',
                            expression=mean(neq(lenet.models[-1].y_pred, y)),
                            valid=True,
                            test=True,
                            out_service=FileService('outputs/lenet_error.txt'))
    # optimize our model to minimize loss given the dataset using SGD
    optimizer = SGD(model=lenet,
                    dataset=data,
                    loss=loss,
                    epochs=200,
                    batch_size=128,
                    learning_rate=.1,
                    momentum=False)
    optimizer.train(monitor_channels=error_monitor)
Example #8
0
    def testAutoEncoder(self):
        try:
            s = (None, 3)
            x = matrix('xs')
            e = Dense(inputs=(s, x),
                      outputs=int(s[1] * 2),
                      activation='sigmoid')
            W = e.get_param("W")
            d = Dense(inputs=e,
                      outputs=s[1],
                      params={'W': W.T},
                      activation='sigmoid')
            ae = Prototype([e, d])

            x2 = matrix('xs1')
            W2 = d.get_param("W")
            e2 = Dense(inputs=(s, x2),
                       outputs=int(s[1] * 2),
                       params={
                           "W": W2.T,
                           "b": e.get_param('b')
                       },
                       activation='sigmoid')
            W3 = e2.get_param("W")
            d2 = Dense(inputs=e2,
                       outputs=s[1],
                       params={
                           "W": W3.T,
                           'b': d.get_param('b')
                       },
                       activation='sigmoid')
            ae2 = Prototype([e2, d2])

            aerun1 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
            ae2run1 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
            self.assertTrue(np.array_equal(aerun1, ae2run1))

            data = np.ones((10, 3), dtype='float32') * .1
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .2])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .3])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .4])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .5])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .6])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .7])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .8])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * .9])
            data = np.vstack([data, np.ones((10, 3), dtype='float32') * 0])
            dataset = NumpyDataset(data)
            sgd = SGD(dataset=dataset,
                      model=ae,
                      loss=BinaryCrossentropy(inputs=ae.get_outputs(),
                                              targets=x),
                      epochs=5)
            sgd.train()

            aerun2 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
            ae2run2 = ae2.run(np.array([[.1, .5, .9]], dtype='float32'))

            self.assertFalse(np.array_equal(aerun2, aerun1))
            self.assertFalse(np.array_equal(ae2run2, ae2run1))
            self.assertTrue(np.array_equal(aerun2, ae2run2))

            sgd2 = SGD(dataset=dataset,
                       model=ae2,
                       loss=BinaryCrossentropy(inputs=ae2.get_outputs(),
                                               targets=x2),
                       epochs=5)
            sgd2.train()

            aerun3 = ae.run(np.array([[.1, .5, .9]], dtype='float32'))
            ae2run3 = ae2.run(np.array([[.1, .5, .9]], dtype='float32'))

            self.assertFalse(np.array_equal(aerun3, aerun2))
            self.assertFalse(np.array_equal(ae2run3, ae2run2))
            self.assertTrue(np.array_equal(aerun3, ae2run3))

        finally:
            del x, e, d, ae, x2, e2, d2, ae2
Example #9
0
    def testLeNet(self):
        try:
            # quick and dirty way to create a model from arbitrary layers
            lenet = Prototype(outdir=None)
            # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
            x = ((None, 1, 28, 28), ftensor4('x'))
            # our first convolutional layer
            lenet.add(
                Conv2D(inputs=x, n_filters=20, filter_size=(5, 5), outdir=None)
            )
            # our first pooling layer, automatically hooking inputs to the previous convolutional outputs
            lenet.add(
                Pool2D, size=(2, 2)
            )
            # our second convolutional layer
            lenet.add(
                Conv2D, n_filters=50, filter_size=(5, 5), outdir=None
            )
            # our second pooling layer
            lenet.add(
                Pool2D, size=(2, 2)
            )
            # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
            lenet.add(
                Flatten, ndim=2
            )
            # one dense hidden layer
            lenet.add(
                Dense, outputs=500, activation='tanh', outdir=None
            )
            # hook a softmax classification layer, outputting the probabilities.
            lenet.add(
                Softmax, outputs=10, out_as_probs=True, outdir=None
            )

            # Grab the MNIST dataset
            data = MNIST(path="../../../datasets/{!s}".format(mnist_name), concat_train_valid=False, flatten=False)
            # define our loss to optimize for the model (and the target variable)
            # targets from MNIST are int64 numbers 0-9
            y = lvector('y')
            loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
            # monitor
            error_monitor = Monitor(name='error', expression=mean(neq(lenet.models[-1].y_pred, y)), valid=True,
                                    test=True, out_service=FileService('outputs/lenet_error.txt'))
            # optimize our model to minimize loss given the dataset using SGD
            optimizer = SGD(model=lenet,
                            dataset=data,
                            loss=loss,
                            epochs=10,
                            batch_size=128,
                            learning_rate=.1,
                            momentum=False)
            print("Training LeNet...")
            optimizer.train(monitor_channels=error_monitor)

            def test_subset(filename, expected, conf=0.001):
                with open(filename, 'r') as f:
                    errs = [float(err) for err in f]
                for i, (err, exp) in enumerate(zip(errs, expected)):
                    if i == 0:
                        c = conf*10
                    else:
                        c = conf
                    self.assertTrue(exp-c < round(err, 4) < exp+c,
                                    "Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}".format(
                                        errs, expected, err, exp)
                                    )

            test_subset('outputs/lenet_error_train.txt',
                        [.0753,
                         .0239,
                         .0159,
                         .0113,
                         .0088,
                         .0064,
                         .0050,
                         .0037,
                         .0026,
                         .0019]
                        )
            test_subset('outputs/lenet_error_valid.txt',
                        [.0283,
                         .0209,
                         .0170,
                         .0151,
                         .0139,
                         .0129,
                         .0121,
                         .0118,
                         .0112,
                         .0113]
                        )
            test_subset('outputs/lenet_error_test.txt',
                        [.0319,
                         .0213,
                         .0167,
                         .0134,
                         .0122,
                         .0119,
                         .0116,
                         .0107,
                         .0104,
                         .0105]
                        )
            shutil.rmtree('outputs/')

        finally:
            if 'lenet' in locals():
                del lenet
            if 'data' in locals():
                del data
            if 'y' in locals():
                del y
            if 'x' in locals():
                del x
            if 'loss' in locals():
                del loss
            if 'optimizer' in locals():
                del optimizer