Example #1
0
def run_mlp():
    # test the new way to automatically fill in inputs for models
    mlp = Prototype()
    x = ((None, 784), matrix("x"))
    mlp.add(Dense(inputs=x, outputs=1000, activation='rectifier'))
    mlp.add(Dense, outputs=1500, activation='tanh')
    mlp.add(Softmax, outputs=10, out_as_probs=False)

    # define our loss to optimize for the model (and the target variable)
    # targets from MNIST are int64 numbers 0-9
    y = lvector('y')
    loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x, targets=y, one_hot=False)

    mnist = MNIST()

    optimizer = AdaDelta(model=mlp, loss=loss, dataset=mnist, epochs=10)
    optimizer.train()

    test_data, test_labels = mnist.test_inputs, mnist.test_targets
    test_data = test_data[:25]
    test_labels = test_labels[:25]
    # use the run function!
    yhat = mlp.run(test_data)
    print('-------')
    print('Prediction: %s' % str(yhat))
    print('Actual:     %s' % str(test_labels.astype('int32')))
def create_mlp():
    # define the model layers
    relu_layer1 = Dense(input_size=784,
                        output_size=1000,
                        activation='rectifier')
    relu_layer2 = Dense(inputs_hook=(1000, relu_layer1.get_outputs()),
                        output_size=1000,
                        activation='rectifier')
    class_layer3 = SoftmaxLayer(inputs_hook=(1000, relu_layer2.get_outputs()),
                                output_size=10,
                                out_as_probs=False)
    # add the layers as a Prototype
    mlp = Prototype(layers=[relu_layer1, relu_layer2, class_layer3])

    mnist = MNIST()

    optimizer = AdaDelta(model=mlp, dataset=mnist, epochs=20)
    optimizer.train()

    test_data, test_labels = mnist.test_inputs[:25], mnist.test_targets[:25]

    # use the run function!
    preds = mlp.run(test_data)
    log.info('-------')
    log.info("predicted: %s", str(preds))
    log.info("actual:    %s", str(test_labels.astype('int32')))
def main():
    # First, let's create a simple feedforward MLP with one hidden layer as a Prototype.
    mlp = Prototype()
    mlp.add(
        Dense(input_size=28 * 28,
              output_size=1000,
              activation='rectifier',
              noise='dropout'))
    mlp.add(SoftmaxLayer(output_size=10))

    # Now, we get to choose what values we want to monitor, and what datasets we would like to monitor on!
    # Each Model (in our case, the Prototype), has a get_monitors method that will return a useful
    # dictionary of {string_name: monitor_theano_expression} for various computations of the model we might
    # care about. By default, this method returns an empty dictionary - it was the model creator's job to
    # include potential monitor values.
    mlp_monitors = mlp.get_monitors()
    mlp_channel = MonitorsChannel(name="error")
    for name, expression in mlp_monitors.items():
        mlp_channel.add(
            Monitor(name=name,
                    expression=expression,
                    train=True,
                    valid=True,
                    test=True))

    # create some monitors for statistics about the hidden and output weights!
    # let's look at the mean, variance, and standard deviation of the weights matrices.
    weights_channel = MonitorsChannel(name="weights")
    hiddens_1 = mlp[0].get_params()[0]
    hiddens1_mean = T.mean(hiddens_1)
    weights_channel.add(
        Monitor(name="hiddens_mean", expression=hiddens1_mean, train=True))

    hiddens_2 = mlp[1].get_params()[0]
    hiddens2_mean = T.mean(hiddens_2)
    weights_channel.add(
        Monitor(name="out_mean", expression=hiddens2_mean, train=True))

    # create our plot object to do live plotting!
    plot = Plot(bokeh_doc_name="Monitor Tutorial",
                monitor_channels=[mlp_channel, weights_channel],
                open_browser=True)

    # use SGD optimizer
    optimizer = SGD(model=mlp,
                    dataset=MNIST(concat_train_valid=False),
                    epochs=500,
                    save_freq=100,
                    batch_size=600,
                    learning_rate=.01,
                    lr_decay=False,
                    momentum=.9,
                    nesterov_momentum=True)

    # train, with the plot!
    optimizer.train(plot=plot)
Example #4
0
def get_sequenced_mnist(sequence_number=0, seq_3d=False, seq_length=30, rng=None, flatten=True):
    """
        sequence_number : int, optional
            The sequence method to use if we want to put the input images into a specific order. 0 defaults to random.
        seq_3d : bool, optional
            When sequencing, whether the output should be
            3D tensors (batches, subsequences, data) or 2D (sequence, data).
        seq_length: int, optional
            The length of subsequences to split the data.
        rng : random, optional
            The random number generator to use when sequencing.
    """
    mnist = MNIST(flatten=flatten)

    # sequence the dataset
    if sequence_number is not None:
        _sequence(mnist, sequence_number=sequence_number, rng=rng)

    # optionally make 3D instead of 2D
    if seq_3d:
        print("Making 3D....")
        # chop up into sequences of length seq_length
        # first make sure to chop off the remainder of the data so seq_length can divide evenly.
        if mnist.train_inputs.shape[0] % seq_length != 0:
            length, dim = mnist.train_inputs.shape
            if mnist.train_targets.ndim == 1:
                ydim = 1
            else:
                ydim = mnist.train_targets.shape[-1]
            mnist.train_inputs = mnist.train_inputs[:seq_length * int(math.floor(length / seq_length))]
            mnist.train_targets = mnist.train_targets[:seq_length * int(math.floor(length / seq_length))]
            # now create the 3D tensor of sequences - they will be (num_sequences, sequence_size, 784)
            mnist.train_inputs = numpy.reshape(mnist.train_inputs, (length / seq_length, seq_length, dim))
            mnist.train_targets = numpy.reshape(mnist.train_targets, (length / seq_length, seq_length, ydim))

        if mnist.valid_inputs.shape[0] % seq_length != 0:
            length, dim = mnist.valid_inputs.shape
            if mnist.valid_targets.ndim == 1:
                ydim = 1
            else:
                ydim = mnist.valid_targets.shape[-1]
            mnist.valid_inputs = mnist.valid_inputs[:seq_length * int(math.floor(length / seq_length))]
            mnist.valid_targets = mnist.valid_targets[:seq_length * int(math.floor(length / seq_length))]
            # now create the 3D tensor of sequences - they will be (num_sequences, sequence_size, 784)
            mnist.valid_inputs = numpy.reshape(mnist.valid_inputs, (length / seq_length, seq_length, dim))
            mnist.valid_targets = numpy.reshape(mnist.valid_targets, (length / seq_length, seq_length, ydim))

        if mnist.test_inputs.shape[0] % seq_length != 0:
            length, dim = mnist.test_inputs.shape
            if mnist.test_targets.ndim == 1:
                ydim = 1
            else:
                ydim = mnist.test_targets.shape[-1]
            mnist.test_inputs = mnist.test_inputs[:seq_length * int(math.floor(length / seq_length))]
            mnist.test_targets = mnist.test_targets[:seq_length * int(math.floor(length / seq_length))]
            # now create the 3D tensor of sequences - they will be (num_sequences, sequence_size, 784)
            mnist.test_inputs = numpy.reshape(mnist.test_inputs, (length / seq_length, seq_length, dim))
            mnist.test_targets = numpy.reshape(mnist.test_targets, (length / seq_length, seq_length, ydim))

    print('Train shape is: {!s}, {!s}'.format(mnist.train_inputs.shape, mnist.train_targets.shape))
    print('Valid shape is: {!s}, {!s}'.format(mnist.valid_inputs.shape, mnist.valid_targets.shape))
    print('Test shape is: {!s}, {!s}'.format(mnist.test_inputs.shape, mnist.test_targets.shape))
    return mnist
Example #5
0
    # first need a target variable
    labels = T.lvector('ys')
    # negative log-likelihood for classification cost
    loss = Neg_LL(inputs=lenet.models[-1].p_y_given_x,
                  targets=labels,
                  one_hot=False)
    # make a monitor to view average accuracy per batch
    accuracy = Monitor(name='Accuracy',
                       expression=1 -
                       (T.mean(T.neq(lenet.models[-1].y_pred, labels))),
                       valid=True,
                       test=True)

    # Now grab our MNIST dataset. The version given here has each image as a single 784-dimensional vector.
    # because convolutions work over 2d, let's reshape our data into the (28,28) images they originally were
    # (only one channel because they are black/white images not rgb)
    mnist = MNIST()
    process_image = lambda img: np.reshape(img, (1, 28, 28))
    mnist.train_inputs = ModifyStream(mnist.train_inputs, process_image)
    mnist.valid_inputs = ModifyStream(mnist.valid_inputs, process_image)
    mnist.test_inputs = ModifyStream(mnist.test_inputs, process_image)

    # finally define our optimizer and train the model!
    optimizer = AdaDelta(model=lenet,
                         dataset=mnist,
                         loss=loss,
                         epochs=10,
                         batch_size=64)
    # train!
    optimizer.train(monitor_channels=accuracy)
Example #6
0
                         output_size=512,
                         activation='rectifier',
                         noise='dropout')

    hidden2 = Dense(inputs_hook=(512, hidden1.get_outputs()),
                         output_size=512,
                         activation='rectifier',
                         noise='dropout')

    class_layer = SoftmaxLayer(inputs_hook=(512, hidden2.get_outputs()),
                               output_size=10)

    mlp = Prototype([hidden1, hidden2, class_layer])
    return mlp


if __name__ == '__main__':
    mlp = sequential_add_layers()
    data = MNIST(concat_train_valid=True)
    print data.train_inputs.shape
    print data.valid_inputs.shape
    print data.test_inputs.shape
    optimizer = SGD(model=mlp,
                    dataset=data,
                    epochs=500,
                    batch_size=600,
                    learning_rate=.01,
                    momentum=.9,
                    nesterov_momentum=True)
    optimizer.train()
Example #7
0
    # Optimization #
    ################
    # Now that our model is complete, let's define the loss function to optimize
    # first need a target variable
    labels = T.lvector('ys')
    # negative log-likelihood for classification cost
    loss = Neg_LL(inputs=lenet.models[-1].p_y_given_x, targets=labels, one_hot=False)
    # make a monitor to view average accuracy per batch
    accuracy = Monitor(name='Accuracy',
                       expression=1-(T.mean(T.neq(lenet.models[-1].y_pred, labels))),
                       valid=True, test=True)

    # Now grab our MNIST dataset. The version given here has each image as a single 784-dimensional vector.
    # because convolutions work over 2d, let's reshape our data into the (28,28) images they originally were
    # (only one channel because they are black/white images not rgb)
    mnist = MNIST()
    process_image = lambda img: np.reshape(img, (1, 28, 28))
    mnist.train_inputs = ModifyStream(mnist.train_inputs, process_image)
    mnist.valid_inputs = ModifyStream(mnist.valid_inputs, process_image)
    mnist.test_inputs = ModifyStream(mnist.test_inputs, process_image)

    # finally define our optimizer and train the model!
    optimizer = AdaDelta(
        model=lenet,
        dataset=mnist,
        loss=loss,
        epochs=10,
        batch_size=64
    )
    # train!
    optimizer.train(monitor_channels=accuracy)
    def testLeNet(self):
        try:
            # quick and dirty way to create a model from arbitrary layers
            lenet = Prototype(outdir=None)
            # our input is going to be 4D tensor of images with shape (batch_size, 1, 28, 28)
            x = ((None, 1, 28, 28), ftensor4('x'))
            # our first convolutional layer
            lenet.add(
                Conv2D(inputs=x, n_filters=20, filter_size=(5, 5),
                       outdir=None))
            # our first pooling layer, automatically hooking inputs to the previous convolutional outputs
            lenet.add(Pool2D, size=(2, 2))
            # our second convolutional layer
            lenet.add(Conv2D, n_filters=50, filter_size=(5, 5), outdir=None)
            # our second pooling layer
            lenet.add(Pool2D, size=(2, 2))
            # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
            lenet.add(Flatten, ndim=2)
            # one dense hidden layer
            lenet.add(Dense, outputs=500, activation='tanh', outdir=None)
            # hook a softmax classification layer, outputting the probabilities.
            lenet.add(Softmax, outputs=10, out_as_probs=True, outdir=None)

            # Grab the MNIST dataset
            data = MNIST(path="../../../datasets/{!s}".format(mnist_name),
                         concat_train_valid=False,
                         flatten=False)
            # define our loss to optimize for the model (and the target variable)
            # targets from MNIST are int64 numbers 0-9
            y = lvector('y')
            loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
            # monitor
            error_monitor = Monitor(
                name='error',
                expression=mean(neq(lenet.models[-1].y_pred, y)),
                valid=True,
                test=True,
                out_service=FileService('outputs/lenet_error.txt'))
            # optimize our model to minimize loss given the dataset using SGD
            optimizer = SGD(model=lenet,
                            dataset=data,
                            loss=loss,
                            epochs=10,
                            batch_size=128,
                            learning_rate=.1,
                            momentum=False)
            print("Training LeNet...")
            optimizer.train(monitor_channels=error_monitor)

            def test_subset(filename, expected, conf=0.001):
                with open(filename, 'r') as f:
                    errs = [float(err) for err in f]
                for i, (err, exp) in enumerate(zip(errs, expected)):
                    if i == 0:
                        c = conf * 10
                    else:
                        c = conf
                    self.assertTrue(
                        exp - c < round(err, 4) < exp + c,
                        "Errors: {!s} and Expected: {!s} -- Error at {!s} and {!s}"
                        .format(errs, expected, err, exp))

            test_subset('outputs/lenet_error_train.txt', [
                .0753, .0239, .0159, .0113, .0088, .0064, .0050, .0037, .0026,
                .0019
            ])
            test_subset('outputs/lenet_error_valid.txt', [
                .0283, .0209, .0170, .0151, .0139, .0129, .0121, .0118, .0112,
                .0113
            ])
            test_subset('outputs/lenet_error_test.txt', [
                .0319, .0213, .0167, .0134, .0122, .0119, .0116, .0107, .0104,
                .0105
            ])
            shutil.rmtree('outputs/')

        finally:
            if 'lenet' in locals():
                del lenet
            if 'data' in locals():
                del data
            if 'y' in locals():
                del y
            if 'x' in locals():
                del x
            if 'loss' in locals():
                del loss
            if 'optimizer' in locals():
                del optimizer
Example #9
0
    )
    # one dense hidden layer
    lenet.add(
        Dense, outputs=500, activation='tanh'
    )
    # hook a softmax classification layer, outputting the probabilities.
    lenet.add(
        Softmax, outputs=10, out_as_probs=True
    )

    return lenet


if __name__ == '__main__':
    # Grab the MNIST dataset
    data = MNIST(concat_train_valid=False)

    # we need to convert the (784,) flat example from MNIST to (1, 28, 28) for a 2D greyscale image
    process_mnist = lambda img: np.reshape(img, (1, 28, 28))

    # we can do this by using ModifyStreams over the inputs!
    data.train_inputs = ModifyStream(data.train_inputs, process_mnist)
    data.valid_inputs = ModifyStream(data.valid_inputs, process_mnist)
    data.test_inputs = ModifyStream(data.test_inputs, process_mnist)

    # now build the actual model
    lenet = build_lenet()
    # define our loss to optimize for the model (and the target variable)
    # targets from MNIST are int64 numbers 0-9
    y = lvector('y')
    loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
Example #10
0
    mlp.add(Softmax, outputs=10, out_as_probs=False)

    ################
    # Optimization #
    ################
    # Now that our model is complete, let's define the loss function to optimize
    # first need a target variable
    labels = T.lvector('ys')
    # negative log-likelihood for classification cost
    loss = Neg_LL(inputs=mlp.models[-1].p_y_given_x,
                  targets=labels,
                  one_hot=False)
    # make a monitor to view average accuracy per batch
    accuracy = Monitor(name='Accuracy',
                       expression=1 -
                       (T.mean(T.neq(mlp.models[-1].y_pred, labels))),
                       valid=True,
                       test=True)

    # Now grab our MNIST dataset.
    mnist = MNIST()

    # finally define our optimizer and train the model!
    optimizer = AdaDelta(model=mlp,
                         dataset=mnist,
                         loss=loss,
                         epochs=10,
                         batch_size=64)
    # train!
    optimizer.train(monitor_channels=accuracy)
Example #11
0
    lenet.add(Conv2D, n_filters=50, filter_size=(5, 5))
    # our second pooling layer
    lenet.add(Pool2D, size=(2, 2))
    # now we need to flatten the 4D convolution outputs into 2D matrix (just flatten the trailing dimensions)
    lenet.add(Flatten, ndim=2)
    # one dense hidden layer
    lenet.add(Dense, outputs=500, activation='tanh')
    # hook a softmax classification layer, outputting the probabilities.
    lenet.add(Softmax, outputs=10, out_as_probs=True)

    return lenet


if __name__ == '__main__':
    # Grab the MNIST dataset
    data = MNIST(concat_train_valid=False)

    # we need to convert the (784,) flat example from MNIST to (1, 28, 28) for a 2D greyscale image
    process_mnist = lambda img: np.reshape(img, (1, 28, 28))

    # we can do this by using ModifyStreams over the inputs!
    data.train_inputs = ModifyStream(data.train_inputs, process_mnist)
    data.valid_inputs = ModifyStream(data.valid_inputs, process_mnist)
    data.test_inputs = ModifyStream(data.test_inputs, process_mnist)

    # now build the actual model
    lenet = build_lenet()
    # define our loss to optimize for the model (and the target variable)
    # targets from MNIST are int64 numbers 0-9
    y = lvector('y')
    loss = Neg_LL(inputs=lenet.get_outputs(), targets=y, one_hot=False)
Example #12
0
try:
    import PIL.Image as Image
except ImportError:
    import Image

if __name__ == '__main__':
    # set up the logging environment to display outputs (optional)
    # although this is recommended over print statements everywhere
    import logging
    from opendeep import config_root_logger
    config_root_logger()
    log = logging.getLogger(__name__)
    log.info("Creating RBM!")

    # grab the MNIST dataset
    mnist = MNIST(concat_train_valid=False)
    # create the RBM
    rng = numpy.random.RandomState(1234)
    mrg = theano.tensor.shared_randomstreams.RandomStreams(rng.randint(2**30))
    config_args = {
        'input_size': 28 * 28,
        'hidden_size': 500,
        'k': 15,
        'weights_init': 'uniform',
        'weights_interval': 4 * numpy.sqrt(6. / 28 * 28 + 500),
        'mrg': mrg
    }
    rbm = RBM(**config_args)
    # rbm.load_params('rbm_trained.pkl')

    optimizer = Optimizer(learning_rate=0.1,