Пример #1
0
def main(num_epochs=100):
    x = tensor.tensor4('features')
    y = tensor.lmatrix('targets')

    num_filters = 16
    layers = [ConvolutionalLayer(filter_size=(3, 3), num_filters=num_filters,
                                 num_channels=1, step=(1, 1),
                                 border_mode='valid'),
              MaxPooling(pooling_size=(2, 2)),
              ConvolutionalLayer(filter_size=(3, 3), num_filters=num_filters,
                                 num_channels=num_filters, step=(1, 1),
                                 border_mode='valid')]
    convnet = ConvolutionalNetwork(layers=layers)
    output_shape = convnet.get_output_shape((28, 28))
    convnet.set_input_shape((28, 28))
    fc_net = NeuralSoftmax(input_dim=numpy.prod(output_shape) * num_filters,
                           n_classes=10, n_hidden=[100])

    convnet_features = convnet.apply(x)
    probs = fc_net.get_probs(features=convnet_features.flatten(2))
    params = convnet.get_params() + fc_net.get_params()
    weights = convnet.get_weights()
    cost = fc_net.get_cost(probs=probs, targets=y).mean()
    cost.name = 'cost'
    misclassification = fc_net.get_misclassification(
        probs=probs, targets=y
    ).mean()
    misclassification.name = 'misclassification'

    train_dataset = MNIST('train', flatten=False)
    test_dataset = MNIST('test', flatten=False)

    algorithm = GradientDescent(
        cost=cost,
        params=params,
        step_rule=Momentum(learning_rate=.1,
                           momentum=0.1))

    train_data_stream = ForceFloatX(
        data_stream=Mapping(
            data_stream=DataStream(
                dataset=train_dataset,
                iteration_scheme=ShuffledScheme(
                    examples=range(50000),
                    batch_size=100,
                )
            ),
            mapping=reshape_features
        )
    )
    valid_data_stream = ForceFloatX(
        data_stream=Mapping(
            data_stream=DataStream(
                dataset=train_dataset,
                iteration_scheme=SequentialScheme(
                    examples=range(50000, 60000),
                    batch_size=1000,
                )
            ),
            mapping=reshape_features
        )
    )
    test_data_stream = ForceFloatX(
        data_stream=Mapping(
            data_stream=DataStream(
                dataset=test_dataset,
                iteration_scheme=SequentialScheme(
                    examples=10,
                    batch_size=1000,
                )
            ),
            mapping=reshape_features
        )
    )

    model = Model(cost)

    extensions = []
    extensions.append(Timing())
    extensions.append(FinishAfter(after_n_epochs=num_epochs))
    extensions.append(DataStreamMonitoring(
        [cost, misclassification],
        test_data_stream,
        prefix='test'))
    extensions.append(DataStreamMonitoring(
        [cost, misclassification],
        valid_data_stream,
        prefix='valid'))
    extensions.append(TrainingDataMonitoring(
        [cost, misclassification],
        prefix='train',
        after_epoch=True))

    plotters = []
    plotters.append(Plotter(
        channels=[['test_cost', 'test_misclassification',
                   'train_cost', 'train_misclassification']],
        titles=['Costs']))
    display_train = ImageDataStreamDisplay(
        data_stream=copy.deepcopy(train_data_stream),
        image_shape=(1, 28, 28),
        axes=('c', 0, 1),
        shift=-0.5,
        rescale=2.,
    )
    weight_display = WeightDisplay(
        weights=weights,
        transpose=(0, 1, 2, 3),
        image_shape=(1, 3, 3),
        axes=('c', 0, 1),
        shift=-0.5,
        rescale=2.,
    )

    # Feature maps
    one_example_train_data_stream = ForceFloatX(
        data_stream=Mapping(
            data_stream=DataStream(
                dataset=train_dataset,
                iteration_scheme=ShuffledScheme(
                    examples=train_dataset.num_examples,
                    batch_size=1,
                )
            ),
            mapping=reshape_features
        )
    )
    displayable_convnet_features = convnet_features.dimshuffle((1, 0, 2, 3))
    convnet_features_normalizer = abs(
        displayable_convnet_features
    ).max(axis=(1, 2, 3))
    displayable_convnet_features = displayable_convnet_features \
        / convnet_features_normalizer.dimshuffle((0, 'x', 'x', 'x'))
    get_displayable_convnet_features = theano.function(
        [x], displayable_convnet_features
    )
    display_feature_maps_data_stream = Mapping(
        data_stream=one_example_train_data_stream,
        mapping=TupleMapping(get_displayable_convnet_features,
                             same_len_out=True)
    )
    display_feature_maps = ImageDataStreamDisplay(
        data_stream=display_feature_maps_data_stream,
        image_shape=(1, ) + output_shape,
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )

    # Saliency map
    displayable_saliency_map = tensor.grad(cost, x)
    saliency_map_normalizer = abs(
        displayable_saliency_map
    ).max(axis=(1, 2, 3))
    displayable_saliency_map = displayable_saliency_map \
        / saliency_map_normalizer.dimshuffle((0, 'x', 'x', 'x'))
    get_displayable_saliency_map = theano.function(
        [x, y], displayable_saliency_map
    )

    display_saliency_map_data_stream = Mapping(
        data_stream=copy.deepcopy(train_data_stream),
        mapping=TupleMapping(get_displayable_saliency_map,
                             same_len_out=True,
                             same_len_in=True)
    )
    display_saliency_map = ImageDataStreamDisplay(
        data_stream=display_saliency_map_data_stream,
        image_shape=(1, 28, 28),
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )

    # Deconvolution
    x_repeated = x.repeat(num_filters, axis=0)
    convnet_features_repeated = convnet.apply(x_repeated)
    convnet_features_selected = convnet_features_repeated \
        * tensor.eye(num_filters).repeat(
            x.shape[0], axis=0
        ).dimshuffle((0, 1, 'x', 'x'))
    displayable_deconvolution = tensor.grad(misclassification, x_repeated,
                                            known_grads={
                                                convnet_features_selected:
                                                    convnet_features_selected
                                            })
    deconvolution_normalizer = abs(
        displayable_deconvolution
    ).max(axis=(1, 2, 3))
    displayable_deconvolution = displayable_deconvolution \
        / deconvolution_normalizer.dimshuffle((0, 'x', 'x', 'x'))
    get_displayable_deconvolution = theano.function(
        [x], displayable_deconvolution
    )
    display_deconvolution_data_stream = Mapping(
        data_stream=one_example_train_data_stream,
        mapping=TupleMapping(get_displayable_deconvolution,
                             same_len_out=True)
    )
    display_deconvolution = ImageDataStreamDisplay(
        data_stream=display_deconvolution_data_stream,
        image_shape=(1, 28, 28),
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )

    images_displayer = DisplayImage(
        image_getters=[display_train, weight_display, display_feature_maps,
                       display_saliency_map, display_deconvolution],
        titles=['Training examples', 'Convolutional weights', 'Feature maps',
                'Sensitivity', 'Deconvolution']
    )
    plotters.append(images_displayer)

    extensions.append(PlotManager('MNIST convnet examples', plotters=plotters,
                                  after_epoch=False,
                                  every_n_epochs=10,
                                  after_training=True))
    extensions.append(Printing())
    main_loop = MainLoop(model=model,
                         data_stream=train_data_stream,
                         algorithm=algorithm,
                         extensions=extensions)

    main_loop.run()
Пример #2
0
def main(num_epochs=10000):
    x = tensor.matrix('features')
    y = tensor.lmatrix('targets')

    neural_net = NeuralSoftmax(input_dim=784, n_classes=10, n_hidden=[100])
    probs = neural_net.get_probs(features=x)
    params = neural_net.get_params()
    weights = neural_net.get_weights()
    cost = neural_net.get_cost(probs=probs, targets=y).mean()
    cost.name = 'cost'
    misclassification = neural_net.get_misclassification(
        probs=probs, targets=y
    ).mean()
    misclassification.name = 'misclassification'

    train_dataset = MNIST('train')
    test_dataset = MNIST('test')

    algorithm = GradientDescent(
        cost=cost,
        params=params,
        step_rule=Momentum(learning_rate=0.1,
                           momentum=0.1))

    train_data_stream = ForceFloatX(
        data_stream=DataStream(
            dataset=train_dataset,
            iteration_scheme=ShuffledScheme(
                examples=range(50000),
                batch_size=100,
            )
        )
    )
    valid_data_stream = ForceFloatX(
        data_stream=DataStream(
            dataset=train_dataset,
            iteration_scheme=SequentialScheme(
                examples=range(50000, 60000),
                batch_size=1000,
            )
        )
    )
    test_data_stream = ForceFloatX(
        data_stream=DataStream(
            dataset=test_dataset,
            iteration_scheme=SequentialScheme(
                examples=test_dataset.num_examples,
                batch_size=1000,
            )
        )
    )

    model = Model(cost)

    extensions = []
    extensions.append(Timing())
    extensions.append(FinishAfter(after_n_epochs=num_epochs))
    extensions.append(DataStreamMonitoring(
        [cost, misclassification],
        test_data_stream,
        prefix='test'))
    extensions.append(DataStreamMonitoring(
        [cost, misclassification],
        valid_data_stream,
        prefix='valid'))
    extensions.append(TrainingDataMonitoring(
        [cost, misclassification],
        prefix='train',
        after_epoch=True))

    plotters = []
    plotters.append(Plotter(
        channels=[['test_cost', 'test_misclassification',
                   'train_cost', 'train_misclassification']],
        titles=['Costs']))
    display_train = ImageDataStreamDisplay(
        data_stream=copy.deepcopy(train_data_stream),
        image_shape=(28, 28, 1),
        axes=(0, 1, 'c'),
        shift=-0.5,
        rescale=2.,
    )
    weight_display = WeightDisplay(
        weights=weights,
        transpose=(1, 0),
        image_shape=(28, 28, 1),
        axes=(0, 1, 'c'),
        shift=-0.5,
        rescale=2.
    )
    images_displayer = DisplayImage(
        image_getters=[display_train, weight_display],
        titles=['Training examples', 'Softmax weights']
    )
    plotters.append(images_displayer)

    extensions.append(PlotManager('MNIST neural network examples',
                                  plotters=plotters,
                                  after_epoch=False,
                                  every_n_epochs=10,
                                  after_training=True))
    extensions.append(Printing())
    main_loop = MainLoop(model=model,
                         data_stream=train_data_stream,
                         algorithm=algorithm,
                         extensions=extensions)

    main_loop.run()
def main(num_epochs=100):
    x = tensor.tensor4('features')
    y = tensor.lmatrix('targets')

    num_filters = 64
    layers = [ConvolutionalLayer(filter_size=(8, 8), num_filters=num_filters,
                                 num_channels=3, step=(1, 1),
                                 border_mode='valid'),
              MaxPooling(pooling_size=(2, 2)),
              ConvolutionalLayer(filter_size=(5, 5), num_filters=num_filters,
                                 num_channels=num_filters, step=(1, 1),
                                 border_mode='valid')]
    convnet = ConvolutionalNetwork(layers=layers)
    output_shape = convnet.get_output_shape((32, 32))
    convnet.set_input_shape((32, 32))
    fc_net = NeuralSoftmax(input_dim=numpy.prod(output_shape) * num_filters,
                           n_classes=10, n_hidden=[500])

    convnet_features = convnet.apply(x)
    probs = fc_net.get_probs(features=convnet_features.flatten(2))
    params = convnet.get_params() + fc_net.get_params()
    weights = convnet.get_weights()
    cost = fc_net.get_cost(probs=probs, targets=y).mean()
    cost.name = 'cost'
    misclassification = fc_net.get_misclassification(
        probs=probs, targets=y
    ).mean()
    misclassification.name = 'misclassification'

    train_dataset = CIFAR10('train', flatten=False)
    test_dataset = CIFAR10('test', flatten=False)

    algorithm = GradientDescent(
        cost=cost,
        params=params,
        step_rule=Momentum(learning_rate=.01,
                           momentum=0.1))

    train_data_stream = Mapping(
        data_stream=ForceFloatX(
            data_stream=DataStream(
                dataset=train_dataset,
                iteration_scheme=ShuffledScheme(
                    examples=range(50000),
                    batch_size=100,
                )
            )
        ),
        mapping=Rescale(scale=1/127.5, shift=-127.5)
    )
    valid_data_stream = Mapping(
        data_stream=ForceFloatX(
            data_stream=DataStream(
                dataset=train_dataset,
                iteration_scheme=SequentialScheme(
                    examples=range(40000, 50000),
                    batch_size=1000,
                )
            )
        ),
        mapping=Rescale(scale=1/127.5, shift=-127.5)
    )
    test_data_stream = Mapping(
        data_stream=ForceFloatX(
            data_stream=DataStream(
                dataset=test_dataset,
                iteration_scheme=SequentialScheme(
                    examples=test_dataset.num_examples,
                    batch_size=100,
                )
            )
        ),
        mapping=Rescale(scale=1/127.5, shift=-127.5)
    )

    model = Model(cost)

    extensions = []
    extensions.append(Timing())
    extensions.append(FinishAfter(after_n_epochs=num_epochs))
    extensions.append(DataStreamMonitoring(
        [cost, misclassification],
        test_data_stream,
        prefix='test'))
    extensions.append(DataStreamMonitoring(
        [cost, misclassification],
        valid_data_stream,
        prefix='valid'))
    extensions.append(TrainingDataMonitoring(
        [cost, misclassification],
        prefix='train',
        after_epoch=True))

    plotters = []
    plotters.append(Plotter(
        channels=[['test_cost', 'test_misclassification',
                   'train_cost', 'train_misclassification']],
        titles=['Costs']))
    display_train = ImageDataStreamDisplay(
        data_stream=copy.deepcopy(train_data_stream),
        image_shape=(3, 32, 32),
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )
    weight_display = WeightDisplay(
        weights=weights,
        transpose=(0, 1, 2, 3),
        image_shape=(3, 8, 8),
        axes=('c', 0, 1),
        shift=-0.5,
        rescale=2.,
    )

    # Feature maps
    one_example_train_data_stream = Mapping(
        data_stream=ForceFloatX(
            data_stream=DataStream(
                dataset=train_dataset,
                iteration_scheme=ShuffledScheme(
                    examples=train_dataset.num_examples,
                    batch_size=1,
                )
            )
        ),
        mapping=Rescale(scale=1/127.5, shift=-127.5)
    )
    displayable_convnet_features = convnet_features.dimshuffle((1, 0, 2, 3))
    convnet_features_normalizer = abs(
        displayable_convnet_features
    ).max(axis=(1, 2, 3))
    displayable_convnet_features = displayable_convnet_features \
        / convnet_features_normalizer.dimshuffle((0, 'x', 'x', 'x'))
    get_displayable_convnet_features = theano.function(
        [x], displayable_convnet_features
    )
    display_feature_maps_data_stream = Mapping(
        data_stream=one_example_train_data_stream,
        mapping=TupleMapping(get_displayable_convnet_features,
                             same_len_out=True)
    )
    display_feature_maps = ImageDataStreamDisplay(
        data_stream=display_feature_maps_data_stream,
        image_shape=(1, ) + output_shape,
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )

    # Saliency map
    displayable_saliency_map = tensor.grad(cost, x)
    saliency_map_normalizer = abs(
        displayable_saliency_map
    ).max(axis=(1, 2, 3))
    displayable_saliency_map = displayable_saliency_map \
        / saliency_map_normalizer.dimshuffle((0, 'x', 'x', 'x'))
    get_displayable_saliency_map = theano.function(
        [x, y], displayable_saliency_map
    )

    display_saliency_map_data_stream = Mapping(
        data_stream=copy.deepcopy(train_data_stream),
        mapping=TupleMapping(get_displayable_saliency_map,
                             same_len_out=True,
                             same_len_in=True)
    )
    display_saliency_map = ImageDataStreamDisplay(
        data_stream=display_saliency_map_data_stream,
        image_shape=(3, 32, 32),
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )

    # Deconvolution
    x_repeated = x.repeat(num_filters, axis=0)
    convnet_features_repeated = convnet.apply(x_repeated)
    convnet_features_selected = convnet_features_repeated \
        * tensor.eye(num_filters).repeat(
            x.shape[0], axis=0
        ).dimshuffle((0, 1, 'x', 'x'))
    displayable_deconvolution = tensor.grad(misclassification, x_repeated,
                                            known_grads={
                                                convnet_features_selected:
                                                    convnet_features_selected
                                            })
    deconvolution_normalizer = abs(
        displayable_deconvolution
    ).max(axis=(1, 2, 3))
    displayable_deconvolution = displayable_deconvolution \
        / deconvolution_normalizer.dimshuffle((0, 'x', 'x', 'x'))
    get_displayable_deconvolution = theano.function(
        [x], displayable_deconvolution
    )
    display_deconvolution_data_stream = Mapping(
        data_stream=one_example_train_data_stream,
        mapping=TupleMapping(get_displayable_deconvolution,
                             same_len_out=True)
    )
    display_deconvolution = ImageDataStreamDisplay(
        data_stream=display_deconvolution_data_stream,
        image_shape=(3, 32, 32),
        axes=('c', 0, 1),
        shift=0,
        rescale=1.,
    )

    images_displayer = DisplayImage(
        image_getters=[display_train, weight_display, display_feature_maps,
                       display_saliency_map, display_deconvolution],
        titles=['Training examples', 'Convolutional weights', 'Feature maps',
                'Sensitivity', 'Deconvolution']
    )
    plotters.append(images_displayer)

    extensions.append(PlotManager('CIFAR-10 convnet examples',
                                  plotters=plotters,
                                  after_epoch=False,
                                  every_n_epochs=10,
                                  after_training=True))
    extensions.append(Printing())
    main_loop = MainLoop(model=model,
                         data_stream=train_data_stream,
                         algorithm=algorithm,
                         extensions=extensions)

    main_loop.run()
Пример #4
0
def main(num_epochs=10000):
    x = tensor.matrix('features')
    y = tensor.lmatrix('targets')

    neural_net = NeuralSoftmax(input_dim=784, n_classes=10, n_hidden=[100])
    probs = neural_net.get_probs(features=x)
    params = neural_net.get_params()
    weights = neural_net.get_weights()
    cost = neural_net.get_cost(probs=probs, targets=y).mean()
    cost.name = 'cost'
    misclassification = neural_net.get_misclassification(probs=probs,
                                                         targets=y).mean()
    misclassification.name = 'misclassification'

    train_dataset = MNIST('train')
    test_dataset = MNIST('test')

    algorithm = GradientDescent(cost=cost,
                                params=params,
                                step_rule=Momentum(learning_rate=0.1,
                                                   momentum=0.1))

    train_data_stream = ForceFloatX(
        data_stream=DataStream(dataset=train_dataset,
                               iteration_scheme=ShuffledScheme(
                                   examples=range(50000),
                                   batch_size=100,
                               )))
    valid_data_stream = ForceFloatX(
        data_stream=DataStream(dataset=train_dataset,
                               iteration_scheme=SequentialScheme(
                                   examples=range(50000, 60000),
                                   batch_size=1000,
                               )))
    test_data_stream = ForceFloatX(
        data_stream=DataStream(dataset=test_dataset,
                               iteration_scheme=SequentialScheme(
                                   examples=test_dataset.num_examples,
                                   batch_size=1000,
                               )))

    model = Model(cost)

    extensions = []
    extensions.append(Timing())
    extensions.append(FinishAfter(after_n_epochs=num_epochs))
    extensions.append(
        DataStreamMonitoring([cost, misclassification],
                             test_data_stream,
                             prefix='test'))
    extensions.append(
        DataStreamMonitoring([cost, misclassification],
                             valid_data_stream,
                             prefix='valid'))
    extensions.append(
        TrainingDataMonitoring([cost, misclassification],
                               prefix='train',
                               after_epoch=True))

    plotters = []
    plotters.append(
        Plotter(channels=[[
            'test_cost', 'test_misclassification', 'train_cost',
            'train_misclassification'
        ]],
                titles=['Costs']))
    display_train = ImageDataStreamDisplay(
        data_stream=copy.deepcopy(train_data_stream),
        image_shape=(28, 28, 1),
        axes=(0, 1, 'c'),
        shift=-0.5,
        rescale=2.,
    )
    weight_display = WeightDisplay(weights=weights,
                                   transpose=(1, 0),
                                   image_shape=(28, 28, 1),
                                   axes=(0, 1, 'c'),
                                   shift=-0.5,
                                   rescale=2.)
    images_displayer = DisplayImage(
        image_getters=[display_train, weight_display],
        titles=['Training examples', 'Softmax weights'])
    plotters.append(images_displayer)

    extensions.append(
        PlotManager('MNIST neural network examples',
                    plotters=plotters,
                    after_epoch=False,
                    every_n_epochs=10,
                    after_training=True))
    extensions.append(Printing())
    main_loop = MainLoop(model=model,
                         data_stream=train_data_stream,
                         algorithm=algorithm,
                         extensions=extensions)

    main_loop.run()