Exemplo n.º 1
0
    def residual_block(l, increase_dim=False, first=False, filters=16):
        if increase_dim:
            first_stride = (2, 2)
        else:
            first_stride = (1, 1)

        if first:
            # hacky solution to keep layers correct
            bn_pre_relu = l
        else:
            # contains the BN -> ReLU portion, steps 1 to 2
            bn_pre_conv = BatchNormLayer(l)
            bn_pre_relu = NonlinearityLayer(bn_pre_conv, rectify)

        # contains the weight -> BN -> ReLU portion, steps 3 to 5
        conv_1 = batch_norm(
            ConvLayer(bn_pre_relu,
                      num_filters=filters,
                      filter_size=(3, 3),
                      stride=first_stride,
                      nonlinearity=rectify,
                      pad='same',
                      W=HeNormal(gain='relu')))

        dropout = DropoutLayer(conv_1, p=0.3)

        # contains the last weight portion, step 6
        conv_2 = ConvLayer(dropout,
                           num_filters=filters,
                           filter_size=(3, 3),
                           stride=(1, 1),
                           nonlinearity=None,
                           pad='same',
                           W=HeNormal(gain='relu'))

        # add shortcut connections
        if increase_dim:
            # projection shortcut, as option B in paper
            projection = ConvLayer(l,
                                   num_filters=filters,
                                   filter_size=(1, 1),
                                   stride=(2, 2),
                                   nonlinearity=None,
                                   pad='same',
                                   b=None)
            block = ElemwiseSumLayer([conv_2, projection])
        elif first:
            # projection shortcut, as option B in paper
            projection = ConvLayer(l,
                                   num_filters=filters,
                                   filter_size=(1, 1),
                                   stride=(1, 1),
                                   nonlinearity=None,
                                   pad='same',
                                   b=None)
            block = ElemwiseSumLayer([conv_2, projection])
        else:
            block = ElemwiseSumLayer([conv_2, l])

        return block
Exemplo n.º 2
0
    def contraction(depth, deepest):
        n_filters = filter_for_depth(depth)
        incoming = net['input'] if depth == 0 else net['pool{}'.format(depth -
                                                                       1)]

        net['conv{}_1'.format(depth)] = Conv2DLayer(incoming,
                                                    num_filters=n_filters,
                                                    filter_size=3,
                                                    pad='valid',
                                                    W=HeNormal(gain='relu'),
                                                    nonlinearity=nonlinearity)
        net['conv{}_2'.format(depth)] = Conv2DLayer(
            net['conv{}_1'.format(depth)],
            num_filters=n_filters,
            filter_size=3,
            pad='valid',
            W=HeNormal(gain='relu'),
            nonlinearity=nonlinearity)

        if P.BATCH_NORMALIZATION:
            net['conv{}_2'.format(depth)] = batch_norm(
                net['conv{}_2'.format(depth)],
                alpha=P.BATCH_NORMALIZATION_ALPHA)

        if not deepest:
            net['pool{}'.format(depth)] = MaxPool2DLayer(
                net['conv{}_2'.format(depth)], pool_size=2, stride=2)
Exemplo n.º 3
0
def add_layer(incoming, num_channels, dropout):
    layer = ScaleLayer(incoming)
    layer = BiasLayer(layer)

    # Bottleneck layer to reduce number of input channels to 4 times the number of output channels
    layer = NonlinearityLayer(layer, nonlinearity=rectify)
    layer = Conv2DLayer(layer,
                        num_filters=4 * num_channels,
                        filter_size=(1, 1),
                        stride=(1, 1),
                        W=HeNormal(gain='relu'), b=None,
                        flip_filters=False,
                        nonlinearity=None)
    layer = BatchNormLayer(layer, beta=None, gamma=None)
    if dropout > 0:
        layer = DropoutLayer(layer, p=dropout)

    # Convolutional layer (using padding to keep same dimensions)
    layer = NonlinearityLayer(layer, nonlinearity=rectify)
    layer = Conv2DLayer(layer,
                        num_filters=num_channels,
                        filter_size=(3, 3),
                        stride=(1, 1),
                        W=HeNormal(gain='relu'), b=None,
                        pad='same',
                        flip_filters=False,
                        nonlinearity=None)
    layer = BatchNormLayer(layer, beta=None, gamma=None)
    if dropout > 0:
        layer = DropoutLayer(layer, p=dropout)

    # Concatenate the input filters with the new filters
    layer = ConcatLayer([incoming, layer], axis=1)

    return layer
def test_he_normal_c01b_4d_only():
    from lasagne.init import HeNormal

    with pytest.raises(RuntimeError):
        HeNormal(c01b=True).sample((100, ))

    with pytest.raises(RuntimeError):
        HeNormal(c01b=True).sample((100, 100))

    with pytest.raises(RuntimeError):
        HeNormal(c01b=True).sample((100, 100, 100))
Exemplo n.º 5
0
def residual_block(l,
                   increase_dim=False,
                   projection=True,
                   first=False,
                   filters=16):
    if increase_dim:
        first_stride = (2, 2)
    else:
        first_stride = (1, 1)
    if first:
        bn_pre_relu = l
    else:
        bn_pre_conv = BatchNormLayer(l)
        bn_pre_relu = NonlinearityLayer(bn_pre_conv, rectify)
    conv_1 = batch_norm(
        ConvLayer(bn_pre_relu,
                  num_filters=filters,
                  filter_size=(3, 3),
                  stride=first_stride,
                  nonlinearity=rectify,
                  pad='same',
                  W=HeNormal(gain='relu')))
    dropout = DropoutLayer(conv_1, p=0.3)
    conv_2 = ConvLayer(dropout,
                       num_filters=filters,
                       filter_size=(3, 3),
                       stride=(1, 1),
                       nonlinearity=None,
                       pad='same',
                       W=HeNormal(gain='relu'))
    if increase_dim:
        projection = ConvLayer(l,
                               num_filters=filters,
                               filter_size=(1, 1),
                               stride=(2, 2),
                               nonlinearity=None,
                               pad='same',
                               b=None)
        block = ElemwiseSumLayer([conv_2, projection])
    elif first:
        projection = ConvLayer(l,
                               num_filters=filters,
                               filter_size=(1, 1),
                               stride=(1, 1),
                               nonlinearity=None,
                               pad='same',
                               b=None)
        block = ElemwiseSumLayer([conv_2, projection])
    else:
        block = ElemwiseSumLayer([conv_2, l])
    return block
Exemplo n.º 6
0
    def cifar_model(cls, n=9, incoming=None, classes=10, **kwargs):
        model = incoming or InputLayer(shape=(None, 3, 32, 32))
        builder = cls(model, **kwargs)

        # first layer, output is 16 x 32 x 32
        builder.model = builder.convolution(model, 16, init_gain=1.0)

        # first stack of residual blocks, output is 16 x 32 x 32
        for _ in range(n):
            builder.add_residual_block(16)

        # second stack of residual blocks, output is 32 x 16 x 16
        builder.add_residual_block(32, dim_inc=True)
        for _ in range(1, n):
            builder.add_residual_block(32)

        # third stack of residual blocks, output is 64 x 8 x 8
        builder.add_residual_block(64, dim_inc=True)
        for _ in range(1, n):
            builder.add_residual_block(64)

        model = builder.nonlinearity(BatchNormLayer(builder.model))

        # average pooling
        model = GlobalPoolLayer(model)

        # fully connected layer
        model = DenseLayer(model,
                           num_units=classes,
                           W=HeNormal(gain='relu'),
                           nonlinearity=softmax)
        return model
Exemplo n.º 7
0
    def expansion(depth, deepest):
        n_filters = filter_for_depth(depth)

        incoming = net['conv{}_2'.format(depth + 1)] if deepest else net[
            '_conv{}_2'.format(depth + 1)]

        upscaling = Upscale2DLayer(incoming, 4)
        net['upconv{}'.format(depth)] = Conv2DLayer(upscaling,
                                                    num_filters=n_filters,
                                                    filter_size=2,
                                                    stride=2,
                                                    W=HeNormal(gain='relu'),
                                                    nonlinearity=nonlinearity)

        if P.SPATIAL_DROPOUT > 0:
            bridge_from = DropoutLayer(net['conv{}_2'.format(depth)],
                                       P.SPATIAL_DROPOUT)
        else:
            bridge_from = net['conv{}_2'.format(depth)]

        net['bridge{}'.format(depth)] = ConcatLayer(
            [net['upconv{}'.format(depth)], bridge_from],
            axis=1,
            cropping=[None, None, 'center', 'center'])

        net['_conv{}_1'.format(depth)] = Conv2DLayer(
            net['bridge{}'.format(depth)],
            num_filters=n_filters,
            filter_size=3,
            pad='valid',
            W=HeNormal(gain='relu'),
            nonlinearity=nonlinearity)

        #if P.BATCH_NORMALIZATION:
        #    net['_conv{}_1'.format(depth)] = batch_norm(net['_conv{}_1'.format(depth)])

        if P.DROPOUT > 0:
            net['_conv{}_1'.format(depth)] = DropoutLayer(
                net['_conv{}_1'.format(depth)], P.DROPOUT)

        net['_conv{}_2'.format(depth)] = Conv2DLayer(
            net['_conv{}_1'.format(depth)],
            num_filters=n_filters,
            filter_size=3,
            pad='valid',
            W=HeNormal(gain='relu'),
            nonlinearity=nonlinearity)
def test_he_normal_gain():
    from lasagne.init import HeNormal

    sample = HeNormal(gain=10.0).sample((100, 100))
    assert -0.1 < sample.mean() < 0.1
    assert 0.9 < sample.std() < 1.1

    sample = HeNormal(gain='relu').sample((200, 50))
    assert -0.1 < sample.mean() < 0.1
    assert 0.07 < sample.std() < 0.12
Exemplo n.º 9
0
def ResNet_FullPre_Wide(input_shape=(None, 3, PIXELS, PIXELS),
                        input_var=None,
                        n_classes=10,
                        n=6,
                        k=4):
    """
    Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.

    Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)

    And 'Wide Residual Networks', Sergey Zagoruyko, Nikos Komodakis 2016 (http://arxiv.org/pdf/1605.07146v1.pdf)

    Depth = 6n + 2
    """
    n_filters = {0: 16, 1: 16 * k, 2: 32 * k, 3: 64 * k}

    # Building the network
    l_in = InputLayer(shape=input_shape, input_var=input_var)

    # first layer, output is 16 x 64 x 64
    l = batch_norm(
        ConvLayer(l_in,
                  num_filters=n_filters[0],
                  filter_size=(3, 3),
                  stride=(1, 1),
                  nonlinearity=rectify,
                  pad='same',
                  W=he_norm))

    # first stack of residual blocks, output is 32 x 64 x 64
    l = residual_wide_block(l, first=True, filters=n_filters[1])
    for _ in range(1, n):
        l = residual_wide_block(l, filters=n_filters[1])

    # second stack of residual blocks, output is 64 x 32 x 32
    l = residual_wide_block(l, increase_dim=True, filters=n_filters[2])
    for _ in range(1, (n + 2)):
        l = residual_wide_block(l, filters=n_filters[2])

    # third stack of residual blocks, output is 128 x 16 x 16
    l = residual_wide_block(l, increase_dim=True, filters=n_filters[3])
    for _ in range(1, (n + 2)):
        l = residual_wide_block(l, filters=n_filters[3])

    bn_post_conv = BatchNormLayer(l)
    bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)

    # average pooling
    avg_pool = GlobalPoolLayer(bn_post_relu)

    # fully connected layer
    network = DenseLayer(avg_pool,
                         num_units=n_classes,
                         W=HeNormal(),
                         nonlinearity=softmax)

    return network
 def lrelu_conv(l_in, feat_out, stride=1, filter_size=3):
     l = NonlinearityLayer(l_in, nonlin)
     return Conv3DLayer(l,
                        feat_out,
                        filter_size,
                        stride,
                        'same',
                        nonlinearity=linear,
                        W=HeNormal(gain='relu'))
Exemplo n.º 11
0
 def build_model(self, incoming):
     net, params = super(ResNet_BottleNeck_FullPreActivation_Monoclass,
                         self).build_model(incoming)
     net['probsout'] = lasagne.layers.DenseLayer(
         net['featsout'],
         W=HeNormal(),
         num_units=self.out_size,
         nonlinearity=lasagne.nonlinearities.softmax)
     params = lasagne.layers.get_all_params(net['probsout'], trainable=True)
     return net, params
Exemplo n.º 12
0
 def build_model(self, incoming):
     net, params = super(ResNet_FullPre_Wide_Multiclass,
                         self).build_model(incoming)
     net['probsout'] = lasagne.layers.DenseLayer(
         net['featsout'],
         W=HeNormal(),
         num_units=self.out_size,
         nonlinearity=lasagne.nonlinearities.sigmoid)
     params = lasagne.layers.get_all_params(net['probsout'], trainable=True)
     return net, params
Exemplo n.º 13
0
def ResNet_BottleNeck_FullPreActivation(
        input_shape=(None, 3, PIXELS, PIXELS), input_var=None, n_classes=10,
        n=18):
    '''
    Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.
    Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)

    Judging from https://github.com/KaimingHe/resnet-1k-layers/blob/master/resnet-pre-act.lua.
    Number of filters go 16 -> 64 -> 128 -> 256

    Forumala to figure out depth: 9n + 2
    '''

    # Building the network
    l_in = InputLayer(shape=input_shape, input_var=input_var)

    # first layer, output is 16x16x16
    l = batch_norm(
        ConvLayer(l_in,
                  num_filters=16,
                  filter_size=(3, 3),
                  stride=(1, 1),
                  nonlinearity=rectify,
                  pad='same',
                  W=he_norm))

    # first stack of residual blocks, output is 64x16x16
    l = residual_bottleneck_block(l, first=True)
    for _ in range(1, n):
        l = residual_bottleneck_block(l)

    # second stack of residual blocks, output is 128x8x8
    l = residual_bottleneck_block(l, increase_dim=True)
    for _ in range(1, n):
        l = residual_bottleneck_block(l)

    # third stack of residual blocks, output is 256x4x4
    l = residual_bottleneck_block(l, increase_dim=True)
    for _ in range(1, n):
        l = residual_bottleneck_block(l)

    bn_post_conv = BatchNormLayer(l)
    bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)

    # average pooling
    avg_pool = GlobalPoolLayer(bn_post_relu)

    # fully connected layer
    network = DenseLayer(avg_pool,
                         num_units=n_classes,
                         W=HeNormal(),
                         nonlinearity=softmax)

    return network
Exemplo n.º 14
0
 def build_cnn(self):
     # Input Layer
     l_in = InputLayer(self.input_shape, input_var=self.input_var)
     # Conv1
     l_conv1 = Conv2DLayer(l_in, num_filters=NUM_FILTERS1, filter_size=3,
                           nonlinearity=rectify, W=HeNormal())
     l_drop1 = spatial_dropout(l_conv1, DROPOUT)
     # Conv2
     l_conv2 = Conv2DLayer(l_drop1, num_filters=NUM_FILTERS2, filter_size=2,
                           stride=2, nonlinearity=rectify, W=HeNormal())
     l_drop2 = spatial_dropout(l_conv2, 2*DROPOUT)
     # Pool
     l_max = MaxPool2DLayer(l_drop2, pool_size=(2, 2))
     l_max = batch_norm(l_max)
     # FC
     l_dense = DenseLayer(l_max, num_units=NUM_FILTERS3, nonlinearity=rectify)
     l_drop3 = dropout(l_dense, 4*DROPOUT)
     # Softmax Output
     l_out = DenseLayer(l_drop3, num_units=2, nonlinearity=softmax)
     self.model = l_out
 def conv_norm_lrelu(l_in, feat_out):
     l = Conv3DLayer(l_in,
                     feat_out,
                     3,
                     1,
                     'same',
                     nonlinearity=linear,
                     W=HeNormal(gain='relu'))
     if do_norm:
         l = BatchNormLayer(l, axes=axes)
     return NonlinearityLayer(l, nonlin)
 def norm_lrelu_conv(l_in, feat_out, stride=1, filter_size=3):
     if do_norm:
         l_in = BatchNormLayer(l_in, axes=axes)
     l = NonlinearityLayer(l_in, nonlin)
     return Conv3DLayer(l,
                        feat_out,
                        filter_size,
                        stride,
                        'same',
                        nonlinearity=linear,
                        W=HeNormal(gain='relu'))
Exemplo n.º 17
0
def make_net(W, H, size1=20, size2=15):
    net = NeuralNet(
        layers=[
            ('input', InputLayer),
            ('dense1', DenseLayer),
            ('dense2', DenseLayer),
            ('output', DenseLayer),
        ],
        input_shape=(None, W * H),
        dense1_num_units=size1,
        dense1_nonlinearity=LeakyRectify(leakiness=0.1),
        dense1_W=HeNormal(),
        dense1_b=Constant(),
        dense2_num_units=size2,
        dense2_nonlinearity=LeakyRectify(leakiness=0.1),
        dense2_W=HeNormal(),
        dense2_b=Constant(),
        output_num_units=4,
        output_nonlinearity=softmax,
        output_W=HeNormal(),
        output_b=Constant(),
        update=nesterov_momentum,  # todo
        update_learning_rate=shared(float32(1.)),
        update_momentum=0.9,
        max_epochs=200,
        on_epoch_finished=[
            StopWhenOverfitting(),
            StopAfterMinimum(),
            AdjustLearningRate(1., 0.0001),
        ],

        #label_encoder = False,
        regression=True,
        verbose=1,
        batch_iterator_train=BatchIterator(batch_size=128),  # todo
        batch_iterator_test=BatchIterator(batch_size=128),
        train_split=TrainSplit(eval_size=0.1),
    )
    net.initialize()

    return net
Exemplo n.º 18
0
def getNet6():
  inputLayer = layers.InputLayer(shape=(None, 1, imageShape[0], imageShape[1])) #120x120

  conv1Layer = layers.Conv2DLayer(inputLayer, num_filters=32, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #120x120
  conv2Layer = layers.Conv2DLayer(conv1Layer, num_filters=32, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #120x120
  pool1Layer = layers.MaxPool2DLayer(conv2Layer, pool_size=(2,2)) #60x60
  conv3Layer = layers.Conv2DLayer(pool1Layer, num_filters=64, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #60x60
  conv4Layer = layers.Conv2DLayer(conv3Layer, num_filters=64, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #60x60
  conv5Layer = layers.Conv2DLayer(conv4Layer, num_filters=64, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #60x60
  pool2Layer = layers.MaxPool2DLayer(conv5Layer, pool_size=(2,2)) #30x30
  conv6Layer = layers.Conv2DLayer(pool2Layer, num_filters=128, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #30x30
  conv7Layer = layers.Conv2DLayer(conv6Layer, num_filters=128, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #30x30
  conv8Layer = layers.Conv2DLayer(conv7Layer, num_filters=128, filter_size=(3,3), pad=(1,1), W=HeNormal('relu'), nonlinearity=rectify) #30x30
  pool3Layer = layers.MaxPool2DLayer(conv8Layer, pool_size=(2,2)) #15x15
  conv9Layer = layers.Conv2DLayer(pool3Layer, num_filters=256, filter_size=(4,4), W=HeNormal('relu'), nonlinearity=rectify) #12x12
  flattenLayer = layers.FlattenLayer(conv9Layer)
  hidden1Layer = layers.DenseLayer(flattenLayer, num_units=1024, W=HeNormal('relu'), nonlinearity=rectify)
  dropout1Layer = layers.DropoutLayer(hidden1Layer, p=0.5)
  hidden2Layer = layers.DenseLayer(dropout1Layer, num_units=512, W=HeNormal('relu'), nonlinearity=rectify)
  dropout2Layer = layers.DropoutLayer(hidden2Layer, p=0.5)
  hidden3Layer = layers.DenseLayer(dropout2Layer, num_units=256, W=HeNormal('relu'), nonlinearity=rectify)
  dropout3Layer = layers.DropoutLayer(hidden3Layer, p=0.5)
  hidden4Layer = layers.DenseLayer(dropout3Layer, num_units=128, W=HeNormal('relu'), nonlinearity=rectify)
  outputLayer = layers.DenseLayer(hidden4Layer, num_units=10, W=HeNormal('relu'), nonlinearity=softmax)
  return outputLayer
Exemplo n.º 19
0
def define_network(inputs):

    network = lasagne.layers.InputLayer(shape=(None, params.CHANNELS, params.INPUT_SIZE, params.INPUT_SIZE, params.INPUT_SIZE),
                                input_var=inputs)

    network = Conv3DDNNLayer(
            network, num_filters=64, filter_size=(5, 5, 5),
            nonlinearity=lasagne.nonlinearities.leaky_rectify,
            W=HeNormal(gain='relu'))

    network = MaxPool3DDNNLayer(network, pool_size=(2, 2, 2))

    if params.BATCH_NORMALIZATION:
        network = lasagne.layers.batch_norm(network)

    network = Conv3DDNNLayer(
            network, num_filters=64, filter_size=(5, 5, 5),
            nonlinearity=lasagne.nonlinearities.leaky_rectify,
            W=HeNormal(gain='relu'))

    network = Conv3DDNNLayer(
            network, num_filters=96, filter_size=(5, 5, 5),
            nonlinearity=lasagne.nonlinearities.leaky_rectify,
            W=HeNormal(gain='relu'))

    if params.BATCH_NORMALIZATION:
        network = lasagne.layers.batch_norm(network)

    network = lasagne.layers.DenseLayer(
            network,
            num_units=420,
            nonlinearity=lasagne.nonlinearities.leaky_rectify,
            W=HeNormal(gain='relu')
    )

    network = lasagne.layers.DenseLayer(
            network, num_units=params.N_CLASSES,
            nonlinearity=lasagne.nonlinearities.softmax)

    return network
Exemplo n.º 20
0
 def get_network(self):
     network = lasagne.layers.InputLayer(shape=(None, self.num_features),
                                         input_var=self.input_var)
     network = lasagne.layers.DropoutLayer(network, p=self.dropout)
     network = DenseLayer(network,
                          nonlinearity=rectify,
                          num_units=self.num_nodes,
                          W=HeNormal(gain=.01))
     for _ in xrange(0, self.num_layers):
         network = self.add_residual_dense_maxout_block(
             network, self.num_nodes, self.dropout)
     return lasagne.layers.DenseLayer(
         network, num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
Exemplo n.º 21
0
 def build_model(self, img_batch, pose_code):        
         
     img_size = self.options['img_size']
     pose_code_size = self.options['pose_code_size']                        
     filter_size = self.options['filter_size']        
     batch_size = img_batch.shape[0]
     
     # image encoding        
     l_in = InputLayer(shape = [None, img_size[0], img_size[1], img_size[2]], input_var=img_batch)
     l_in_dimshuffle = DimshuffleLayer(l_in, (0,3,1,2))
     
     l_conv1_1 = Conv2DLayer(l_in_dimshuffle, num_filters=64, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))        
     l_conv1_2 = Conv2DLayer(l_conv1_1, num_filters=64, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
     l_pool1 = MaxPool2DLayer(l_conv1_2, pool_size=(2,2)) 
     
     # pose encoding
     l_in_2 = InputLayer(shape=(None, pose_code_size), input_var=pose_code)     
     l_pose_1 = DenseLayer(l_in_2, num_units=512, W=HeNormal(),nonlinearity=rectify)
     l_pose_2 = DenseLayer(l_pose_1, num_units=pose_code_size*l_pool1.output_shape[2]*l_pool1.output_shape[3], W=HeNormal(),nonlinearity=rectify)
     l_pose_reshape = ReshapeLayer(l_pose_2, shape=(batch_size, pose_code_size, l_pool1.output_shape[2], l_pool1.output_shape[3])) 
     
     # deeper fusion
     l_concat = ConcatLayer([l_pool1, l_pose_reshape], axis=1)
     l_pose_conv_1 = Conv2DLayer(l_concat, num_filters=128, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2)) 
     l_pose_conv_2 = Conv2DLayer(l_pose_conv_1, num_filters=128, filter_size=filter_size, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
     
     l_pool2 = MaxPool2DLayer(l_pose_conv_2, pool_size=(2,2))         
     l_conv_3 = Conv2DLayer(l_pool2, num_filters=128, filter_size=(1,1), W=HeNormal()) 
     l_unpool1 = Unpool2DLayer(l_conv_3, ds = (2,2))
     
     # image decoding
     l_deconv_conv1_1 = Conv2DLayer(l_unpool1, num_filters=128, filter_size=filter_size, nonlinearity=rectify,W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
     l_deconv_conv1_2 = Conv2DLayer(l_deconv_conv1_1, num_filters=64, filter_size=filter_size, nonlinearity=rectify,W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
     
     l_unpool2 = Unpool2DLayer(l_deconv_conv1_2, ds = (2,2))
     l_deconv_conv2_1 = Conv2DLayer(l_unpool2, num_filters=64, filter_size=filter_size, nonlinearity=None, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))
     l_deconv_conv2_2 = Conv2DLayer(l_deconv_conv2_1, num_filters=img_size[2], filter_size=filter_size, nonlinearity=None, W=HeNormal(), pad=(filter_size[0]//2, filter_size[1]//2))  
             
     return l_deconv_conv2_2, l_pose_reshape
Exemplo n.º 22
0
    def cifar_model(cls, n=9, incoming=None, classes=10, **kwargs):
        """Create model for the CIFAR data set like in section 4.2.

        Parameters
        ----------
        n : integer (``9``)
            A parameter used to govern the size of the network as
            described in the paper.
        incoming :  a :class:`Layer` instance or ``None``
            The input layer, if it is ``None`` a new one will be created.
        classes : integer (`10``)
            The number of classes to train, usually ``10`` or ``100``.
        kwargs : key-word arguments
            The key-word arguments that get passed down to the constructor.

        Returns
        -------
        a :class:`DenseLayer` instance
            The model in form of its last layer.
        """
        model = incoming or InputLayer(shape=(None, 3, 32, 32))
        builder = cls(model, **kwargs)

        # first layer, output is 16 x 32 x 32
        model = builder.convolution(model, 16, init_gain=1.0)
        model = builder.nonlinearity(model)
        builder.model = model

        # first stack of residual blocks, output is 16 x 32 x 32
        for _ in range(n):
            builder.add_residual_block(16)

        # second stack of residual blocks, output is 32 x 16 x 16
        builder.add_residual_block(32, dim_inc=True)
        for _ in range(1, n):
            builder.add_residual_block(32)

        # third stack of residual blocks, output is 64 x 8 x 8
        builder.add_residual_block(64, dim_inc=True)
        for _ in range(1, n):
            builder.add_residual_block(64)

        # average pooling
        model = GlobalPoolLayer(builder.model)

        # fully connected layer
        model = DenseLayer(model,
                           num_units=classes,
                           W=HeNormal(gain='relu'),
                           nonlinearity=softmax)
        return model
Exemplo n.º 23
0
def ResNet_FullPreActivation(input_var=None, n=18):
    """
    Adapted from https://github.com/Lasagne/Recipes/tree/master/papers/deep_residual_learning.
    Tweaked to be consistent with 'Identity Mappings in Deep Residual Networks', Kaiming He et al. 2016 (https://arxiv.org/abs/1603.05027)

    Formula to figure out depth: 6n + 2
    """

    # Building the network
    l_in = InputLayer(shape=(None, 3, PIXELS, PIXELS), input_var=input_var)

    # first layer, output is 16 x 32 x 32
    l = batch_norm(
        ConvLayer(l_in,
                  num_filters=16,
                  filter_size=(3, 3),
                  stride=(1, 1),
                  nonlinearity=rectify,
                  pad='same',
                  W=he_norm))

    # first stack of residual blocks, output is 16 x 32 x 32
    l = residual_block(l, first=True)
    for _ in range(1, n):
        l = residual_block(l)

    # second stack of residual blocks, output is 32 x 16 x 16
    l = residual_block(l, increase_dim=True)
    for _ in range(1, n):
        l = residual_block(l)

    # third stack of residual blocks, output is 64 x 8 x 8
    l = residual_block(l, increase_dim=True)
    for _ in range(1, n):
        l = residual_block(l)

    bn_post_conv = BatchNormLayer(l)
    bn_post_relu = NonlinearityLayer(bn_post_conv, rectify)

    # average pooling
    avg_pool = GlobalPoolLayer(bn_post_relu)

    # fully connected layer
    network = DenseLayer(avg_pool,
                         num_units=10,
                         W=HeNormal(),
                         nonlinearity=softmax)

    return network
Exemplo n.º 24
0
 def convolution(incoming,
                 num_filters,
                 filter_size=(3, 3),
                 stride=(1, 1),
                 init_gain='relu'):
     """Standard convolution method."""
     return Conv2DLayer(incoming,
                        num_filters=num_filters,
                        filter_size=filter_size,
                        stride=stride,
                        nonlinearity=None,
                        pad='same',
                        W=HeNormal(gain=init_gain),
                        b=None,
                        flip_filters=False)
Exemplo n.º 25
0
    def _model_definition(self, net):
        """
        Builds the architecture of the network
        """
        he_norm = HeNormal(gain='relu')
        # Input filtering and downsampling with max pooling
        net = batch_norm(
            Conv2DLayer(net,
                        num_filters=32,
                        filter_size=7,
                        pad='same',
                        nonlinearity=rectify,
                        W=he_norm))
        net = MaxPool2DLayer(net, 2)

        net = batch_norm(
            Conv2DLayer(net,
                        num_filters=64,
                        filter_size=3,
                        pad='same',
                        nonlinearity=rectify,
                        W=he_norm))
        net = MaxPool2DLayer(net, 2)

        net = batch_norm(
            Conv2DLayer(net,
                        num_filters=128,
                        filter_size=3,
                        pad='same',
                        nonlinearity=rectify,
                        W=he_norm))
        net = batch_norm(
            Conv2DLayer(net,
                        num_filters=128,
                        filter_size=3,
                        pad='same',
                        nonlinearity=rectify,
                        W=he_norm))

        net = batch_norm(
            DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm))
        net = batch_norm(
            DenseLayer(net, num_units=1024, nonlinearity=rectify, W=he_norm))

        # Pooling
        #net = MaxPool1DLayer(net, 1)

        return net
Exemplo n.º 26
0
def test_he_normal_gain():
    from lasagne.init import HeNormal

    sample = HeNormal(gain=10.0).sample((100, 100))
    assert -0.1 < sample.mean() < 0.1
    assert 0.9 < sample.std() < 1.1

    sample = HeNormal(gain='relu').sample((200, 50))
    assert -0.1 < sample.mean() < 0.1
    assert 0.07 < sample.std() < 0.12
 def norm_lrelu_upscale_conv_norm_lrelu(l_in, feat_out):
     if do_norm:
         l_in = BatchNormLayer(l_in, axes=axes)
     l = NonlinearityLayer(l_in, nonlin)
     l = Upscale3DLayer(l, 2)
     l = Conv3DLayer(l,
                     feat_out,
                     3,
                     1,
                     'same',
                     nonlinearity=linear,
                     W=HeNormal(gain='relu'))
     if do_norm:
         l = BatchNormLayer(l, axes=axes)
     l = NonlinearityLayer(l, nonlin)
     return l
def fusion_predict(fusion_snapshot_path, out_test_data_path,
                   stage2_data_root_dir, submist_save_file_path, fixed_size):
    # all adapters
    adapters = []
    adapters.append(adapter1((48, 48), 'fusion/fcn1/96.npz'))
    adapters.append(adapter2((48, 48), 'fusion/fcn2/92.npz'))
    adapters.append(adapter3((48, 48), 'fusion/fcn3/52.npz'))
    adapters.append(adapter4((48, 48), 'fusion/fcn4/470.npz'))
    adapters.append(adapter5((48, 48), 'fusion/fcn5/280.npz'))
    adapters.append(adapter6((48, 48), 'fusion/fcn6/114.npz'))

    # input tensor
    pred = T.tensor4('pred')
    location = T.vector('location')
    resolution = T.matrix('resolution')
    target_volume = T.fscalar('volume')

    # fusion layers
    l_in = InputLayer(shape=(None, len(adapters), fixed_size[0],
                             fixed_size[1]),
                      input_var=pred)
    mid = Conv2DLayer(l_in, num_filters=1, filter_size=(1, 1), W=HeNormal())
    l_out = GlobalPoolLayer(mid)

    test_area = lasagne.layers.get_output(l_out, deterministic=True).flatten()

    test_pred_volume = utee.build_volume2(test_area, location, resolution,
                                          fixed_size)

    test_fn = theano.function([pred, location, resolution],
                              [test_area, test_pred_volume])

    area_fn = theano.function([pred], test_area)

    if os.path.exists(fusion_snapshot_path):
        with np.load(fusion_snapshot_path) as f:
            param_values = [f['arr_{}'.format(i)] for i in range(len(f.files))]
        print('resuming snapshot from {}'.format(fusion_snapshot_path))
        param_cur = lasagne.layers.get_all_params(l_out)
        assert len(param_cur) == len(param_values)
        for p, v in zip(param_cur, param_values):
            p.set_value(v)
    else:
        print("snapshot {} not found".format(fusion_snapshot_path))

    bm.fusion_submit(stage2_data_root_dir, out_test_data_path, adapters,
                     area_fn, test_fn, fixed_size, submist_save_file_path)
Exemplo n.º 29
0
 def add_residual_dense_maxout_block(self,
                                     network,
                                     num_nodes=240,
                                     dropout=0.5):
     network = lasagne.layers.DropoutLayer(network, p=self.dropout)
     identity = network
     network = DenseLayer(network,
                          nonlinearity=rectify,
                          num_units=self.num_nodes,
                          W=HeNormal(gain=.01))
     network = FeaturePoolLayer(incoming=network,
                                pool_size=2,
                                axis=1,
                                pool_function=theano.tensor.max)
     return NonlinearityLayer(ElemwiseSumLayer(
         [identity, network.input_layer]),
                              nonlinearity=rectify)
Exemplo n.º 30
0
    def image_net_model(cls, config, incoming=None, classes=1000, **kwargs):
        # staring block
        model = incoming or InputLayer(shape=(None, 3, 224, 224))
        builder = cls(model, **kwargs)

        model = builder.convolution(model,
                                    64,
                                    filter_size=(7, 7),
                                    stride=(2, 2),
                                    init_gain=1.0)
        model = MaxPool2DLayer(model,
                               pool_size=(3, 3),
                               stride=(2, 2),
                               ignore_border=False)
        builder.model = model

        config = iter(config)

        # no increasing dimensions on the first chuck
        n, num_filters = next(config)
        for _ in range(n):
            builder.add_residual_block(num_filters)

        # add other residual blocks
        for n, num_filters in config:
            builder.add_residual_block(num_filters, dim_inc=True)
            for _ in range(1, n):
                builder.add_residual_block(num_filters)

        model = builder.nonlinearity(BatchNormLayer(builder.model))

        # final part of the network
        model = Pool2DLayer(model,
                            pool_size=(7, 7),
                            stride=(1, 1),
                            mode='average_exc_pad',
                            ignore_border=False)
        model = DenseLayer(model,
                           num_units=classes,
                           W=HeNormal(gain='relu'),
                           nonlinearity=softmax)
        return model
Exemplo n.º 31
0
def add_transition(incoming, num_filters, dropout):
    layer = ScaleLayer(incoming)
    layer = BiasLayer(layer)
    layer = NonlinearityLayer(layer, nonlinearity=rectify)
    # Reduce the number of filters
    layer = Conv2DLayer(layer,
                        num_filters=num_filters,
                        filter_size=(1, 1),
                        stride=(1, 1),
                        W=HeNormal(gain='relu'), b=None,
                        flip_filters=False,
                        nonlinearity=None)
    if dropout > 0:
        layer = DropoutLayer(layer, p=dropout)
    # Pooling layer to reduce the last two dimensions by half
    layer = Pool2DLayer(layer,
                        pool_size=(2, 2),
                        stride=(2, 2),
                        mode='average_exc_pad')
    layer = BatchNormLayer(layer, beta=None, gamma=None)
    return layer
Exemplo n.º 32
0
def test_he_normal():
    from lasagne.init import HeNormal

    sample = HeNormal().sample((100, 100))
    assert -0.01 < sample.mean() < 0.01
    assert 0.09 < sample.std() < 0.11
Exemplo n.º 33
0
def test_he_normal_receptive_field():
    from lasagne.init import HeNormal

    sample = HeNormal().sample((50, 50, 2))
    assert -0.01 < sample.mean() < 0.01
    assert 0.09 < sample.std() < 0.11
Exemplo n.º 34
0
def test_he_normal_c01b():
    from lasagne.init import HeNormal

    sample = HeNormal(c01b=True).sample((25, 2, 2, 25))
    assert -0.01 < sample.mean() < 0.01
    assert 0.09 < sample.std() < 0.11
                for p, udn in zip(tparams, updir_new)]
    f_update = theano.function([], [], updates=updir_new + param_up,
                               on_unused_input='ignore',
                               name='rmsprop_f_update')
    params = [zipped_grads, running_grads, running_grads2, updir]
    return (f_grad_shared, f_update, params)


##################################################################
# Variables and Constants
MAX_ITERS = 150  # 450

CONV_EPS = 1e-5
DFLT_VDIM = 100

_HE_NORMAL = HeNormal()
HE_NORMAL = lambda x: floatX(_HE_NORMAL.sample(x))

_HE_UNIFORM = HeUniform()
HE_UNIFORM = lambda x: floatX(_HE_UNIFORM.sample(x))

_HE_UNIFORM_RELU = HeUniform(gain=np.sqrt(2))
HE_UNIFORM_RELU = lambda x: floatX(_HE_UNIFORM_RELU.sample(x))

_RELU_ALPHA = 0.
_HE_UNIFORM_LEAKY_RELU = HeUniform(
    gain=np.sqrt(2. / (1 + (_RELU_ALPHA or 1e-6)**2)))
HE_UNIFORM_LEAKY_RELU = lambda x: \
    floatX(_HE_UNIFORM_LEAKY_RELU.sample(x))

_ORTHOGONAL = Orthogonal()