Esempio n. 1
0
 def test_broadcast(self, input_data, input_var, input_layer):
     from lasagne.layers.shape import DimshuffleLayer
     ds = DimshuffleLayer(input_layer, [0, 1, 2, 3, 4, 'x'])
     assert ds.output_shape == (2, 3, 1, 5, 7, 1)
     assert ds.get_output_for(input_var).eval({
         input_var: input_data
     }).shape == (2, 3, 1, 5, 7, 1)
Esempio n. 2
0
 def test_rearrange(self, input_data, input_var, input_layer):
     from lasagne.layers.shape import DimshuffleLayer
     ds = DimshuffleLayer(input_layer, [4, 3, 2, 1, 0])
     assert ds.output_shape == (7, 5, 1, 3, 2)
     assert ds.get_output_for(input_var).eval({
         input_var: input_data
     }).shape == (7, 5, 1, 3, 2)
Esempio n. 3
0
 def test_collapse_None(self, input_data, input_var, input_layer_with_None):
     from lasagne.layers.shape import DimshuffleLayer
     ds_ok = DimshuffleLayer(input_layer_with_None, [0, 1, 3, 4])
     assert ds_ok.output_shape == (2, 3, 5, 7)
     assert ds_ok.get_output_for(input_var).eval(
         {input_var: input_data}).shape == (2, 3, 5, 7)
     with pytest.raises(ValueError):
         DimshuffleLayer(input_layer_with_None, [0, 1, 2, 4])
Esempio n. 4
0
 def test_collapse_None(self, input_data, input_var, input_layer_with_None):
     from lasagne.layers.shape import DimshuffleLayer
     ds_ok = DimshuffleLayer(input_layer_with_None, [0, 1, 3, 4])
     assert ds_ok.output_shape == (2, 3, 5, 7)
     assert ds_ok.get_output_for(input_var).eval(
         {input_var: input_data}).shape == (2, 3, 5, 7)
     with pytest.raises(ValueError):
         DimshuffleLayer(input_layer_with_None, [0, 1, 2, 4])
Esempio n. 5
0
 def test_invalid_pattern(self, input_data, input_var, input_layer):
     from lasagne.layers.shape import DimshuffleLayer
     with pytest.raises(ValueError):
         DimshuffleLayer(input_layer, ['q'])
     with pytest.raises(ValueError):
         DimshuffleLayer(input_layer, [0, 0, 1, 3, 4])
     with pytest.raises(ValueError):
         # There is no dimension 42
         DimshuffleLayer(input_layer, [0, 1, 2, 4, 42])
Esempio n. 6
0
 def test_invalid_pattern(self, input_data, input_var, input_layer):
     from lasagne.layers.shape import DimshuffleLayer
     with pytest.raises(ValueError):
         DimshuffleLayer(input_layer, ['q'])
     with pytest.raises(ValueError):
         DimshuffleLayer(input_layer, [0, 0, 1, 3, 4])
     # There is no dimension 42
     ds_bad = DimshuffleLayer(input_layer, [0, 1, 2, 4, 42])
     with pytest.raises(ValueError):
         ds_bad.get_output_shape()
Esempio n. 7
0
    def get_layers(self):
        l = InputLayer([None, self.in_chans, self.input_time_length, 1])
        if self.split_first_layer:
            l = DimshuffleLayer(l, pattern=[0, 3, 2, 1])
            l = Conv2DLayer(l,
                            num_filters=self.n_filters_time,
                            filter_size=[self.filter_time_length, 1],
                            nonlinearity=identity,
                            name='time_conv')
            l = Conv2DAllColsLayer(l,
                                   num_filters=self.n_filters_spat,
                                   filter_size=[1, -1],
                                   nonlinearity=identity,
                                   name='spat_conv')
        else:  #keep channel dim in first dim, so it will also be convolved over
            l = Conv2DLayer(l,
                            num_filters=self.num_filters_time,
                            filter_size=[self.filter_time_length, 1],
                            nonlinearity=identity,
                            name='time_conv')
        if self.batch_norm:
            l = BatchNormLayer(l,
                               epsilon=1e-4,
                               alpha=self.batch_norm_alpha,
                               nonlinearity=self.conv_nonlin)
        else:
            l = NonlinearityLayer(l, nonlinearity=self.conv_nonlin)

        l = Pool2DLayer(l,
                        pool_size=[self.pool_time_length, 1],
                        stride=[1, 1],
                        mode=self.pool_mode)
        l = NonlinearityLayer(l, self.pool_nonlin)
        l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
        l = DropoutLayer(l, p=self.drop_prob)

        l = Conv2DLayer(l,
                        num_filters=self.n_classes,
                        filter_size=[self.final_dense_length, 1],
                        nonlinearity=identity,
                        name='final_dense')
        l = FinalReshapeLayer(l)
        l = NonlinearityLayer(l, softmax)
        return lasagne.layers.get_all_layers(l)
Esempio n. 8
0
    def get_layers(self):
        in_l = InputLayer((self.n_examples, self.n_time_steps, self.n_chans))
        in_bandpass = InputLayer(
            (self.n_examples, self.n_time_steps, self.n_chans, self.n_filters))

        l_bandpass = BandpassLayer([in_l, in_bandpass],
                                   n_filt_order=self.n_filt_order,
                                   truncate_gradient=self.truncate_gradient)

        # out comes examples x timesteps x chans x filters
        l_spat_filt = TensorDotLayer(l_bandpass,
                                     n_filters=self.n_spat_filters,
                                     axis=2)
        # still examples x timesteps x chans x filters
        l_square = NonlinearityLayer(l_spat_filt, T.sqr)
        # now adding empty chan dim so we can make pooling per output chan
        l_shape_pad = DimshuffleLayer(l_square, (0, 'x', 1, 2, 3))

        # examples x convchans x timesteps x chans x filters
        l_pooled = Pool3DDNNLayer(l_shape_pad,
                                  pool_size=(self.n_pool_len, 1, 1),
                                  stride=1,
                                  mode='average_exc_pad')

        l_log = NonlinearityLayer(l_pooled, safe_log)

        # removing empty convchan dim again
        l_sliced = SliceLayer(l_log, indices=0, axis=1)
        # now examples x timesteps x chans x filters
        l_flat = FlattenLayer(l_sliced, outdim=3)
        # now examples x timesteps x features (chans * filters)

        l_dense = TensorDotLayer(l_flat, n_filters=1, axis=2)
        # now examples x timesteps x 1
        l_nonlin = NonlinearityLayer(l_dense, sigmoid)
        return lasagne.layers.get_all_layers(l_nonlin)
Esempio n. 9
0
 def test_broadcast(self, input_data, input_var, input_layer):
     from lasagne.layers.shape import DimshuffleLayer
     ds = DimshuffleLayer(input_layer, [0, 1, 2, 3, 4, 'x'])
     assert ds.output_shape == (2, 3, 1, 5, 7, 1)
     assert ds.get_output_for(input_var).eval(
         {input_var: input_data}).shape == (2, 3, 1, 5, 7, 1)
Esempio n. 10
0
 def test_rearrange(self, input_data, input_var, input_layer):
     from lasagne.layers.shape import DimshuffleLayer
     ds = DimshuffleLayer(input_layer, [4, 3, 2, 1, 0])
     assert ds.output_shape == (7, 5, 1, 3, 2)
     assert ds.get_output_for(input_var).eval(
         {input_var: input_data}).shape == (7, 5, 1, 3, 2)
Esempio n. 11
0
    def get_layers(self):
        l = InputLayer([None, self.in_chans, self.input_time_length, 1])
        if self.split_first_layer:
            l = DimshuffleLayer(l, pattern=[0, 3, 2, 1])
            l = DropoutLayer(l, p=self.drop_in_prob)
            l = Conv2DLayer(l,
                            num_filters=self.num_filters_time,
                            filter_size=[self.filter_time_length, 1],
                            nonlinearity=identity,
                            name='time_conv')
            if self.double_time_convs:
                l = Conv2DLayer(l,
                                num_filters=self.num_filters_time,
                                filter_size=[self.filter_time_length, 1],
                                nonlinearity=identity,
                                name='time_conv')
            l = Conv2DAllColsLayer(l,
                                   num_filters=self.num_filters_spat,
                                   filter_size=[1, -1],
                                   nonlinearity=identity,
                                   name='spat_conv')
        else:  #keep channel dim in first dim, so it will also be convolved over
            l = DropoutLayer(l, p=self.drop_in_prob)
            l = Conv2DLayer(l,
                            num_filters=self.num_filters_time,
                            filter_size=[self.filter_time_length, 1],
                            nonlinearity=identity,
                            name='time_conv')
            if self.double_time_convs:
                l = Conv2DLayer(l,
                                num_filters=self.num_filters_time,
                                filter_size=[self.filter_time_length, 1],
                                nonlinearity=identity,
                                name='time_conv')
        if self.batch_norm:
            l = BatchNormLayer(l,
                               epsilon=1e-4,
                               alpha=self.batch_norm_alpha,
                               nonlinearity=self.first_nonlin)
        else:
            l = NonlinearityLayer(l, nonlinearity=self.first_nonlin)
        l = Pool2DLayer(l,
                        pool_size=[self.pool_time_length, 1],
                        stride=[1, 1],
                        mode=self.first_pool_mode)
        l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
        l = NonlinearityLayer(l, self.first_pool_nonlin)

        def conv_pool_block(l, num_filters, filter_length, i_block):
            l = DropoutLayer(l, p=self.drop_prob)
            l = Conv2DLayer(l,
                            num_filters=num_filters,
                            filter_size=[filter_length, 1],
                            nonlinearity=identity,
                            name='combined_conv_{:d}'.format(i_block))
            if self.double_time_convs:
                l = Conv2DLayer(l,
                                num_filters=num_filters,
                                filter_size=[filter_length, 1],
                                nonlinearity=identity,
                                name='combined_conv_{:d}'.format(i_block))
            if self.batch_norm:
                l = BatchNormLayer(l,
                                   epsilon=1e-4,
                                   alpha=self.batch_norm_alpha,
                                   nonlinearity=self.later_nonlin)
            else:
                l = NonlinearityLayer(l, nonlinearity=self.later_nonlin)
            l = Pool2DLayer(l,
                            pool_size=[self.pool_time_length, 1],
                            stride=[1, 1],
                            mode=self.later_pool_mode)
            l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
            l = NonlinearityLayer(l, self.later_pool_nonlin)
            return l

        l = conv_pool_block(l, self.num_filters_2, self.filter_length_2, 2)
        l = conv_pool_block(l, self.num_filters_3, self.filter_length_3, 3)
        l = conv_pool_block(l, self.num_filters_4, self.filter_length_4, 4)
        # Final part, transformed dense layer
        l = DropoutLayer(l, p=self.drop_prob)
        l = Conv2DLayer(l,
                        num_filters=self.n_classes,
                        filter_size=[self.final_dense_length, 1],
                        nonlinearity=identity,
                        name='final_dense')
        l = FinalReshapeLayer(l)
        l = NonlinearityLayer(l, self.final_nonlin)
        return lasagne.layers.get_all_layers(l)
Esempio n. 12
0
    def get_layers(self):
        def resnet_residual_block(model,
                                  increase_units_factor=None,
                                  half_time=False):
            """Calling residual_block function with correct attributes
            from this object.

            Parameters
            ----------
            model :
                
            increase_units_factor :
                (Default value = None)
            half_time :
                (Default value = False)

            Returns
            -------
            Final layer of created residual block.
            
            """
            return residual_block(model,
                                  batch_norm_epsilon=self.batch_norm_epsilon,
                                  batch_norm_alpha=self.batch_norm_alpha,
                                  increase_units_factor=increase_units_factor,
                                  half_time=half_time,
                                  nonlinearity=self.nonlinearity,
                                  projection=self.projection,
                                  survival_prob=self.survival_prob,
                                  add_after_nonlin=self.add_after_nonlin,
                                  reduction_method=self.reduction_method,
                                  reduction_pool_mode=self.reduction_pool_mode)

        model = InputLayer([None, self.in_chans, self.input_time_length, 1])

        if self.split_first_layer:
            # shift channel dim out
            model = DimshuffleLayer(model, (0, 3, 2, 1))
            # first timeconv
            model = Conv2DLayer(model,
                                num_filters=self.n_first_filters,
                                filter_size=(self.first_filter_length, 1),
                                stride=(1, 1),
                                nonlinearity=identity,
                                pad='same',
                                W=lasagne.init.HeNormal(gain='relu'))
            # now spatconv
            model = batch_norm(Conv2DLayer(
                model,
                num_filters=self.n_first_filters,
                filter_size=(1, self.in_chans),
                stride=(1, 1),
                nonlinearity=self.nonlinearity,
                pad=0,
                W=lasagne.init.HeNormal(gain='relu')),
                               epsilon=self.batch_norm_epsilon,
                               alpha=self.batch_norm_alpha)
        else:
            model = batch_norm(Conv2DLayer(
                model,
                num_filters=self.n_first_filters,
                filter_size=(self.first_filter_length, 1),
                stride=(1, 1),
                nonlinearity=self.nonlinearity,
                pad='same',
                W=lasagne.init.HeNormal(gain='relu')),
                               epsilon=self.batch_norm_epsilon,
                               alpha=self.batch_norm_alpha)
        for _ in range(self.n_layers_per_block):
            model = resnet_residual_block(model)

        model = resnet_residual_block(model,
                                      increase_units_factor=2,
                                      half_time=True)
        for _ in range(1, self.n_layers_per_block):
            model = resnet_residual_block(model)

        model = resnet_residual_block(model,
                                      increase_units_factor=1.5,
                                      half_time=True)
        for _ in range(1, self.n_layers_per_block):
            model = resnet_residual_block(model)

        model = resnet_residual_block(model, half_time=True)
        for _ in range(1, self.n_layers_per_block):
            model = resnet_residual_block(model)

        model = resnet_residual_block(model, half_time=True)
        for _ in range(1, self.n_layers_per_block):
            model = resnet_residual_block(model)

        model = resnet_residual_block(model, half_time=True)
        for _ in range(1, self.n_layers_per_block):
            model = resnet_residual_block(model)

        model = resnet_residual_block(model, half_time=True)

        if self.drop_before_pool:
            model = DropoutLayer(model, p=0.5)
        # Replacement for global mean pooling
        if self.final_aggregator == 'pool':
            model = Pool2DLayer(model,
                                pool_size=(self.final_pool_length, 1),
                                stride=(1, 1),
                                mode='average_exc_pad')
            model = Conv2DLayer(model,
                                filter_size=(1, 1),
                                num_filters=4,
                                W=lasagne.init.HeNormal(),
                                nonlinearity=identity)
        elif self.final_aggregator == 'conv':
            model = Conv2DLayer(model,
                                filter_size=(self.final_pool_length, 1),
                                num_filters=4,
                                W=lasagne.init.HeNormal(),
                                nonlinearity=identity)
        else:
            raise ValueError("Unknown final aggregator {:s}".format(
                self.final_aggregator))

        model = FinalReshapeLayer(model)
        model = NonlinearityLayer(model, nonlinearity=self.final_nonlin)
        model = set_survival_probs_to_linear_decay(model, self.survival_prob)
        return lasagne.layers.get_all_layers(model)
def build_nmt_encoder_decoder(dim_word=1,
                              n_embd=100,
                              n_units=500,
                              n_proj=200,
                              state=None,
                              rev_state=None,
                              context_type=None,
                              attention=False,
                              drop_p=None):
    enc = OrderedDict()
    enc['input'] = InputLayer((None, None), name='input')
    enc_mask = enc['mask'] = InputLayer((None, None), name='mask')
    enc_rev_mask = enc['rev_mask'] = InputLayer((None, None), name='rev_mask')

    enc['input_emb'] = EmbeddingLayer(enc.values()[-1],
                                      input_size=dim_word,
                                      output_size=n_embd,
                                      name='input_emb')

    ### ENCODER PART ###
    # rnn encoder unit
    hid_init = Constant(0.)
    hid_init_rev = Constant(0.)
    encoder_unit = get_rnn_unit(enc.values()[-1],
                                enc_mask,
                                enc_rev_mask,
                                hid_init,
                                hid_init_rev,
                                n_units,
                                prefix='encoder_')
    enc.update(encoder_unit)

    # context layer = decoder's initial state of shape (batch_size, num_units)
    context = enc.values()[-1]  # net['context']
    if context_type == 'last':
        enc['context2init'] = SliceLayer(context,
                                         indices=-1,
                                         axis=1,
                                         name='last_encoder_context')
    elif context_type == 'mean':
        enc['context2init'] = ExpressionLayer(context,
                                              mean_over_1_axis,
                                              output_shape='auto',
                                              name='mean_encoder_context')

    ### DECODER PART ###
    W_init2proj, b_init2proj = GlorotUniform(), Constant(0.)

    enc['init_state'] = DenseLayer(enc['context2init'],
                                   num_units=n_units,
                                   W=W_init2proj,
                                   b=b_init2proj,
                                   nonlinearity=tanh,
                                   name='decoder_init_state')
    if state is None:
        init_state = enc['init_state']
        init_state_rev = None  #if rev_state is None else init_state
        if not attention:
            # if simple attetion the context is 2D, else 3D
            context = enc['context2init']
    else:
        init_state = state
        init_state_rev = rev_state
        context = enc['context_input'] = \
            InputLayer((None, n_units), name='ctx_input')
    # (batch_size, nfeats)

    # (batch_size, valid ntsteps)
    enc['target'] = InputLayer((None, None), name='target')
    dec_mask = enc['target_mask'] = InputLayer((None, None),
                                               name='target_mask')

    enc['target_emb'] = EmbeddingLayer(enc.values()[-1],
                                       input_size=dim_word,
                                       output_size=n_embd,
                                       name='target_emb')
    prevdim = n_embd
    prev2rnn = enc.values()[-1]  # it's either emb or prev2rnn/noise

    decoder_unit = get_rnn_unit(prev2rnn,
                                dec_mask,
                                None,
                                init_state,
                                None,
                                n_units,
                                prefix='decoder_',
                                context=context,
                                attention=attention)
    enc.update(decoder_unit)

    if attention:
        ctxs = enc.values()[-1]
        ctxs_shape = ctxs.output_shape

        def get_ctx(x):
            return ctxs.ctx

        context = enc['context'] = ExpressionLayer(ctxs,
                                                   function=get_ctx,
                                                   output_shape=ctxs_shape,
                                                   name='context')

    # return all values'
    # reshape for feed-forward layer
    # 2D shapes of (batch_size * num_steps, num_units/num_feats)
    enc['rnn2proj'] = rnn2proj = ReshapeLayer(enc.values()[-1], (-1, n_units),
                                              name='flatten_rnn2proj')

    enc['prev2proj'] = prev2proj = ReshapeLayer(prev2rnn, (-1, prevdim),
                                                name='flatten_prev')

    if isinstance(context, ExpressionLayer):
        ctx2proj = enc['ctx2proj'] = ReshapeLayer(context,
                                                  (-1, ctxs_shape[-1]),
                                                  name='flatten_ctxs')
    else:
        ctx2proj = context

    # load shared parameters
    W_rnn2proj, b_rnn2proj = GlorotUniform(), Constant(0.)
    W_prev2proj, b_prev2proj = GlorotUniform(), Constant(0.)
    W_ctx2proj, b_ctx2proj = GlorotUniform(), Constant(0.)

    # perturb rnn-to-projection by noise
    if drop_p is not None:
        rnn2proj = enc['noise_rnn2proj'] = DropoutLayer(rnn2proj,
                                                        sigma=drop_p,
                                                        name='noise_rnn2proj')

        prev2proj = enc['drop_prev2proj'] = DropoutLayer(prev2proj,
                                                         sigma=drop_p,
                                                         name='drop_prev2proj')

        ctx2proj = enc['noise_ctx2proj'] = DropoutLayer(ctx2proj,
                                                        sigma=drop_p,
                                                        name='noise_ctx2proj')

    # project rnn
    enc['rnn_proj'] = DenseLayer(rnn2proj,
                                 num_units=n_proj,
                                 nonlinearity=linear,
                                 W=W_rnn2proj,
                                 b=b_rnn2proj,
                                 name='rnn_proj')

    # project raw targets
    enc['prev_proj'] = DenseLayer(prev2proj,
                                  num_units=n_proj,
                                  nonlinearity=linear,
                                  W=W_prev2proj,
                                  b=b_prev2proj,
                                  name='prev_proj')

    # project context
    enc['ctx_proj'] = DenseLayer(ctx2proj,
                                 num_units=n_proj,
                                 nonlinearity=linear,
                                 W=W_ctx2proj,
                                 b=b_ctx2proj,
                                 name='ctx_proj')

    # reshape back for merging
    n_batch = enc['input'].input_var.shape[0]
    rnn2merge = enc['rnn2merge'] = ReshapeLayer(enc['rnn_proj'],
                                                (n_batch, -1, n_proj),
                                                name='reshaped_rnn2proj')

    prev2merge = enc['prev2merge'] = ReshapeLayer(enc['prev_proj'],
                                                  (n_batch, -1, n_proj),
                                                  name='reshaped_prev')

    if isinstance(context, ExpressionLayer):
        ctx2merge = ReshapeLayer(enc['ctx_proj'], (n_batch, -1, n_proj),
                                 name='reshaped_prev')
    else:
        ctx2merge = enc['ctx2merge'] = DimshuffleLayer(enc['ctx_proj'],
                                                       pattern=(0, 'x', 1),
                                                       name='reshaped_context')

    # combine projections into shape (batch_size, n_steps, n_proj)
    enc['proj_merge'] = ElemwiseMergeLayer([rnn2merge, prev2merge, ctx2merge],
                                           merge_function=tanh_add,
                                           name='proj_merge')

    # reshape for output regression projection
    enc['merge2proj'] = ReshapeLayer(enc.values()[-1], (-1, n_proj),
                                     name='flatten_proj_merge')

    # perturb concatenated regressors by noise
    if drop_p is not None:
        # if noise_type == 'binary':
        enc['noise_output'] = DropoutLayer(enc.values()[-1],
                                           p=drop_p,
                                           name='noise_output')

    # regress on combined (perturbed) projections
    out = get_output_unit(enc['target'], enc.values()[-1], dim_word)
    enc.update(out)  # update graph

    return enc