def conv_pool_block(l, num_filters, filter_length, i_block): l = DropoutLayer(l, p=self.drop_prob) l = Conv2DLayer(l, num_filters=num_filters, filter_size=[filter_length, 1], nonlinearity=identity, name='combined_conv_{:d}'.format(i_block)) if self.double_time_convs: l = Conv2DLayer(l, num_filters=num_filters, filter_size=[filter_length, 1], nonlinearity=identity, name='combined_conv_{:d}'.format(i_block)) if self.batch_norm: l = BatchNormLayer(l, epsilon=1e-4, alpha=self.batch_norm_alpha, nonlinearity=self.later_nonlin) else: l = NonlinearityLayer(l, nonlinearity=self.later_nonlin) l = Pool2DLayer(l, pool_size=[self.pool_time_length, 1], stride=[1, 1], mode=self.later_pool_mode) l = StrideReshapeLayer(l, n_stride=self.pool_time_stride) l = NonlinearityLayer(l, self.later_pool_nonlin) return l
def test_init_none_nonlinearity(self, NonlinearityLayer, dummy_input_layer): import lasagne.nonlinearities layer = NonlinearityLayer( dummy_input_layer, nonlinearity=None, ) assert layer.nonlinearity == lasagne.nonlinearities.identity
def get_layers(self): l = InputLayer([None, self.in_chans, self.input_time_length, 1]) if self.split_first_layer: l = DimshuffleLayer(l, pattern=[0, 3, 2, 1]) l = Conv2DLayer(l, num_filters=self.n_filters_time, filter_size=[self.filter_time_length, 1], nonlinearity=identity, name='time_conv') l = Conv2DAllColsLayer(l, num_filters=self.n_filters_spat, filter_size=[1, -1], nonlinearity=identity, name='spat_conv') else: #keep channel dim in first dim, so it will also be convolved over l = Conv2DLayer(l, num_filters=self.num_filters_time, filter_size=[self.filter_time_length, 1], nonlinearity=identity, name='time_conv') if self.batch_norm: l = BatchNormLayer(l, epsilon=1e-4, alpha=self.batch_norm_alpha, nonlinearity=self.conv_nonlin) else: l = NonlinearityLayer(l, nonlinearity=self.conv_nonlin) l = Pool2DLayer(l, pool_size=[self.pool_time_length, 1], stride=[1, 1], mode=self.pool_mode) l = NonlinearityLayer(l, self.pool_nonlin) l = StrideReshapeLayer(l, n_stride=self.pool_time_stride) l = DropoutLayer(l, p=self.drop_prob) l = Conv2DLayer(l, num_filters=self.n_classes, filter_size=[self.final_dense_length, 1], nonlinearity=identity, name='final_dense') l = FinalReshapeLayer(l) l = NonlinearityLayer(l, softmax) return lasagne.layers.get_all_layers(l)
def layer_vars(self, NonlinearityLayer, dummy_input_layer): nonlinearity = Mock() layer = NonlinearityLayer( dummy_input_layer, nonlinearity=nonlinearity, ) return { 'nonlinearity': nonlinearity, 'layer': layer, }
def batch_norm(layer, **kwargs): """ Apply batch normalization to an existing layer. This is a convenience function modifying an existing layer to include batch normalization: It will steal the layer's nonlinearity if there is one (effectively introducing the normalization right before the nonlinearity), remove the layer's bias if there is one (because it would be redundant), and add a :class:`BatchNormLayer` and :class:`NonlinearityLayer` on top. Parameters ---------- layer : A :class:`Layer` instance The layer to apply the normalization to; note that it will be irreversibly modified as specified above **kwargs Any additional keyword arguments are passed on to the :class:`BatchNormLayer` constructor. Returns ------- BatchNormLayer or NonlinearityLayer instance A batch normalization layer stacked on the given modified `layer`, or a nonlinearity layer stacked on top of both if `layer` was nonlinear. Examples -------- Just wrap any layer into a :func:`batch_norm` call on creating it: >>> from lasagne.layers import InputLayer, DenseLayer, batch_norm >>> from lasagne.nonlinearities import tanh >>> l1 = InputLayer((64, 768)) >>> l2 = batch_norm(DenseLayer(l1, num_units=500, nonlinearity=tanh)) This introduces batch normalization right before its nonlinearity: >>> from lasagne.layers import get_all_layers >>> [l.__class__.__name__ for l in get_all_layers(l2)] ['InputLayer', 'DenseLayer', 'BatchNormLayer', 'NonlinearityLayer'] """ nonlinearity = getattr(layer, 'nonlinearity', None) if nonlinearity is not None: layer.nonlinearity = nonlinearities.identity if hasattr(layer, 'b') and layer.b is not None: del layer.params[layer.b] layer.b = None bn_name = (kwargs.pop('name', None) or (getattr(layer, 'name', None) and layer.name + '_bn')) layer = BatchNormLayer(layer, name=bn_name, **kwargs) if nonlinearity is not None: from lasagne.layers.special import NonlinearityLayer nonlin_name = bn_name and bn_name + '_nonlin' layer = NonlinearityLayer(layer, nonlinearity, name=nonlin_name) return layer
def batch_norm2(layer, **kwargs): nonlinearity = getattr(layer, 'nonlinearity', None) if nonlinearity is not None: layer.nonlinearity = nn.nonlinearities.identity if hasattr(layer, 'b') and layer.b is not None: del layer.params[layer.b] layer.b = None layer = BatchNormLayer(layer, **kwargs) if nonlinearity is not None: from lasagne.layers.special import NonlinearityLayer layer = NonlinearityLayer(layer, nonlinearity, name='%s_nl' % layer.name) return layer
def build_encoder(net, encoder_specs, activation, name, p_drop_hidden, shared_net): # encoder specs is a tuple of string and tuple of integers for i, (transform, specs) in enumerate(encoder_specs): if transform == 'unpool': specs = net.get(specs) # if specs have already the name of the corresponding pool layer update = build_enc_layer(net.values()[-1], name, transform, specs, activation, i, p_drop_hidden, shared_net) net.update(update) # apply activation if i < len(encoder_specs) - 1: act_name = 'enc_activation_{}'.format(i) net[act_name] = NonlinearityLayer(net.values()[-1], nonlinearity=activation, name='{}_{}'.format( name, act_name)) # classfication layer activation -> softmax net['enc_softmax'] = NonlinearityLayer(net.values()[-1], nonlinearity=softmax, name=name + '_enc_softmax') return net['enc_softmax'], net
def get_layers(self): in_l = InputLayer((self.n_examples, self.n_time_steps, self.n_chans)) in_bandpass = InputLayer( (self.n_examples, self.n_time_steps, self.n_chans, self.n_filters)) l_bandpass = BandpassLayer([in_l, in_bandpass], n_filt_order=self.n_filt_order, truncate_gradient=self.truncate_gradient) # out comes examples x timesteps x chans x filters l_spat_filt = TensorDotLayer(l_bandpass, n_filters=self.n_spat_filters, axis=2) # still examples x timesteps x chans x filters l_square = NonlinearityLayer(l_spat_filt, T.sqr) # now adding empty chan dim so we can make pooling per output chan l_shape_pad = DimshuffleLayer(l_square, (0, 'x', 1, 2, 3)) # examples x convchans x timesteps x chans x filters l_pooled = Pool3DDNNLayer(l_shape_pad, pool_size=(self.n_pool_len, 1, 1), stride=1, mode='average_exc_pad') l_log = NonlinearityLayer(l_pooled, safe_log) # removing empty convchan dim again l_sliced = SliceLayer(l_log, indices=0, axis=1) # now examples x timesteps x chans x filters l_flat = FlattenLayer(l_sliced, outdim=3) # now examples x timesteps x features (chans * filters) l_dense = TensorDotLayer(l_flat, n_filters=1, axis=2) # now examples x timesteps x 1 l_nonlin = NonlinearityLayer(l_dense, sigmoid) return lasagne.layers.get_all_layers(l_nonlin)
def residual_block( l, batch_norm_alpha, batch_norm_epsilon, nonlinearity, survival_prob, add_after_nonlin, reduction_method, reduction_pool_mode, increase_units_factor=None, half_time=False, projection=False, ): assert survival_prob <= 1 and survival_prob >= 0 input_num_filters = l.output_shape[1] if increase_units_factor is not None: out_num_filters = int(input_num_filters * increase_units_factor) assert (out_num_filters - input_num_filters) % 2 == 0, ( "Need even " "number of extra channels in order to be able to pad correctly") else: out_num_filters = input_num_filters if (not half_time) or (reduction_method == 'conv'): stack_1 = batch_norm(Conv2DLayer(l, num_filters=out_num_filters, filter_size=(3, 3), stride=(1, 1), nonlinearity=nonlinearity, pad='same', W=lasagne.init.HeNormal(gain='relu')), epsilon=batch_norm_epsilon, alpha=batch_norm_alpha) else: assert half_time and reduction_method == 'pool' stack_1 = Pool2DLayer(l, pool_size=(3, 1), stride=(1, 1), pad=(1, 0), mode=reduction_pool_mode) # 1x1 conv here, therefore can do stride later without problems # otherwise would have to do stride here before # and make extra if condition later (only reshape with stride # in case of reduction method conv)... stack_1 = batch_norm(Conv2DLayer(stack_1, num_filters=out_num_filters, filter_size=(1, 1), stride=(1, 1), nonlinearity=nonlinearity, pad='same', W=lasagne.init.HeNormal(gain='relu')), epsilon=batch_norm_epsilon, alpha=batch_norm_alpha) if half_time: stack_1 = StrideReshapeLayer(stack_1, n_stride=2) stack_2 = batch_norm(Conv2DLayer(stack_1, num_filters=out_num_filters, filter_size=(3, 3), stride=(1, 1), nonlinearity=None, pad='same', W=lasagne.init.HeNormal(gain='relu')), epsilon=batch_norm_epsilon, alpha=batch_norm_alpha) # add shortcut connections shortcut = l if half_time: # note since we are only reshaping # this is ok both for later identity and later projection # 1x1 conv of projection is same if we do it before or after this reshape # (would not be true if it was anything but 1x1 conv(!)) shortcut = StrideReshapeLayer(shortcut, n_stride=2) if increase_units_factor is not None: if projection: # projection shortcut, as option B in paper shortcut = batch_norm(Conv2DLayer(shortcut, num_filters=out_num_filters, filter_size=(1, 1), stride=(1, 1), nonlinearity=None, pad='same', b=None), epsilon=batch_norm_epsilon, alpha=batch_norm_alpha) else: # identity shortcut, as option A in paper n_extra_chans = out_num_filters - input_num_filters shortcut = PadLayer(shortcut, [n_extra_chans // 2, 0, 0], batch_ndim=1) if add_after_nonlin: stack_2 = NonlinearityLayer(stack_2) block = ElemwiseSumLayer([stack_2, shortcut]) else: block = NonlinearityLayer(ElemwiseSumLayer([stack_2, shortcut]), nonlinearity=nonlinearity) if survival_prob != 1: # Hack to make both be broadcastable along empty third dim # Otherwise I get an error that they are of different type: # shortcut: TensorType(False,False,False,True) # block: TensorType4d(32) or sth shortcut = ExpressionLayer(shortcut, lambda x: T.addbroadcast(x, 3)) block = ExpressionLayer(block, lambda x: T.addbroadcast(x, 3)) block = RandomSwitchLayer(block, shortcut, survival_prob) return block
def get_layers(self): l = InputLayer([None, self.in_chans, self.input_time_length, 1]) if self.split_first_layer: l = DimshuffleLayer(l, pattern=[0, 3, 2, 1]) l = DropoutLayer(l, p=self.drop_in_prob) l = Conv2DLayer(l, num_filters=self.num_filters_time, filter_size=[self.filter_time_length, 1], nonlinearity=identity, name='time_conv') if self.double_time_convs: l = Conv2DLayer(l, num_filters=self.num_filters_time, filter_size=[self.filter_time_length, 1], nonlinearity=identity, name='time_conv') l = Conv2DAllColsLayer(l, num_filters=self.num_filters_spat, filter_size=[1, -1], nonlinearity=identity, name='spat_conv') else: #keep channel dim in first dim, so it will also be convolved over l = DropoutLayer(l, p=self.drop_in_prob) l = Conv2DLayer(l, num_filters=self.num_filters_time, filter_size=[self.filter_time_length, 1], nonlinearity=identity, name='time_conv') if self.double_time_convs: l = Conv2DLayer(l, num_filters=self.num_filters_time, filter_size=[self.filter_time_length, 1], nonlinearity=identity, name='time_conv') if self.batch_norm: l = BatchNormLayer(l, epsilon=1e-4, alpha=self.batch_norm_alpha, nonlinearity=self.first_nonlin) else: l = NonlinearityLayer(l, nonlinearity=self.first_nonlin) l = Pool2DLayer(l, pool_size=[self.pool_time_length, 1], stride=[1, 1], mode=self.first_pool_mode) l = StrideReshapeLayer(l, n_stride=self.pool_time_stride) l = NonlinearityLayer(l, self.first_pool_nonlin) def conv_pool_block(l, num_filters, filter_length, i_block): l = DropoutLayer(l, p=self.drop_prob) l = Conv2DLayer(l, num_filters=num_filters, filter_size=[filter_length, 1], nonlinearity=identity, name='combined_conv_{:d}'.format(i_block)) if self.double_time_convs: l = Conv2DLayer(l, num_filters=num_filters, filter_size=[filter_length, 1], nonlinearity=identity, name='combined_conv_{:d}'.format(i_block)) if self.batch_norm: l = BatchNormLayer(l, epsilon=1e-4, alpha=self.batch_norm_alpha, nonlinearity=self.later_nonlin) else: l = NonlinearityLayer(l, nonlinearity=self.later_nonlin) l = Pool2DLayer(l, pool_size=[self.pool_time_length, 1], stride=[1, 1], mode=self.later_pool_mode) l = StrideReshapeLayer(l, n_stride=self.pool_time_stride) l = NonlinearityLayer(l, self.later_pool_nonlin) return l l = conv_pool_block(l, self.num_filters_2, self.filter_length_2, 2) l = conv_pool_block(l, self.num_filters_3, self.filter_length_3, 3) l = conv_pool_block(l, self.num_filters_4, self.filter_length_4, 4) # Final part, transformed dense layer l = DropoutLayer(l, p=self.drop_prob) l = Conv2DLayer(l, num_filters=self.n_classes, filter_size=[self.final_dense_length, 1], nonlinearity=identity, name='final_dense') l = FinalReshapeLayer(l) l = NonlinearityLayer(l, self.final_nonlin) return lasagne.layers.get_all_layers(l)
def conv_layer(incoming, num_filters): tmp = Conv2DLayer(incoming, num_filters, 3, pad='valid') tmp = BatchNormLayer(tmp) if dropout: tmp = DropoutLayer(tmp, 0.3) return NonlinearityLayer(tmp)