def conv_pool_block(l, num_filters, filter_length, i_block):
l = DropoutLayer(l, p=self.drop_prob)
l = Conv2DLayer(l,
num_filters=num_filters,
filter_size=[filter_length, 1],
nonlinearity=identity,
name='combined_conv_{:d}'.format(i_block))
if self.double_time_convs:
l = Conv2DLayer(l,
num_filters=num_filters,
filter_size=[filter_length, 1],
nonlinearity=identity,
name='combined_conv_{:d}'.format(i_block))
if self.batch_norm:
l = BatchNormLayer(l,
epsilon=1e-4,
alpha=self.batch_norm_alpha,
nonlinearity=self.later_nonlin)
else:
l = NonlinearityLayer(l, nonlinearity=self.later_nonlin)
l = Pool2DLayer(l,
pool_size=[self.pool_time_length, 1],
stride=[1, 1],
mode=self.later_pool_mode)
l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
l = NonlinearityLayer(l, self.later_pool_nonlin)
return l
def test_init_none_nonlinearity(self, NonlinearityLayer,
dummy_input_layer):
import lasagne.nonlinearities
layer = NonlinearityLayer(
dummy_input_layer,
nonlinearity=None,
)
assert layer.nonlinearity == lasagne.nonlinearities.identity
def get_layers(self):
l = InputLayer([None, self.in_chans, self.input_time_length, 1])
if self.split_first_layer:
l = DimshuffleLayer(l, pattern=[0, 3, 2, 1])
l = Conv2DLayer(l,
num_filters=self.n_filters_time,
filter_size=[self.filter_time_length, 1],
nonlinearity=identity,
name='time_conv')
l = Conv2DAllColsLayer(l,
num_filters=self.n_filters_spat,
filter_size=[1, -1],
nonlinearity=identity,
name='spat_conv')
else: #keep channel dim in first dim, so it will also be convolved over
l = Conv2DLayer(l,
num_filters=self.num_filters_time,
filter_size=[self.filter_time_length, 1],
nonlinearity=identity,
name='time_conv')
if self.batch_norm:
l = BatchNormLayer(l,
epsilon=1e-4,
alpha=self.batch_norm_alpha,
nonlinearity=self.conv_nonlin)
else:
l = NonlinearityLayer(l, nonlinearity=self.conv_nonlin)
l = Pool2DLayer(l,
pool_size=[self.pool_time_length, 1],
stride=[1, 1],
mode=self.pool_mode)
l = NonlinearityLayer(l, self.pool_nonlin)
l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
l = DropoutLayer(l, p=self.drop_prob)
l = Conv2DLayer(l,
num_filters=self.n_classes,
filter_size=[self.final_dense_length, 1],
nonlinearity=identity,
name='final_dense')
l = FinalReshapeLayer(l)
l = NonlinearityLayer(l, softmax)
return lasagne.layers.get_all_layers(l)
def layer_vars(self, NonlinearityLayer, dummy_input_layer):
nonlinearity = Mock()
layer = NonlinearityLayer(
dummy_input_layer,
nonlinearity=nonlinearity,
)
return {
'nonlinearity': nonlinearity,
'layer': layer,
}
def batch_norm(layer, **kwargs):
"""
Apply batch normalization to an existing layer. This is a convenience
function modifying an existing layer to include batch normalization: It
will steal the layer's nonlinearity if there is one (effectively
introducing the normalization right before the nonlinearity), remove
the layer's bias if there is one (because it would be redundant), and add
a :class:`BatchNormLayer` and :class:`NonlinearityLayer` on top.
Parameters
----------
layer : A :class:`Layer` instance
The layer to apply the normalization to; note that it will be
irreversibly modified as specified above
**kwargs
Any additional keyword arguments are passed on to the
:class:`BatchNormLayer` constructor.
Returns
-------
BatchNormLayer or NonlinearityLayer instance
A batch normalization layer stacked on the given modified `layer`, or
a nonlinearity layer stacked on top of both if `layer` was nonlinear.
Examples
--------
Just wrap any layer into a :func:`batch_norm` call on creating it:
>>> from lasagne.layers import InputLayer, DenseLayer, batch_norm
>>> from lasagne.nonlinearities import tanh
>>> l1 = InputLayer((64, 768))
>>> l2 = batch_norm(DenseLayer(l1, num_units=500, nonlinearity=tanh))
This introduces batch normalization right before its nonlinearity:
>>> from lasagne.layers import get_all_layers
>>> [l.__class__.__name__ for l in get_all_layers(l2)]
['InputLayer', 'DenseLayer', 'BatchNormLayer', 'NonlinearityLayer']
"""
nonlinearity = getattr(layer, 'nonlinearity', None)
if nonlinearity is not None:
layer.nonlinearity = nonlinearities.identity
if hasattr(layer, 'b') and layer.b is not None:
del layer.params[layer.b]
layer.b = None
bn_name = (kwargs.pop('name', None)
or (getattr(layer, 'name', None) and layer.name + '_bn'))
layer = BatchNormLayer(layer, name=bn_name, **kwargs)
if nonlinearity is not None:
from lasagne.layers.special import NonlinearityLayer
nonlin_name = bn_name and bn_name + '_nonlin'
layer = NonlinearityLayer(layer, nonlinearity, name=nonlin_name)
return layer
def batch_norm2(layer, **kwargs):
nonlinearity = getattr(layer, 'nonlinearity', None)
if nonlinearity is not None:
layer.nonlinearity = nn.nonlinearities.identity
if hasattr(layer, 'b') and layer.b is not None:
del layer.params[layer.b]
layer.b = None
layer = BatchNormLayer(layer, **kwargs)
if nonlinearity is not None:
from lasagne.layers.special import NonlinearityLayer
layer = NonlinearityLayer(layer,
nonlinearity,
name='%s_nl' % layer.name)
return layer
def build_encoder(net, encoder_specs, activation, name, p_drop_hidden,
shared_net):
# encoder specs is a tuple of string and tuple of integers
for i, (transform, specs) in enumerate(encoder_specs):
if transform == 'unpool':
specs = net.get(specs)
# if specs have already the name of the corresponding pool layer
update = build_enc_layer(net.values()[-1], name, transform, specs,
activation, i, p_drop_hidden, shared_net)
net.update(update)
# apply activation
if i < len(encoder_specs) - 1:
act_name = 'enc_activation_{}'.format(i)
net[act_name] = NonlinearityLayer(net.values()[-1],
nonlinearity=activation,
name='{}_{}'.format(
name, act_name))
# classfication layer activation -> softmax
net['enc_softmax'] = NonlinearityLayer(net.values()[-1],
nonlinearity=softmax,
name=name + '_enc_softmax')
return net['enc_softmax'], net
def get_layers(self):
in_l = InputLayer((self.n_examples, self.n_time_steps, self.n_chans))
in_bandpass = InputLayer(
(self.n_examples, self.n_time_steps, self.n_chans, self.n_filters))
l_bandpass = BandpassLayer([in_l, in_bandpass],
n_filt_order=self.n_filt_order,
truncate_gradient=self.truncate_gradient)
# out comes examples x timesteps x chans x filters
l_spat_filt = TensorDotLayer(l_bandpass,
n_filters=self.n_spat_filters,
axis=2)
# still examples x timesteps x chans x filters
l_square = NonlinearityLayer(l_spat_filt, T.sqr)
# now adding empty chan dim so we can make pooling per output chan
l_shape_pad = DimshuffleLayer(l_square, (0, 'x', 1, 2, 3))
# examples x convchans x timesteps x chans x filters
l_pooled = Pool3DDNNLayer(l_shape_pad,
pool_size=(self.n_pool_len, 1, 1),
stride=1,
mode='average_exc_pad')
l_log = NonlinearityLayer(l_pooled, safe_log)
# removing empty convchan dim again
l_sliced = SliceLayer(l_log, indices=0, axis=1)
# now examples x timesteps x chans x filters
l_flat = FlattenLayer(l_sliced, outdim=3)
# now examples x timesteps x features (chans * filters)
l_dense = TensorDotLayer(l_flat, n_filters=1, axis=2)
# now examples x timesteps x 1
l_nonlin = NonlinearityLayer(l_dense, sigmoid)
return lasagne.layers.get_all_layers(l_nonlin)
def residual_block(
l,
batch_norm_alpha,
batch_norm_epsilon,
nonlinearity,
survival_prob,
add_after_nonlin,
reduction_method,
reduction_pool_mode,
increase_units_factor=None,
half_time=False,
projection=False,
):
assert survival_prob <= 1 and survival_prob >= 0
input_num_filters = l.output_shape[1]
if increase_units_factor is not None:
out_num_filters = int(input_num_filters * increase_units_factor)
assert (out_num_filters - input_num_filters) % 2 == 0, (
"Need even "
"number of extra channels in order to be able to pad correctly")
else:
out_num_filters = input_num_filters
if (not half_time) or (reduction_method == 'conv'):
stack_1 = batch_norm(Conv2DLayer(l,
num_filters=out_num_filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=nonlinearity,
pad='same',
W=lasagne.init.HeNormal(gain='relu')),
epsilon=batch_norm_epsilon,
alpha=batch_norm_alpha)
else:
assert half_time and reduction_method == 'pool'
stack_1 = Pool2DLayer(l,
pool_size=(3, 1),
stride=(1, 1),
pad=(1, 0),
mode=reduction_pool_mode)
# 1x1 conv here, therefore can do stride later without problems
# otherwise would have to do stride here before
# and make extra if condition later (only reshape with stride
# in case of reduction method conv)...
stack_1 = batch_norm(Conv2DLayer(stack_1,
num_filters=out_num_filters,
filter_size=(1, 1),
stride=(1, 1),
nonlinearity=nonlinearity,
pad='same',
W=lasagne.init.HeNormal(gain='relu')),
epsilon=batch_norm_epsilon,
alpha=batch_norm_alpha)
if half_time:
stack_1 = StrideReshapeLayer(stack_1, n_stride=2)
stack_2 = batch_norm(Conv2DLayer(stack_1,
num_filters=out_num_filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=None,
pad='same',
W=lasagne.init.HeNormal(gain='relu')),
epsilon=batch_norm_epsilon,
alpha=batch_norm_alpha)
# add shortcut connections
shortcut = l
if half_time:
# note since we are only reshaping
# this is ok both for later identity and later projection
# 1x1 conv of projection is same if we do it before or after this reshape
# (would not be true if it was anything but 1x1 conv(!))
shortcut = StrideReshapeLayer(shortcut, n_stride=2)
if increase_units_factor is not None:
if projection:
# projection shortcut, as option B in paper
shortcut = batch_norm(Conv2DLayer(shortcut,
num_filters=out_num_filters,
filter_size=(1, 1),
stride=(1, 1),
nonlinearity=None,
pad='same',
b=None),
epsilon=batch_norm_epsilon,
alpha=batch_norm_alpha)
else:
# identity shortcut, as option A in paper
n_extra_chans = out_num_filters - input_num_filters
shortcut = PadLayer(shortcut, [n_extra_chans // 2, 0, 0],
batch_ndim=1)
if add_after_nonlin:
stack_2 = NonlinearityLayer(stack_2)
block = ElemwiseSumLayer([stack_2, shortcut])
else:
block = NonlinearityLayer(ElemwiseSumLayer([stack_2, shortcut]),
nonlinearity=nonlinearity)
if survival_prob != 1:
# Hack to make both be broadcastable along empty third dim
# Otherwise I get an error that they are of different type:
# shortcut: TensorType(False,False,False,True)
# block: TensorType4d(32) or sth
shortcut = ExpressionLayer(shortcut, lambda x: T.addbroadcast(x, 3))
block = ExpressionLayer(block, lambda x: T.addbroadcast(x, 3))
block = RandomSwitchLayer(block, shortcut, survival_prob)
return block
def get_layers(self):
l = InputLayer([None, self.in_chans, self.input_time_length, 1])
if self.split_first_layer:
l = DimshuffleLayer(l, pattern=[0, 3, 2, 1])
l = DropoutLayer(l, p=self.drop_in_prob)
l = Conv2DLayer(l,
num_filters=self.num_filters_time,
filter_size=[self.filter_time_length, 1],
nonlinearity=identity,
name='time_conv')
if self.double_time_convs:
l = Conv2DLayer(l,
num_filters=self.num_filters_time,
filter_size=[self.filter_time_length, 1],
nonlinearity=identity,
name='time_conv')
l = Conv2DAllColsLayer(l,
num_filters=self.num_filters_spat,
filter_size=[1, -1],
nonlinearity=identity,
name='spat_conv')
else: #keep channel dim in first dim, so it will also be convolved over
l = DropoutLayer(l, p=self.drop_in_prob)
l = Conv2DLayer(l,
num_filters=self.num_filters_time,
filter_size=[self.filter_time_length, 1],
nonlinearity=identity,
name='time_conv')
if self.double_time_convs:
l = Conv2DLayer(l,
num_filters=self.num_filters_time,
filter_size=[self.filter_time_length, 1],
nonlinearity=identity,
name='time_conv')
if self.batch_norm:
l = BatchNormLayer(l,
epsilon=1e-4,
alpha=self.batch_norm_alpha,
nonlinearity=self.first_nonlin)
else:
l = NonlinearityLayer(l, nonlinearity=self.first_nonlin)
l = Pool2DLayer(l,
pool_size=[self.pool_time_length, 1],
stride=[1, 1],
mode=self.first_pool_mode)
l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
l = NonlinearityLayer(l, self.first_pool_nonlin)
def conv_pool_block(l, num_filters, filter_length, i_block):
l = DropoutLayer(l, p=self.drop_prob)
l = Conv2DLayer(l,
num_filters=num_filters,
filter_size=[filter_length, 1],
nonlinearity=identity,
name='combined_conv_{:d}'.format(i_block))
if self.double_time_convs:
l = Conv2DLayer(l,
num_filters=num_filters,
filter_size=[filter_length, 1],
nonlinearity=identity,
name='combined_conv_{:d}'.format(i_block))
if self.batch_norm:
l = BatchNormLayer(l,
epsilon=1e-4,
alpha=self.batch_norm_alpha,
nonlinearity=self.later_nonlin)
else:
l = NonlinearityLayer(l, nonlinearity=self.later_nonlin)
l = Pool2DLayer(l,
pool_size=[self.pool_time_length, 1],
stride=[1, 1],
mode=self.later_pool_mode)
l = StrideReshapeLayer(l, n_stride=self.pool_time_stride)
l = NonlinearityLayer(l, self.later_pool_nonlin)
return l
l = conv_pool_block(l, self.num_filters_2, self.filter_length_2, 2)
l = conv_pool_block(l, self.num_filters_3, self.filter_length_3, 3)
l = conv_pool_block(l, self.num_filters_4, self.filter_length_4, 4)
# Final part, transformed dense layer
l = DropoutLayer(l, p=self.drop_prob)
l = Conv2DLayer(l,
num_filters=self.n_classes,
filter_size=[self.final_dense_length, 1],
nonlinearity=identity,
name='final_dense')
l = FinalReshapeLayer(l)
l = NonlinearityLayer(l, self.final_nonlin)
return lasagne.layers.get_all_layers(l)
def conv_layer(incoming, num_filters):
tmp = Conv2DLayer(incoming, num_filters, 3, pad='valid')
tmp = BatchNormLayer(tmp)
if dropout:
tmp = DropoutLayer(tmp, 0.3)
return NonlinearityLayer(tmp)
