def ResidualModule(input_layer,
num_filters=64,
nonlinearity=rectify,
normalize=False,
stride=(1, 1),
conv_dropout=0.0):
input_conv = Conv2DLayer(incoming=input_layer,
num_filters=num_filters,
filter_size=(3, 1),
stride=stride,
pad='same',
W=lasagne.init.GlorotUniform(),
nonlinearity=None,
b=None,
name='Residual module layer 1')
l_prev = BatchNormalizeLayer(input_conv,
normalize=normalize,
nonlinearity=nonlinearity)
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout, name='Tied Dropout')
l_prev = Conv2DLayer(incoming=l_prev,
num_filters=num_filters,
filter_size=(3, 1),
stride=(1, 1),
pad='same',
W=lasagne.init.GlorotUniform(),
nonlinearity=None,
b=None,
name='Residual module layer 2')
if normalize:
# Batch normalization is done "immediately after" convolutions
l_prev = BatchNormLayer(l_prev, name='Batch norm')
# Using 1x1 convolutions for shortcut projections. NiNLayer could be used as well
# but doesn't' support strides
l_skip = Conv2DLayer(input_layer,
num_filters=num_filters,
filter_size=(1, 1),
stride=stride,
nonlinearity=None,
b=None,
name='Shortcut')
l_prev = ElemwiseSumLayer((l_prev, l_skip), name='Elementwise sum')
# Add nonlinearity after summation
l_prev = NonlinearityLayer(l_prev,
nonlinearity=nonlinearity,
name='Non-linearity')
if not normalize:
l_prev = BiasLayer(l_prev, name='Bias')
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout, name='Tied Dropout')
return l_prev
def RecurrentConvLayer(input_layer,
t=3,
num_filters=64,
filter_size=7,
nonlinearity=rectify,
normalize=False,
rcl_dropout=0.0):
input_conv = Conv2DLayer(incoming=input_layer,
num_filters=num_filters,
filter_size=(1, 1),
stride=(1, 1),
pad='same',
W=lasagne.init.GlorotNormal(),
nonlinearity=None,
b=None,
name='RCL Conv2D input')
l_prev = BatchNormalizeLayer(input_conv,
normalize=normalize,
nonlinearity=nonlinearity)
l_prev = TiedDropoutLayer(l_prev,
p=rcl_dropout,
name='Recurrent conv dropout')
for _ in range(t):
l_prev = Conv2DLayer(incoming=l_prev,
num_filters=num_filters,
filter_size=(filter_size, 1),
stride=(1, 1),
pad='same',
W=lasagne.init.GlorotNormal(),
nonlinearity=None,
b=None,
name='Conv2D t=%d' % _)
l_prev = ElemwiseSumLayer((input_conv, l_prev), coeffs=1, name='Sum')
l_prev = BatchNormalizeLayer(l_prev,
normalize=normalize,
nonlinearity=nonlinearity)
l_prev = TiedDropoutLayer(l_prev,
p=rcl_dropout,
name='Recurrent conv dropout')
return l_prev
def __init__(self,
n_in,
n_filters,
filter_sizes,
n_out,
pool_sizes=None,
n_hidden=(512),
ccf=False,
trans_func=rectify,
out_func=softmax,
dense_dropout=0.0,
stats=2,
input_noise=0.0,
batch_norm=False,
conv_dropout=0.0):
super(CNN, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
# Define model using lasagne framework
dropout = True if not dense_dropout == 0.0 else False
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length, n_features))
l_prev = self.l_in
# Separate into raw values and statistics
sequence_length -= stats
stats_layer = SliceLayer(l_prev,
indices=slice(sequence_length, None),
axis=1)
stats_layer = ReshapeLayer(stats_layer, (-1, stats * n_features))
print('Stats layer shape', stats_layer.output_shape)
l_prev = SliceLayer(l_prev, indices=slice(0, sequence_length), axis=1)
print('Conv input layer shape', l_prev.output_shape)
# Apply input noise
l_prev = GaussianNoiseLayer(l_prev, sigma=input_noise)
if ccf:
self.log += "\nAdding cross-channel feature layer"
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features))
l_prev = Conv2DLayer(l_prev,
num_filters=4 * n_features,
filter_size=(1, n_features),
nonlinearity=None)
n_features *= 4
if batch_norm:
l_prev = batch_norm_layer(l_prev)
l_prev = ReshapeLayer(l_prev, (-1, n_features, sequence_length))
l_prev = DimshuffleLayer(l_prev, (0, 2, 1))
# 2D Convolutional layers
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features))
l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1))
# Add the convolutional filters
for n_filter, filter_size, pool_size in zip(n_filters, filter_sizes,
pool_sizes):
self.log += "\nAdding 2D conv layer: %d x %d" % (n_filter,
filter_size)
l_prev = Conv2DLayer(l_prev,
num_filters=n_filter,
filter_size=(filter_size, 1),
nonlinearity=self.transf,
pad=filter_size // 2)
if batch_norm:
l_prev = batch_norm_layer(l_prev)
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = Pool2DLayer(l_prev, pool_size=(pool_size, 1))
self.log += "\nAdding dropout layer: %.2f" % conv_dropout
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout)
print("Conv out shape", get_output_shape(l_prev))
# Global pooling layer
l_prev = GlobalPoolLayer(l_prev,
pool_function=T.mean,
name='Global Mean Pool')
print("GlobalPoolLayer out shape", get_output_shape(l_prev))
# Concatenate stats
l_prev = ConcatLayer((l_prev, stats_layer), axis=1)
for n_hid in n_hidden:
self.log += "\nAdding dense layer with %d units" % n_hid
print("Dense input shape", get_output_shape(l_prev))
l_prev = DenseLayer(l_prev, n_hid, init.GlorotNormal(),
init.Normal(1e-3), self.transf)
if batch_norm:
l_prev = batch_norm_layer(l_prev)
if dropout:
self.log += "\nAdding dense dropout with probability: %.2f" % dense_dropout
l_prev = DropoutLayer(l_prev, p=dense_dropout)
if batch_norm:
self.log += "\nUsing batch normalization"
self.model = DenseLayer(l_prev, num_units=n_out, nonlinearity=out_func)
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.matrix('t')
def __init__(self, n_in, n_filters, filter_sizes, n_out, pool_sizes=None, n_hidden=(), ccf=False,
rcl=(), rcl_dropout=0.0, trans_func=rectify, out_func=softmax, dropout_probability=0.0,
batch_norm=False, stats=0):
super(RCNN, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
# Define model using lasagne framework
dropout = True if not dropout_probability == 0.0 else False
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length+stats, n_features), name='Input')
l_prev = self.l_in
print("Input shape: ", get_output_shape(l_prev))
# Separate into raw values and statistics
if stats > 0:
stats_layer = SliceLayer(l_prev, indices=slice(sequence_length, None), axis=1)
stats_layer = ReshapeLayer(stats_layer, (-1, stats*n_features))
l_prev = SliceLayer(l_prev, indices=slice(0, sequence_length), axis=1)
# Input noise
sigma = 0.05
self.log += "\nGaussian input noise: %02f" % sigma
l_prev = GaussianNoiseLayer(l_prev, sigma=sigma)
if ccf:
self.log += "\nAdding cross-channel feature layer"
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features), name='Reshape')
l_prev = Conv2DLayer(l_prev,
num_filters=4*n_features,
filter_size=(1, n_features),
nonlinearity=None,
b=None,
name='Conv2D')
l_prev = BatchNormalizeLayer(l_prev, normalize=batch_norm, nonlinearity=self.transf)
n_features *= 4
l_prev = ReshapeLayer(l_prev, (-1, n_features, sequence_length), name='Reshape')
l_prev = DimshuffleLayer(l_prev, (0, 2, 1), name='Dimshuffle')
# Convolutional layers
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features), name='Reshape')
l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1), name='Dimshuffle')
for n_filter, filter_size, pool_size in zip(n_filters, filter_sizes, pool_sizes):
self.log += "\nAdding 2D conv layer: %d x %d" % (n_filter, filter_size)
l_prev = Conv2DLayer(l_prev,
num_filters=n_filter,
filter_size=(filter_size, 1),
pad="same",
nonlinearity=None,
b=None,
name='Conv2D')
l_prev = BatchNormalizeLayer(l_prev, normalize=batch_norm, nonlinearity=self.transf)
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = Pool2DLayer(l_prev, pool_size=(pool_size, 1), name='Max Pool')
self.log += "\nAdding dropout layer: %.2f" % rcl_dropout
l_prev = TiedDropoutLayer(l_prev, p=rcl_dropout, name='Dropout')
self.log += "\nConv out shape: %s" % str(get_output_shape(l_prev))
# Recurrent Convolutional layers
filter_size = filter_sizes[0]
for t in rcl:
self.log += "\nAdding recurrent conv layer: t: %d, filter size: %s" % (t, filter_size)
l_prev = RecurrentConvLayer(l_prev,
t=t,
filter_size=filter_size,
nonlinearity=self.transf,
normalize=batch_norm)
self.log += "\nAdding max pool layer: 2"
l_prev = Pool2DLayer(l_prev, pool_size=(2, 1), name='Max Pool')
print("RCL out shape", get_output_shape(l_prev))
l_prev = GlobalPoolLayer(l_prev, name='Global Max Pool')
print("GlobalPoolLayer out shape", get_output_shape(l_prev))
# Append statistics
if stats > 0:
l_prev = ConcatLayer((l_prev, stats_layer), axis=1)
for n_hid in n_hidden:
self.log += "\nAdding dense layer with %d units" % n_hid
print("Dense input shape", get_output_shape(l_prev))
l_prev = DenseLayer(l_prev, num_units=n_hid, nonlinearity=None, b=None, name='Dense')
l_prev = BatchNormalizeLayer(l_prev, normalize=batch_norm, nonlinearity=self.transf)
if dropout:
self.log += "\nAdding output dropout with probability %.2f" % dropout_probability
l_prev = DropoutLayer(l_prev, p=dropout_probability, name='Dropout')
if batch_norm:
self.log += "\nUsing batch normalization"
self.model = DenseLayer(l_prev, num_units=n_out, nonlinearity=out_func, name='Output')
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.matrix('t')
def __init__(self, n_in, n_filters, n_out, pool_sizes=None, n_hidden=(), ccf=False,
conv_dropout=0.0, trans_func=rectify, out_func=softmax, dropout=0.0,
batch_norm=False, stats=2):
super(ResNet, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length+stats, n_features), name='Input')
l_prev = self.l_in
print("Input shape: ", get_output_shape(l_prev))
# Separate into raw values and statistics
if stats > 0:
stats_layer = SliceLayer(l_prev, indices=slice(sequence_length, None), axis=1)
stats_layer = ReshapeLayer(stats_layer, (-1, stats*n_features))
l_prev = SliceLayer(l_prev, indices=slice(0, sequence_length), axis=1)
if ccf:
self.log += "\nAdding cross-channel feature layer"
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features), name='Reshape')
l_prev = Conv2DLayer(l_prev,
num_filters=4*n_features,
filter_size=(1, n_features),
nonlinearity=None,
b=None,
name='Conv2D')
l_prev = BatchNormalizeLayer(l_prev, normalize=batch_norm, nonlinearity=self.transf)
n_features *= 4
l_prev = ReshapeLayer(l_prev, (-1, n_features, sequence_length), name='Reshape')
l_prev = DimshuffleLayer(l_prev, (0, 2, 1), name='Dimshuffle')
# Reshape and order dimensionality
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features), name='Reshape')
l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1), name='Dimshuffle')
# Convolutional layers
n_filter, filter_size, pool_size = (16, 3, 2)
self.log += "\nAdding 2D conv layer: %d x %d" % (n_filter, filter_size)
l_prev = Conv2DLayer(l_prev,
num_filters=n_filter,
filter_size=(filter_size, 1),
stride=(1, 1),
pad="same",
nonlinearity=None,
b=None,
name='Conv2D')
l_prev = BatchNormalizeLayer(l_prev, normalize=batch_norm, nonlinearity=self.transf)
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = Pool2DLayer(l_prev, pool_size=(pool_size, 1), name='Max Pool')
if conv_dropout > 0.0:
self.log += "\nAdding dropout layer: %.2f" % conv_dropout
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout, name='Tied Dropout')
print("Conv out shape", get_output_shape(l_prev))
# Residual modules
for n_filter, pool_size in zip(n_filters, pool_sizes):
self.log += "\nAdding residual module: %d" % n_filter
l_prev = ResidualModule(l_prev,
num_filters=n_filter,
nonlinearity=self.transf,
normalize=batch_norm,
stride=(1, 1),
conv_dropout=conv_dropout)
if pool_size > 2:
self.log += "\nAdding max pool layer: %d" % pool_size
l_prev = Pool2DLayer(l_prev, pool_size=(pool_size, 1), name='Max Pool')
print("Residual out shape", get_output_shape(l_prev))
self.log += "\nGlobal Pooling: mean"
l_prev = GlobalPoolLayer(l_prev, pool_function=T.mean, name='Global pooling layer: mean')
print("GlobalPoolLayer out shape", get_output_shape(l_prev))
# Append statistics
if stats > 0:
l_prev = ConcatLayer((l_prev, stats_layer), axis=1)
for n_hid in n_hidden:
self.log += "\nAdding dense layer with %d units" % n_hid
print("Dense input shape", get_output_shape(l_prev))
l_prev = DenseLayer(l_prev, num_units=n_hid, nonlinearity=None, b=None, name='Dense')
l_prev = BatchNormalizeLayer(l_prev, normalize=batch_norm, nonlinearity=self.transf)
if dropout > 0.0:
self.log += "\nAdding output dropout with probability %.2f" % dropout
l_prev = DropoutLayer(l_prev, p=dropout, name='Dropout')
if batch_norm:
self.log += "\nUsing batch normalization"
self.model = DenseLayer(l_prev, num_units=n_out, nonlinearity=out_func, name='Output')
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.matrix('t')
def __init__(self,
n_in,
inception_layers,
n_out,
pool_sizes=None,
n_hidden=512,
trans_func=rectify,
out_func=softmax,
output_dropout=0.0,
stats=0,
batch_norm=False,
inception_dropout=0.0):
super(Incep, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length + stats,
n_features),
name='Input')
l_prev = self.l_in
# Separate into raw values and statistics
if stats > 0:
stats_layer = SliceLayer(l_prev,
indices=slice(sequence_length, None),
axis=1)
stats_layer = ReshapeLayer(stats_layer, (-1, stats * n_features))
l_prev = SliceLayer(l_prev,
indices=slice(0, sequence_length),
axis=1)
# 2D Convolutional layers--------------
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features),
name='Reshape')
l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1), name='Dimshuffle')
# Init with a Conv
self.log += "\nAdding 2D conv layer: %d x %d" % (32, 3)
l_prev = Conv2DLayer(l_prev,
num_filters=32,
filter_size=(3, 1),
pad='same',
nonlinearity=None,
b=None,
name='Input Conv2D')
l_prev = BatchNormalizeLayer(l_prev,
normalize=batch_norm,
nonlinearity=self.transf)
l_prev = Pool2DLayer(l_prev, pool_size=(2, 1), name='Input pool')
l_prev = TiedDropoutLayer(l_prev,
p=inception_dropout,
name='Input conv dropout')
# Inception layers
for inception_layer, pool_size in zip(inception_layers, pool_sizes):
num_1x1, num_2x1_proj, reduce_3x1, num_3x1, reduce_5x1, num_5x1 = inception_layer
self.log += "\nAdding inception layer: %s" % str(inception_layer)
l_prev = inception_module(l_prev,
num_1x1,
num_2x1_proj,
reduce_3x1,
num_3x1,
reduce_5x1,
num_5x1,
batch_norm=batch_norm)
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = Pool2DLayer(l_prev,
pool_size=(pool_size, 1),
name='Inception pool')
self.log += "\nAdding dropout layer: %.2f" % inception_dropout
l_prev = TiedDropoutLayer(l_prev,
p=inception_dropout,
name='Inception dropout')
print("Inception out shape", l_prev.output_shape)
# Global pooling layer
self.log += "\nGlobal Pooling: average"
l_prev = GlobalPoolLayer(l_prev,
pool_function=T.mean,
name='Global average pool')
# Append statistics
if stats > 0:
l_prev = ConcatLayer((l_prev, stats_layer), axis=1)
if n_hidden:
self.log += "\nAdding dense layer with %d units" % n_hidden
print("Dense input shape", l_prev.output_shape)
l_prev = DenseLayer(l_prev,
num_units=n_hidden,
nonlinearity=self.transf,
name='Dense')
if batch_norm:
l_prev = BatchNormLayer(l_prev)
if output_dropout:
self.log += "\nAdding output dropout with probability %.2f" % output_dropout
l_prev = DropoutLayer(l_prev,
p=output_dropout,
name='Dense dropout')
self.model = DenseLayer(l_prev,
num_units=n_out,
nonlinearity=out_func,
name='Output')
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.matrix('t')
def __init__(self,
n_in,
n_hidden,
n_out,
n_filters,
filter_sizes,
pool_sizes,
stats=2,
grad_clip=GRAD_CLIP,
peepholes=False,
trans_func=rectify,
out_func=softmax,
factor=1,
conv_dropout=0.0,
rnn_in_dropout=0.0,
rnn_hid_dropout=0.0,
output_dropout=0.0,
conv_stride=1):
super(tconvRNN, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length + stats,
n_features))
l_prev = self.l_in
print("Input shape", get_output_shape(l_prev))
# Separate into raw values and statistics
if stats > 0:
stats_layer = SliceLayer(l_prev,
indices=slice(sequence_length, None),
axis=1)
stats_layer = ReshapeLayer(stats_layer,
(-1, stats * n_features, 1))
print('Stats layer shape', stats_layer.output_shape)
l_prev = SliceLayer(l_prev,
indices=slice(0, sequence_length),
axis=1)
# Reshape
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features))
# l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1))
# Add input noise
# self.log += "\nAdding noise layer: 0.1"
# l_prev = GaussianNoiseLayer(l_prev, sigma=0.1)
# Adding convolutional layers
print("Conv input shape", get_output_shape(l_prev))
for n_filter, filter_size, pool_size in zip(n_filters, filter_sizes,
pool_sizes):
self.log += "\nAdding 2D conv layer: %d x %d" % (n_filter,
filter_size)
l_prev = Conv2DLayer(l_prev,
num_filters=n_filter,
filter_size=(filter_size, 1),
pad=0,
W=init.GlorotNormal('relu'),
b=init.Normal(1e-3),
nonlinearity=self.transf,
stride=(conv_stride, 1))
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = MaxPool2DLayer(l_prev, pool_size=(pool_size, 1))
if conv_dropout:
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout)
self.log += "\nAdding dropout: %.2f" % conv_dropout
print("Conv out shape", get_output_shape(l_prev))
# Pool filters
# self.log += "\nAdding global pooling"
# l_prev = GlobalPoolLayer(l_prev, pool_function=T.mean)
# Get output shapes from convnet
s1, s2, s3, s4 = l_prev.output_shape
# Reshape for LSTM
l_prev = DimshuffleLayer(l_prev, (0, 2, 1, 3))
l_prev = ReshapeLayer(l_prev, (-1, s3, s2 * s4))
# Concat with statistics
if stats > 0:
self.log += "\nConcatenating stats"
stats_layer = Upscale1DLayer(stats_layer, scale_factor=s3)
stats_layer = DimshuffleLayer(stats_layer, (0, 2, 1))
print("Upscaled stats layer, ", stats_layer.output_shape)
l_prev = ConcatLayer((l_prev, stats_layer), axis=2)
# Add LSTM layers
print("LSTM input shape", get_output_shape(l_prev))
for n_hid in n_hidden:
self.log += "\nAdding LSTM layer with: %d units" % n_hid
self.log += "\nLSTM in dropout: %.2f" % rnn_in_dropout
self.log += "\nLSTM hid dropout: %.2f" % rnn_hid_dropout
l_prev = LSTMLayer(
l_prev,
num_units=n_hid,
grad_clipping=grad_clip,
peepholes=peepholes,
ingate=Gate(W_in=lasagne.init.HeUniform(),
W_hid=lasagne.init.HeUniform()),
forgetgate=Gate(b=lasagne.init.Constant(CONST_FORGET_B)),
nonlinearity=lasagne.nonlinearities.tanh)
print("LSTM output shape", get_output_shape(l_prev))
# Slice last output from RNN
l_prev = SliceLayer(l_prev, indices=-1, axis=1)
print("LSTM slice shape", get_output_shape(l_prev))
# Optional extra dense layer
l_prev = DenseLayer(l_prev, num_units=30, nonlinearity=trans_func)
self.log += "\nAdding dense layer with %d units" % 512
if output_dropout:
self.log += "\nAdding output dropout with probability: %.2f" % output_dropout
l_prev = DropoutLayer(l_prev, p=output_dropout)
# Output
self.model = DenseLayer(l_prev, num_units=n_out, nonlinearity=out_func)
print("Output shape", get_output_shape(self.model))
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.matrix('t')
def __init__(self,
n_in,
n_hidden,
n_out,
n_filters,
filter_sizes,
pool_sizes,
stats=2,
conv_stride=1,
grad_clip=GRAD_CLIP,
peepholes=False,
trans_func=rectify,
out_func=softmax,
factor=1,
conv_dropout=0.0,
output_dropout=0.0):
super(wconvRNN, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length, n_features))
l_prev = self.l_in
print("Input shape", get_output_shape(l_prev))
# Reshape
# batch_size *= factor
sequence_length /= factor
l_prev = ReshapeLayer(
l_prev, (-1, 1, int(sequence_length + stats), n_features))
# Separate into raw values and statistics
if stats > 0:
stats_layer = SliceLayer(l_prev,
indices=slice(sequence_length, None),
axis=2)
stats_layer = ReshapeLayer(stats_layer,
(-1, factor, stats * n_features))
print('Stats layer shape', stats_layer.output_shape)
l_prev = SliceLayer(l_prev,
indices=slice(0, sequence_length),
axis=2)
# Shuffle dimensions so features becomes channels
l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1))
# Add input noise
# self.log += "\nAdding noise layer: 0.05"
# l_prev = GaussianNoiseLayer(l_prev, sigma=0.05)
# Adding convolutional layers
print("Conv input shape", get_output_shape(l_prev))
for n_filter, filter_size, pool_size in zip(n_filters, filter_sizes,
pool_sizes):
self.log += "\nAdding 2D conv layer: %d x %d" % (n_filter,
filter_size)
l_prev = Conv2DLayer(l_prev,
num_filters=n_filter,
filter_size=(filter_size, 1),
pad='same',
W=init.GlorotNormal('relu'),
b=init.Normal(1e-3),
nonlinearity=self.transf,
stride=(conv_stride, 1))
self.log += "\nConv stride: %d" % conv_stride
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = MaxPool2DLayer(l_prev, pool_size=(pool_size, 1))
if conv_dropout:
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout)
self.log += "\nAdding dropout: %.2f" % conv_dropout
print("Conv out shape", get_output_shape(l_prev))
# Pool filters
# self.log += "\nAdding global pooling"
# l_prev = GlobalPoolLayer(l_prev, pool_function=T.mean)
# Get output shapes from convnet
s1, s2, s3, s4 = l_prev.output_shape
# Reshape for LSTM
l_prev = ReshapeLayer(l_prev, (-1, factor, s2 * s3 * s4))
print(l_prev.output_shape)
# Concat with statistics
if stats > 0:
self.log += "\nConcatenating stats"
l_prev = ConcatLayer((l_prev, stats_layer), axis=2)
# Add LSTM layers
print("LSTM input shape", get_output_shape(l_prev))
for n_hid in n_hidden:
self.log += "\nAdding LSTM layer with: %d units" % n_hid
l_prev = LSTMLayer(
l_prev,
num_units=n_hid,
grad_clipping=grad_clip,
peepholes=peepholes,
ingate=Gate(W_in=lasagne.init.GlorotNormal(),
W_hid=lasagne.init.Orthogonal()),
forgetgate=Gate(b=lasagne.init.Constant(CONST_FORGET_B)),
nonlinearity=lasagne.nonlinearities.tanh)
print("LSTM output shape", get_output_shape(l_prev))
# Reshape to process each timestep individually
l_prev = ReshapeLayer(l_prev, (-1, n_hidden[-1]))
# l_prev = DenseLayer(l_prev, num_units=512, nonlinearity=trans_func)
# self.log += "\nAdding dense layer with %d units" % 512
if output_dropout:
self.log += "\nAdding output dropout with probability: %.2f" % output_dropout
l_prev = DropoutLayer(l_prev, p=output_dropout)
# Output
l_prev = DenseLayer(l_prev, num_units=n_out, nonlinearity=out_func)
self.model = ReshapeLayer(l_prev, (-1, factor, n_out))
print("Output shape", get_output_shape(self.model))
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.tensor3('t')
def __init__(self,
n_in,
n_filters,
filter_sizes,
n_out,
pool_sizes=None,
n_hidden=(512),
trans_func=rectify,
out_func=softmax,
dropout=0.0,
input_noise=0.0,
batch_norm=False,
conv_dropout=0.0,
slicers=()):
super(CNN, self).__init__(n_in, n_hidden, n_out, trans_func)
self.outf = out_func
self.log = ""
if pool_sizes is None:
pool_sizes = [0] * len(n_filters)
# Overwrite input layer
sequence_length, n_features = n_in
self.l_in = InputLayer(shape=(None, sequence_length, n_features))
l_prev = self.l_in
# Apply input noise
# l_prev = GaussianNoiseLayer(l_prev, sigma=input_noise)
# 2D Convolutional layers
l_prev = ReshapeLayer(l_prev, (-1, 1, sequence_length, n_features))
l_prev = DimshuffleLayer(l_prev, (0, 3, 2, 1))
# Add the convolutional filters
for n_filter, filter_size, pool_size in zip(n_filters, filter_sizes,
pool_sizes):
self.log += "\nAdding 2D conv layer: %d x %d" % (n_filter,
filter_size)
l_prev = Conv2DLayer(l_prev,
num_filters=n_filter,
filter_size=(filter_size, 1),
nonlinearity=self.transf,
pad=filter_size // 2)
if batch_norm:
l_prev = BatchNormLayer(l_prev)
if pool_size > 1:
self.log += "\nAdding max pooling layer: %d" % pool_size
l_prev = Pool2DLayer(l_prev, pool_size=(pool_size, 1))
self.log += "\nAdding dropout layer: %.2f" % conv_dropout
l_prev = TiedDropoutLayer(l_prev, p=conv_dropout)
print("Conv out shape", get_output_shape(l_prev))
# Global pooling layer
l_prev = GlobalPoolLayer(l_prev,
pool_function=T.mean,
name='Global Mean Pool')
print("GlobalPoolLayer out shape", get_output_shape(l_prev))
for n_hid in n_hidden:
self.log += "\nAdding dense layer with %d units" % n_hid
print("Dense input shape", get_output_shape(l_prev))
l_prev = DenseLayer(l_prev, n_hid, init.GlorotNormal(),
init.Normal(1e-3), self.transf)
if batch_norm:
l_prev = BatchNormLayer(l_prev)
if dropout:
self.log += "\nAdding dense dropout with probability: %.2f" % dropout
l_prev = DropoutLayer(l_prev, p=dropout)
if batch_norm:
self.log += "\nUsing batch normalization"
self.model = DenseLayer(l_prev, num_units=n_out, nonlinearity=out_func)
self.model_params = get_all_params(self.model)
self.sym_x = T.tensor3('x')
self.sym_t = T.matrix('t')