def output_path(net, incoming_layer, n_classes, filter_size, out_nonlin):
'''
Build the output path (including last conv layer to have n_classes
feature maps). Dimshuffle layers to fit with softmax implementation
Parameters
----------
Same as above
incoming_layer : string, name of last layer from bottleneck layers
'''
#Final convolution (n_classes feature maps) with filter_size = 1
net['final_conv'] = ConvLayer(net[incoming_layer], n_classes, 1)
#DimshuffleLayer and all this stuff is necessary to fit with softmax
#implementation. In training, we specify layer = ['probs'] to have the
#right layer but the 2 last reshape layers are necessary only to visualize
#data.
net['final_dimshuffle'] = DimshuffleLayer(net['final_conv'], (0, 2, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:2]), laySize[2]))
net['probs'] = NonlinearityLayer(net['final_reshape'],
nonlinearity=out_nonlin)
net['probs_reshape'] = ReshapeLayer(net['probs'],
(laySize[0], laySize[1], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'], (0, 2, 1))
return net
def SoftmaxLayer(inputs, n_classes):
"""
Performs 1x1 convolution followed by softmax nonlinearity
The output will have the shape (batch_size * n_rows * n_cols, n_classes)
"""
l = Conv2DLayer(inputs,
n_classes,
filter_size=1,
nonlinearity=linear,
W=HeUniform(gain='relu'),
pad='same',
flip_filters=False,
stride=1)
# We perform the softmax nonlinearity in 2 steps :
# 1. Reshape from (batch_size, n_classes, n_rows, n_cols) to (batch_size * n_rows * n_cols, n_classes)
# 2. Apply softmax
l = DimshuffleLayer(l, (0, 2, 3, 1))
batch_size, n_rows, n_cols, _ = get_output(l).shape
l = ReshapeLayer(l, (batch_size * n_rows * n_cols, n_classes))
l = NonlinearityLayer(l, softmax)
l = ReshapeLayer(l, (batch_size, n_rows, n_cols, n_classes))
l = DimshuffleLayer(l, (0, 3, 1, 2))
l = ReshapeLayer(l, (batch_size, n_classes, n_rows, n_cols))
return l
def TransitionalNormalizeLayer(inputs, n_directions):
"""
Performs 1x1 convolution followed by softmax nonlinearity.
The output will have the shape (batch_size * n_rows * n_cols, n_classes)
"""
l = Conv2DLayer(inputs,
n_directions**2,
filter_size=1,
nonlinearity=linear,
W=HeUniform(gain='relu'),
pad='same',
flip_filters=False,
stride=1)
# We perform the softmax nonlinearity in 2 steps :
# 1. Reshape from (batch_size, n_classes, n_rows, n_cols) to (batch_size * n_rows * n_cols, n_classes)
# 2. Apply softmax
batch_size, n_channels, n_rows, n_cols = get_output(l).shape
l = ReshapeLayer(l,
(batch_size, n_directions, n_directions, n_rows, n_cols))
l = DimshuffleLayer(l, (0, 1, 3, 4, 2))
l = ReshapeLayer(
l, (batch_size * n_directions * n_rows * n_cols, n_directions))
l = NormalizeLayer(l)
l = ReshapeLayer(l,
(batch_size, n_directions, n_rows, n_cols, n_directions))
l = DimshuffleLayer(l, (0, 1, 4, 2, 3))
l = ReshapeLayer(l, (batch_size, n_channels, n_rows, n_cols))
return l
def construct_unet_3D(channels=1, no_f_base=8, f_size=3, branches=[2,2,2,2],dropout=0.2,bs=None,
class_nums=2, pad="same",nonlinearity=lasagne.nonlinearities.rectify,
input_dim=[None,None,None],useups=False):
net= InputLayer((bs, channels, input_dim[0], input_dim[1], input_dim[2]))
# Moving downwards the U-shape:
horizontal_pass=[]
for i in xrange(len(branches)):
net = conv_pool_down_3D(net,no_f_base*2**(i),f_size,conv_depth=branches[i],
pad=pad,nonlinearity=nonlinearity,dropout=dropout)
print "Down conv: ",net.output_shape
horizontal_pass.append(net)
net = MaxPool3DDNNLayer(net,pool_size=(2,2,2),stride=(2,2,2))
print "Down Pool: ",net.output_shape
# Bottleneck
net = Conv3DDNNLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu'))
print "Bottleneck conv: ",net.output_shape
net = Conv3DDNNLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu'))
print "Bottleneck conv: ",net.output_shape
#net = Conv3DDNNTransposeLayer(net, no_f_base*2**(len(branches)-1), 2, (2, 2, 2))
if not useups:
net = TransposedConv3DLayer(net,no_f_base*2**(len(branches)-1),2,(2,2,2))
else:
net = upscale_plus_conv_3D(net,no_f_base*2**(len(branches)-1),f_size,pad,nonlinearity)
print "Bottleneck up: ",net.output_shape
# Moving upwards the U-shape:
for i in xrange(len(branches)):
print "Pass before concat: ",horizontal_pass[-(i+1)].output_shape
print "net before concat: ",net.output_shape
if not useups:
net = ConcatLayer([net,horizontal_pass[-(i+1)]],cropping=(None,None,"center","center","center"))
else:
net = ConcatLayer([net,horizontal_pass[-(i+1)]],cropping=(None,None,"center","center","center"))
print "Shape after concat: ",net.output_shape
if i==len(branches)-1:
net = conv_pool_up_3D(net,bs,no_f_base*2**(len(branches)-1-i),f_size,
pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=True,useups=False)
else:
net = conv_pool_up_3D(net,bs,no_f_base*2**(len(branches)-1-i),f_size,
pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=False,useups=False)
print "Conv up: ",net.output_shape
# Class layer: Work around standard softmax bc. it doesn't work with tensor4/3.
# Hence, we reshape and feed it to an external Nonlinearity layer.
# net["class_ns"] is the output in image-related shape.
imageout = net = Conv3DDNNLayer(net, class_nums, 1, nonlinearity=linear,W=lasagne.init.HeNormal(gain='relu'))
print "imageout shape: ",net.output_shape
net = DimshuffleLayer(net, (1, 0, 2, 3, 4))
print "After shuffle shape: ",net.output_shape
net = ReshapeLayer(net, (class_nums, -1))
print "Reshape shape: ",net.output_shape
net = DimshuffleLayer(net, (1, 0))
print "Dimshuffle shape: ",net.output_shape
# Flattened output to be able to feed it to lasagne.objectives.categorical_crossentropy.
net = NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax)
#imout = NonlinearityLayer(imageout,nonlinearity=lasagne.nonlinearities.softmax)
return net,imageout
del net, imageout,imout
def get_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=L.nonlinearities.leaky_rectify,
input_dim=(128, 128), base_n_filters=128):
net = OrderedDict()
net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1]))
net['contr_1_1'] = batch_norm(ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad))
net['contr_1_2'] = batch_norm(ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad))
net['pool1'] = Pool2DLayer(net['contr_1_2'], 2)
net['contr_2_1'] = batch_norm(ConvLayer(net['pool1'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad))
net['contr_2_2'] = batch_norm(ConvLayer(net['contr_2_1'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad))
net['pool2'] = Pool2DLayer(net['contr_2_2'], 2)
net['contr_3_1'] = batch_norm(ConvLayer(net['pool2'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad))
net['contr_3_2'] = batch_norm(ConvLayer(net['contr_3_1'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad))
net['pool3'] = Pool2DLayer(net['contr_3_2'], 2)
net['contr_4_1'] = batch_norm(ConvLayer(net['pool3'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad))
net['contr_4_2'] = batch_norm(ConvLayer(net['contr_4_1'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad))
l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2)
# the paper does not really describe where and how dropout is added. Feel free to try more options
l = DropoutLayer(l, p=0.4)
net['encode_1'] = batch_norm(ConvLayer(l, base_n_filters * 16, 3, nonlinearity=nonlinearity, pad=pad))
net['encode_2'] = batch_norm(ConvLayer(net['encode_1'], base_n_filters * 16, 3, nonlinearity=nonlinearity, pad=pad))
net['deconv1'] = Upscale2DLayer(net['encode_2'], 2)
net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center"))
net['expand_1_1'] = batch_norm(ConvLayer(net['concat1'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad))
net['expand_1_2'] = batch_norm(ConvLayer(net['expand_1_1'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad))
net['deconv2'] = Upscale2DLayer(net['expand_1_2'], 2)
net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center"))
net['expand_2_1'] = batch_norm(ConvLayer(net['concat2'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad))
net['expand_2_2'] = batch_norm(ConvLayer(net['expand_2_1'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad))
net['deconv3'] = Upscale2DLayer(net['expand_2_2'], 2)
net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center"))
net['expand_3_1'] = batch_norm(ConvLayer(net['concat3'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad))
net['expand_3_2'] = batch_norm(ConvLayer(net['expand_3_1'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad))
net['deconv4'] = Upscale2DLayer(net['expand_3_2'], 2)
net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center"))
net['expand_4_1'] = batch_norm(ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad))
net['expand_4_2'] = batch_norm(ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad))
net['conv_5'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) # (bs, nrClasses, x, y)
net['dimshuffle'] = DimshuffleLayer(net['conv_5'], (1, 0, 2, 3)) # (nrClasses, bs, x, y)
net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) # (nrClasses, bs*x*y)
net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) # (bs*x*y, nrClasses)
#Watch out: here is another nonlinearity -> do not use layers before this layer!
net['output_flat'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=L.nonlinearities.sigmoid) # (bs*x*y, nrClasses)
img_shape = net["conv_5"].output_shape
net['output'] = ReshapeLayer(net['output_flat'], (-1, img_shape[2], img_shape[3], img_shape[1])) # (bs, x, y, nrClasses)
return net
def SoftmaxLayer(inputs, n_classes):
l = Conv2DLayer(inputs, n_classes, filter_size=1, nonlinearity=linear, W=HeUniform(gain='relu'), pad='same',
flip_filters=False, stride=1)
l = DimshuffleLayer(l,(1,0,2,3))
l = ReshapeLayer(l, (n_classes,-1))
l = DimshuffleLayer(l, (1,0))
l = NonlinearityLayer(l, nonlinearity=softmax)
return l
def get_model(input_var, target_var, multiply_var):
# input layer with unspecified batch size
layer_input = InputLayer(shape=(None, 30, 80, 80), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var)
layer_0 = DimshuffleLayer(layer_input, (0, 'x', 1, 2, 3))
# Z-score?
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer_1 = batch_norm(Conv3DDNNLayer(incoming=layer_0, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_2 = batch_norm(Conv3DDNNLayer(incoming=layer_1, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_3 = MaxPool3DDNNLayer(layer_2, pool_size=(2, 2, 2), stride=(2, 2, 2), pad=(1, 1, 1))
layer_4 = DropoutLayer(layer_3, p=0.25)
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer_5 = batch_norm(Conv3DDNNLayer(incoming=layer_4, num_filters=32, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_6 = batch_norm(Conv3DDNNLayer(incoming=layer_5, num_filters=32, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_7 = MaxPool3DDNNLayer(layer_6, pool_size=(2, 2, 2), stride=(2, 2, 2), pad=(1, 1, 1))
layer_8 = DropoutLayer(layer_7, p=0.25)
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer_5 = batch_norm(Conv3DDNNLayer(incoming=layer_8, num_filters=64, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_6 = batch_norm(Conv3DDNNLayer(incoming=layer_5, num_filters=64, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_7 = batch_norm(Conv3DDNNLayer(incoming=layer_6, num_filters=64, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify))
layer_8 = MaxPool3DDNNLayer(layer_7, pool_size=(2, 2, 2), stride=(2, 2, 2), pad=(1, 1, 1))
layer_9 = DropoutLayer(layer_8, p=0.25)
# LSTM
layer = DimshuffleLayer(layer_9, (0,2,1,3,4))
# layer_prediction = LSTMLayer(layer, num_units=2, only_return_final=True, learn_init=True, cell=Gate(linear))
layer = LSTMLayer(layer, num_units=2, only_return_final=True, learn_init=True)
layer_prediction = DenseLayer(layer, 2, nonlinearity=linear)
# Output Layer
# layer_hidden = DenseLayer(layer_flatten, 500, nonlinearity=linear)
# layer_prediction = DenseLayer(layer_hidden, 2, nonlinearity=linear)
# Loss
prediction = get_output(layer_prediction) / multiply_var**2
loss = T.abs_(prediction - target_var)
loss = loss.mean()
#Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum
params = get_all_params(layer_prediction, trainable=True)
# Create a loss expression for validation/testing. The crucial difference
# here is that we do a deterministic forward pass through the network, disabling dropout layers.
test_prediction = get_output(layer_prediction, deterministic=True) / multiply_var**2
test_loss = T.abs_(test_prediction - target_var)
test_loss = test_loss.mean()
# crps estimate
crps = T.abs_(test_prediction - target_var).mean()/600
return test_prediction, crps, loss, params
def cnn_fn(self):
l_in = InputLayer((None, self.max_length, self.vocab_size))
l_in_T = DimshuffleLayer(l_in, (0, 2, 1))
l_causal_conv = DilatedConv1DLayer(
l_in_T,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=tanh)
l_gate = DilatedConv1DLayer(l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_final = DimshuffleLayer(l_skip_sum, (0, 2, 1))
return l_final
def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(128, 128), base_n_filters=64, do_dropout=False):
net = OrderedDict()
net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1]))
net['contr_1_1'] = ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)
net['contr_1_2'] = ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)
net['pool1'] = Pool2DLayer(net['contr_1_2'], 2)
net['contr_2_1'] = ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad)
net['contr_2_2'] = ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad)
net['pool2'] = Pool2DLayer(net['contr_2_2'], 2)
net['contr_3_1'] = ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad)
net['contr_3_2'] = ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad)
net['pool3'] = Pool2DLayer(net['contr_3_2'], 2)
net['contr_4_1'] = ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad)
net['contr_4_2'] = ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad)
l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2)
# the paper does not really describe where and how dropout is added. Feel free to try more options
if do_dropout:
l = DropoutLayer(l, p=0.4)
net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad)
net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad)
net['deconv1'] = Upscale2DLayer(net['encode_2'], 2)
net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center"))
net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad)
net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad)
net['deconv2'] = Upscale2DLayer(net['expand_1_2'], 2)
net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center"))
net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad)
net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad)
net['deconv3'] = Upscale2DLayer(net['expand_2_2'], 2)
net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center"))
net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad)
net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad)
net['deconv4'] = Upscale2DLayer(net['expand_3_2'], 2)
net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center"))
net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)
net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)
net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None)
net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3))
net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1))
net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0))
net['output_flattened'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax)
return net
def build_rnn_network(rnnmodel,X_sym,hid_init_sym):
net = {}
net['input0'] = InputLayer((batch_size, seq_len),X_sym)
net['input']=lasagne.layers.EmbeddingLayer(net['input0'],outputclass,units[0])#,W=lasagne.init.Uniform(inial_scale)
net['rnn0']=DimshuffleLayer(net['input'],(1,0,2)) #change to (time, batch_size,hidden_units)
if use_bn_embed:
net['rnn0']=BatchNorm_step_timefirst_Layer(net['rnn0'],axes=(0,1),epsilon=args.epsilon )
for l in range(1, num_layers+1):
net['hiddeninput%d'%l] = InputLayer((batch_size, units[l-1]),hid_init_sym[:,acc_units[l-1]:acc_units[l]])
net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (batch_size* seq_len, -1))
net['rnn%d'%(l-1)]=DenseLayer(net['rnn%d'%(l-1)],units[l-1],W=ini_W,b=lasagne.init.Constant(args.ini_b),nonlinearity=None) #W=Uniform(ini_rernn_in_to_hid), #
net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (seq_len, batch_size, -1))
if args.use_residual and l>args.residual_layers and (l-1)%args.residual_layers==0:# and l!=num_layers
if units[l - 1]!=units[l - 1 - args.residual_layers]:
net['leftbranch%d' % (l - 1)] = ReshapeLayer(net['sum%d'%(l-args.residual_layers)], (batch_size * seq_len, -1))
net['leftbranch%d' % (l - 1)] = DenseLayer(net['leftbranch%d' % (l - 1)], units[l - 1], W=ini_W, nonlinearity=None)
net['leftbranch%d' % (l - 1)] = ReshapeLayer(net['leftbranch%d' % (l - 1)], (seq_len, batch_size, -1))
net['leftbranch%d' % (l - 1)] = BatchNorm_step_timefirst_Layer(net['leftbranch%d' % (l - 1)], axes=(0, 1), epsilon=args.epsilon)
print('left branch')
else:
net['leftbranch%d' % (l - 1)] = net['sum%d'%(l-args.residual_layers)]
net['sum%d'%l]=ElemwiseSumLayer((net['rnn%d'%(l-1)],net['leftbranch%d' % (l - 1)]))
else:
net['sum%d'%l]=net['rnn%d'%(l-1)]
net['rnn%d'%l]=net['sum%d'%l]
if not args.use_bn_afterrnn:
net['rnn%d'%l]=BatchNorm_step_timefirst_Layer(net['rnn%d'%l],axes= (0,1),beta=lasagne.init.Constant(args.ini_b),epsilon=args.epsilon)
ini_hid_start=0
if act==tanh:
ini_hid_start=-1*U_bound
net['rnn%d'%l]=rnnmodel(net['rnn%d'%l],units[l-1],hid_init=net['hiddeninput%d'%l],W_hid_to_hid=Uniform(range=(ini_hid_start,U_bound)),nonlinearity=act,only_return_final=False, grad_clipping=args.gradclipvalue)
net['last_state%d'%l]=SliceLayer(net['rnn%d'%l],-1, axis=0)
if l==1:
net['hid_out']=net['last_state%d'%l]
else:
net['hid_out']=ConcatLayer([net['hid_out'], net['last_state%d'%l]],axis=1)
if use_dropout and l%droplayers==0 and not args.bn_drop:
net['rnn%d'%l]=lasagne.layers.DropoutLayer(net['rnn%d'%l], p=droprate, shared_axes=taxdrop)
if args.use_bn_afterrnn:
net['rnn%d'%l]=BatchNorm_step_timefirst_Layer(net['rnn%d'%l],axes= (0,1),epsilon=args.epsilon)
net['rnn%d'%num_layers]=DimshuffleLayer(net['rnn%d'%num_layers],(1,0,2))
net['reshape_rnn']=ReshapeLayer(net['rnn%d'%num_layers],(-1,units[num_layers-1]))
net['out']=DenseLayer(net['reshape_rnn'],outputclass,nonlinearity=softmax)#lasagne.init.HeNormal(gain='relu'))#,W=Uniform(inial_scale)
return net
def nn_fn(self):
l_in_z = InputLayer((None, self.z_dim))
l_in_x = InputLayer((None, self.max_length, self.emb_dim))
l_in_z_reshape = ReshapeLayer(l_in_z, (
[0],
[1],
1,
))
l_in_z_rep = TileLayer(l_in_z_reshape, (1, 1, self.max_length))
l_x_pre_pad = SliceLayer(PadLayer(l_in_x, [(1, 0), (0, 0)],
batch_ndim=1),
indices=slice(0, -1),
axis=1)
l_x_pre_pad = DimshuffleLayer(l_x_pre_pad, (0, 2, 1))
l_x_pre_pad_drop = DropoutLayer(l_x_pre_pad,
self.nn_word_drop,
shared_axes=(1, ))
l_concat = ConcatLayer((l_in_z_rep, l_x_pre_pad_drop), axis=1)
l_in_d = Conv1DLayer(l_concat,
num_filters=self.nn_channels_external,
pad='same',
filter_size=1,
nonlinearity=None)
for d in self.nn_dilations:
l_cnn1 = Conv1DLayer(l_in_d,
filter_size=1,
num_filters=self.nn_channels_internal)
l_dcnn = DilatedConv1DLayer(l_cnn1,
filter_size=self.nn_filter_size,
num_filters=self.nn_channels_internal,
dilation=d)
l_cnn2 = Conv1DLayer(l_dcnn,
filter_size=1,
num_filters=self.nn_channels_external)
l_in_d = ElemwiseSumLayer([l_in_d, l_cnn2])
l_final = Conv1DLayer(l_in_d,
filter_size=1,
num_filters=self.emb_dim,
nonlinearity=None)
l_out = DimshuffleLayer(l_final, (0, 2, 1))
return (l_in_z, l_in_x), l_out
def q_network(state):
input_state = InputLayer(input_var=state,
shape=(None, self.state_dimension[0],
self.state_dimension[1],
self.state_dimension[2]))
input_state = DimshuffleLayer(input_state, pattern=(0, 3, 1, 2))
conv = Conv2DLayer(input_state,
num_filters=32,
filter_size=(8, 8),
stride=(4, 4),
nonlinearity=rectify)
conv = Conv2DLayer(conv,
num_filters=64,
filter_size=(4, 4),
stride=(2, 2),
nonlinearity=rectify)
conv = Conv2DLayer(conv,
num_filters=64,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=rectify)
flatten = FlattenLayer(conv)
dense = DenseLayer(flatten, num_units=512, nonlinearity=rectify)
q_values = DenseLayer(dense,
num_units=self.action_dimension,
nonlinearity=linear)
return q_values
def smooth_convolution(prediction, n_classes):
from lasagne.layers import Conv1DLayer as ConvLayer
from lasagne.layers import DimshuffleLayer, ReshapeLayer
prediction = ReshapeLayer(prediction, (-1, 200, n_classes))
# channels first
prediction = DimshuffleLayer(prediction, (0, 2, 1))
input_size = lasagne.layers.get_output(prediction).shape
# reshape to put each channel in the batch dimensions, to filter each
# channel independently
prediction = ReshapeLayer(prediction,
(T.prod(input_size[0:2]), 1, input_size[2]))
trans_filter = np.tile(np.array([0, -1., 1.]).astype('float32'), (1, 1, 1))
convolved = ConvLayer(prediction,
num_filters=1,
filter_size=3,
stride=1,
b=None,
nonlinearity=None,
W=trans_filter,
pad='same')
# reshape back
convolved = ReshapeLayer(convolved, input_size)
return convolved
def _create_decoder(self, decoder_input, decoder_input_hid, max_len, n_features_context, resetgate=None, updategate=None, hiddengate=None):
#### Decoder (forwards)
# decoder_input_orig keeps the value of l_encoder, which is given to this method
decoder_input_orig = decoder_input
# decoder_input then becomes a repeated l_encoder
for _ in np.arange(max_len - 1):
decoder_input = ConcatLayer([decoder_input, decoder_input_orig])
print(f"- Decoder input before reshape. shape: {lasagne.layers.get_output_shape(decoder_input)}")
decoder_input = ReshapeLayer(decoder_input, (self.batch_size, n_features_context, max_len), name="reshape_enc_dec")
print(f"- Decoder input after reshape. shape: {lasagne.layers.get_output_shape(decoder_input)}")
decoder_input = DimshuffleLayer(decoder_input, (0, 2, 1), name="dimshuf_enc_dec")
# Use standard gates, if no weight sharing gates are supplied
if resetgate is None:
resetgate = lasagne.layers.Gate(W_cell=None)
if updategate is None:
updategate = lasagne.layers.Gate(W_cell=None)
if hiddengate is None:
hiddengate = lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh)
decoder_output = gated_layer(decoder_input, self.n_hidden,
grad_clipping=self.grad_clip,
only_return_final=False, backwards=False,
cell_init=self.init,
hid_init=decoder_input_hid,
gated_layer_type=self.gated_layer_type,
resetgate=resetgate, updategate=updategate, hidden_update=hiddengate,
name="dec")
if self.dropout > 0.0:
decoder_output = DropoutLayer(decoder_output, p=self.dropout, rescale=False)
l_decoder = decoder_output
return l_decoder
def get_model(input_var, target_var, multiply_var):
# input layer with unspecified batch size
layer = InputLayer(shape=(None, 30, 64, 64), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var)
layer = DimshuffleLayer(layer, (0, 'x', 1, 2, 3))
# Z-score?
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify))
layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify))
layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=1, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify))
layer_prediction = layer
# Loss
prediction = get_output(layer_prediction)
loss = categorical_crossentropy(prediction.flatten(), target_var.flatten())
#Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum
params = get_all_params(layer_prediction, trainable=True)
# Create a loss expression for validation/testing. The crucial difference
# here is that we do a deterministic forward pass through the network, disabling dropout layers.
test_prediction = get_output(layer_prediction, deterministic=True)
test_loss = categorical_crossentropy(test_prediction.flatten(), target_var.flatten())
return test_prediction, prediction, loss, params
def build_convpool_conv1d(input_vars, nb_classes, imsize=32, n_colors=3, n_timewin=3):
"""
Builds the complete network with 1D-conv layer to integrate time from sequences of EEG images.
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 7 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i], imsize=imsize, n_colors=n_colors)
else:
convnet, _ = build_cnn(input_vars[i], w_init=w_init, imsize=imsize, n_colors=n_colors)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
convpool = ReshapeLayer(convpool, ([0], n_timewin, get_output_shape(convnets[0])[1]))
convpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
convpool = Conv1DLayer(convpool, 64, 3)
# A fully-connected layer of 512 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the output layer with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=nb_classes, nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def createConvtScaleSpaceLayer(input_layer_4d, resize_ratio_list, name=None):
input_layer_shape = get_output_shape(input_layer_4d)
batch_size = input_layer_shape[0]
# patch_size = input_layer_shape[2]
# orig_c = (float(input_layer_shape[2]) - 1.0) * 0.5
scale_space_layer_list = []
for resize_ratio in resize_ratio_list:
rf = resize_ratio
c = 0
# the implementation already works on 0-center coordinate system
rescaleA = np.tile(
np.asarray([[rf, 0], [0, rf], [c, c]],
dtype=floatX).T.reshape([1, 2, 3]),
[batch_size, 1, 1]).reshape([-1, 6])
param_layer = InputLayer((batch_size, 6),
input_var=theano.shared(rescaleA))
resize_layer = TransformerLayer(input_layer_4d, param_layer,
input_layer_shape[2],
input_layer_shape[3])
scale_space_layer_list += [
# ReshapeLayer(resize_layer,
# tuple([v for v in input_layer_shape] + [1]))
DimshuffleLayer(resize_layer, (0, 1, 2, 3, 'x'))
]
scale_space_layer = ConcatLayer(scale_space_layer_list, axis=4, name=name)
return scale_space_layer
def build_res_rnn_network(rnnmodel):
net = {}
net['input'] = InputLayer((batch_size, seq_len, feature_size))
net['rnn0']=DimshuffleLayer(net['input'],(1,0,2))
for l in range(1, num_layers+1):
hidini=0
if l==num_layers:
hidini=U_lowbound
net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (batch_size* seq_len, -1))
net['rnn%d'%(l-1)]=DenseLayer(net['rnn%d'%(l-1)],hidden_units,W=ini_W,b=Uniform(range=(0,args.ini_b)),nonlinearity=None) #W=Uniform(ini_rernn_in_to_hid), #
net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (seq_len, batch_size, -1))
net['rnn%d'%l]=net['rnn%d'%(l-1)]
if not args.use_bn_afterrnn:
net['rnn%d'%l]=BatchNormLayer(net['rnn%d'%l],axes= (0,1),beta=Uniform(range=(0,args.ini_b)))
net['rnn%d'%l]=rnnmodel(net['rnn%d'%l],hidden_units,W_hid_to_hid=Uniform(range=(hidini,U_bound)),nonlinearity=act,only_return_final=False, grad_clipping=args.gradclipvalue)
if args.use_bn_afterrnn:
net['rnn%d'%l]=BatchNormLayer(net['rnn%d'%l],axes= (0,1))
if l==num_layers:
net['rnn%d'%num_layers]=lasagne.layers.SliceLayer(net['rnn%d'%num_layers],indices=-1, axis=0)
net['out']=DenseLayer(net['rnn%d'%num_layers],outputclass,nonlinearity=softmax)
return net
def build_convpool_mix(input_vars,
nb_classes,
grad_clip=110,
imsize=32,
n_colors=3,
n_timewin=7):
"""
Builds the complete network with LSTM and 1D-conv layers combined
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param grad_clip: the gradient messages are clipped to the given value during
the backward pass.
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 7 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i],
imsize=imsize,
n_colors=n_colors)
else:
convnet, _ = build_cnn(input_vars[i],
w_init=w_init,
imsize=imsize,
n_colors=n_colors)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
convpool = ReshapeLayer(convpool,
([0], n_timewin, get_output_shape(convnets[0])[1]))
reformConvpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
conv_out = Conv1DLayer(reformConvpool, 64, 3)
conv_out = FlattenLayer(conv_out)
# Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features)
lstm = LSTMLayer(convpool,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
lstm_out = SliceLayer(lstm, -1, 1)
# Merge 1D-Conv and LSTM outputs
dense_input = ConcatLayer([conv_out, lstm_out])
# A fully-connected layer of 256 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5),
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the 10-unit output layer with 50% dropout on its inputs:
convpool = DenseLayer(convpool,
num_units=nb_classes,
nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def _combine_encoder_steps(self, encoder):
#print(lasagne.layers.get_output_shape(encoder))
shuffled_encoder = DimshuffleLayer(encoder, (0, 2, 1))
#print(lasagne.layers.get_output_shape(shuffled_encoder))
encoder = lasagne.layers.DenseLayer(shuffled_encoder, num_units=1, num_leading_axes=2)
#print(lasagne.layers.get_output_shape(encoder))
encoder = ReshapeLayer(encoder, shape=(self.batch_size, self.n_hidden))
#print(lasagne.layers.get_output_shape(encoder))
return encoder
def construct_tiramisu(channels=1, no_f_base=45, f_size_base=3, bs=None, class_nums=2, k=16, denseblocks=[4,5,7,10,12], blockbottom=[15],
dropout=0, input_var=None, pad="same",c_nonlinearity=lasagne.nonlinearities.rectify,
f_nonlinearity=lasagne.nonlinearities.rectify, input_dim=[112,112]):
#Network start
net = InputLayer((bs, channels, input_dim[0], input_dim[1]), input_var)
net = Conv2DLayer(net, no_f_base, f_size_base, pad=pad, W=lasagne.init.HeUniform(gain='relu'), flip_filters=False)
#Downward block building
horizontal_pass=[]
for blocks in xrange(len(denseblocks)):
net_preblock = net
net = tiramisu_denseblock(net, denseblocks[blocks], k, drop_p=dropout)
net = ConcatLayer([net_preblock, net], axis=1) #Connection around Denseblock
print "Input shape: {}, after concat: {}".format(net_preblock.output_shape,net.output_shape)
horizontal_pass.append(net)
net = tiramisu_transistion_down(net, drop_p=dropout)
print "After transition down: ",net.output_shape
print "---Down done---"
#Bottom dense block
for bottom in xrange(len(blockbottom)):
net = tiramisu_denseblock(net, blockbottom[bottom], k, drop_p=dropout)
if bottom < len(blockbottom)-1:
net = tiramisu_transition_bottom(net, drop_p=dropout)
print "Bottom: ",net.output_shape
print "---Bottom done---"
#Up dense block
for block in xrange(len(denseblocks)):
print "Before concat size: ",net.output_shape
net = tiramisu_transistion_up(net, f_size_base, 2)
net = ConcatLayer([net, horizontal_pass[-(block+1)]], axis=1,cropping=[None, None, "center", "center"])
print "After concat size: ",net.output_shape
net = tiramisu_denseblock(net, denseblocks[-(block+1)], k, drop_p=dropout)
print "Denseblock size: ",net.output_shape
print "---Up done---"
#Out block
image_out = net = Conv2DLayer(net, class_nums, 1, pad=pad, W=lasagne.init.HeUniform(gain='relu'), nonlinearity=linear, flip_filters=False)
net = DimshuffleLayer(net,(1,0,2,3))
net = ReshapeLayer(net, (class_nums,-1))
net = DimshuffleLayer(net, (1,0))
net = NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax)
return net,image_out
net = None
image_out=None
def sliding_window_input(input_layer):
window_size = 5
sub_input = []
for i in xrange(window_size):
indices = slice(window_size - i - 1, -i if i > 0 else None)
network = DimshuffleLayer(SliceLayer(input_layer, indices, axis=-1),
(0, 1, 'x'))
sub_input.append(network)
network = ConcatLayer(sub_input, -1)
return network
def build_lstm(input_vars, input_shape=None):
'''
1) InputLayer
2) ReshapeLayer
3) LSTM Layer 1
4) LSTM Layer 2
5) Slice Layer
6) Fully Connected Layer 1 w/ dropout tanh
7) Fully Connected Layer 2 w/ dropout softmax
'''
# Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features)
network = InputLayer(shape=(input_shape[0], None, num_input_channels,
input_shape[-3], input_shape[-2],
input_shape[-1]),
input_var=input_vars)
network = ReshapeLayer(network, ([0], [1], -1))
network = DimshuffleLayer(network, (1, 0, 2))
#network = ReshapeLayer(network, (-1, 128))
#l_inp = InputLayer((None, None, num_inputs))
l_lstm1 = LSTMLayer(network,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
#New LSTM
l_lstm2 = LSTMLayer(l_lstm1,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
#end of insertion
# After LSTM layer you either need to reshape or slice it (depending on whether you
# want to keep all predictions or just the last prediction.
# http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html
# https://github.com/Lasagne/Recipes/blob/master/examples/lstm_text_generation.py
l_lstm_slice = SliceLayer(l_lstm2, -1, 1) # Selecting the last prediction
# A fully-connected layer of 256 units with 50% dropout on its inputs:
l_dense = DenseLayer(lasagne.layers.dropout(l_lstm_slice, p=.5),
num_units=256,
nonlinearity=lasagne.nonlinearities.rectify)
# We only need the final prediction, we isolate that quantity and feed it
# to the next layer.
# And, finally, the output layer with 50% dropout on its inputs:
l_dense = DenseLayer(lasagne.layers.dropout(l_dense, p=.5),
num_units=num_classes,
nonlinearity=lasagne.nonlinearities.softmax)
return l_dense
def build_indrnn_network(X_sym):
net = {}
net['input0'] = InputLayer((batch_size, seq_len, indim, 3), X_sym)
net['input'] = ReshapeLayer(net['input0'],
(batch_size, seq_len, indim * 3))
net['rnn0'] = DimshuffleLayer(net['input'], (1, 0, 2))
for l in range(1, num_layers + 1):
hidini = 0
if l == num_layers:
hidini = U_lowbound
net['rnn%d' % (l - 1)] = ReshapeLayer(net['rnn%d' % (l - 1)],
(batch_size * seq_len, -1))
net['rnn%d' % (l - 1)] = DenseLayer(net['rnn%d' % (l - 1)],
hidden_units,
W=ini_W,
b=lasagne.init.Constant(
args.ini_b),
nonlinearity=None) #
net['rnn%d' % (l - 1)] = ReshapeLayer(net['rnn%d' % (l - 1)],
(seq_len, batch_size, -1))
if args.conv_drop:
net['rnn%d' % (l - 1)] = DropoutLayer(net['rnn%d' % (l - 1)],
p=droprate,
shared_axes=(0, ))
net['rnn%d' % l] = net['rnn%d' % (l - 1)]
if not args.use_bn_afterrnn:
net['rnn%d' % l] = BatchNormLayer(net['rnn%d' % l],
beta=lasagne.init.Constant(
args.ini_b),
axes=(0, 1))
net['rnn%d' % l] = rnnmodel(net['rnn%d' % l],
hidden_units,
W_hid_to_hid=Uniform(range=(hidini,
U_bound)),
nonlinearity=act,
only_return_final=False,
grad_clipping=gradclipvalue)
if args.use_bn_afterrnn:
net['rnn%d' % l] = BatchNormLayer(net['rnn%d' % l], axes=(0, 1))
if args.use_dropout and l % args.drop_layers == 0:
net['rnn%d' % l] = DropoutLayer(net['rnn%d' % l],
p=droprate,
shared_axes=(0, ))
net['rnn%d' % num_layers] = lasagne.layers.SliceLayer(net['rnn%d' %
num_layers],
indices=-1,
axis=0)
net['out'] = DenseLayer(net['rnn%d' % num_layers],
outputclass,
nonlinearity=softmax)
return net
def build_cnn(input_layer):
# Add a channel axis for convolutional nets
network = DimshuffleLayer(input_layer, (0, 'x', 1))
network = Conv1DLayer(network, num_filters=4, filter_size=5,
nonlinearity=rectify)
network = MaxPool1DLayer(network, pool_size=2)
network = Conv1DLayer(network, num_filters=4, filter_size=5,
nonlinearity=rectify)
network = MaxPool1DLayer(network, pool_size=2)
network = DropoutLayer(network, p=.5)
network = DenseLayer(network, num_units=256, nonlinearity=rectify)
return network
def build_convpool_mix(input_vars, input_shape=None):
"""
Builds the complete network with LSTM and 1D-conv layers combined
to integrate time from sequences of EEG images.
:param input_vars: list of EEG images (one image per time window)
:return: a pointer to the output of last layer
"""
convnets = []
W_init = None
# Build 7 parallel CNNs with shared weights
for i in range(input_shape[0]):
if i == 0:
convnet, W_init = build_cnn(input_vars[i], input_shape)
else:
convnet, _ = build_cnn(input_vars[i], input_shape, W_init)
convnets.append(FlattenLayer(convnet))
# at this point convnets shape is [numTimeWin][n_samples, features]
# we want the shape to be [n_samples, features, numTimeWin]
convpool = ConcatLayer(convnets)
# convpool = ReshapeLayer(convpool, ([0], -1, numTimeWin))
convpool = ReshapeLayer(
convpool, ([0], input_shape[0], get_output_shape(convnets[0])[1]))
reformConvpool = DimshuffleLayer(convpool, (0, 2, 1))
# input to 1D convlayer should be in (batch_size, num_input_channels, input_length)
conv_out = Conv1DLayer(reformConvpool, 64, 3)
conv_out = FlattenLayer(conv_out)
# Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features)
lstm = LSTMLayer(convpool,
num_units=128,
grad_clipping=grad_clip,
nonlinearity=lasagne.nonlinearities.tanh)
# After LSTM layer you either need to reshape or slice it (depending on whether you
# want to keep all predictions or just the last prediction.
# http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html
# https://github.com/Lasagne/Recipes/blob/master/examples/lstm_text_generation.py
lstm_out = SliceLayer(lstm, -1, 1)
# Merge 1D-Conv and LSTM outputs
dense_input = ConcatLayer([conv_out, lstm_out])
# A fully-connected layer of 256 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5),
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify)
# We only need the final prediction, we isolate that quantity and feed it
# to the next layer.
# And, finally, the 10-unit output layer with 50% dropout on its inputs:
convpool = DenseLayer(convpool,
num_units=num_classes,
nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def build_model(self):
# reshape to [batch, color, x, y] to allow for convolution layers to work correctly
observation_reshape = DimshuffleLayer(self.observation_layer,
(0, 3, 1, 2))
observation_reshape = Pool2DLayer(observation_reshape,
pool_size=(2, 2))
# memory
window_size = 5
# prev state input
prev_window = InputLayer(
(None, window_size) + tuple(observation_reshape.output_shape[1:]),
name="previous window state")
# our window
memory_layer = WindowAugmentation(observation_reshape,
prev_window,
name="new window state")
memory_dict = {memory_layer: prev_window}
# pixel-wise maximum over the temporal window (to avoid flickering)
memory_layer = ExpressionLayer(memory_layer,
lambda a: a.max(axis=1),
output_shape=(None, ) +
memory_layer.output_shape[2:])
# neural network body
nn = batch_norm(
lasagne.layers.Conv2DLayer(memory_layer,
num_filters=16,
filter_size=(8, 8),
stride=(4, 4)))
nn = batch_norm(
lasagne.layers.Conv2DLayer(nn,
num_filters=32,
filter_size=(4, 4),
stride=(2, 2)))
nn = batch_norm(lasagne.layers.DenseLayer(nn, num_units=256))
# q_eval
policy_layer = DenseLayer(nn,
num_units=self.n_actions,
nonlinearity=lasagne.nonlinearities.linear,
name="QEvaluator")
# resolver
resolver = EpsilonGreedyResolver(policy_layer, name="resolver")
# all together
agent = Agent(self.observation_layer, memory_dict, policy_layer,
resolver)
return resolver, agent
def build_lstm(input_layer):
#network = sliding_window_input(input_layer)
network = DimshuffleLayer(input_layer, (0, 1, 'x'))
n_hidden = 50
grad_clipping = 20
network = LSTMLayer(network, num_units=n_hidden,
grad_clipping=grad_clipping, nonlinearity=tanh)
network = LSTMLayer(network, num_units=n_hidden,
grad_clipping=grad_clipping, nonlinearity=tanh)
network = SliceLayer(network, indices=-1, axis=1)
#network = DenseLayer(network, num_units=256, nonlinearity=rectify)
return network
def construct_unet_recursive(channels=1, no_f_base=8, f_size=3, branches=[2,2,2,2],dropout=0.2,bs=None,
class_nums=2, pad="same",nonlinearity=lasagne.nonlinearities.rectify, input_dim=[512,512]):
net= InputLayer((bs, channels, input_dim[0], input_dim[1]))
# Moving downwards the U-shape:
horizontal_pass=[]
for i in xrange(len(branches)):
net = conv_pool_down(net,no_f_base*2**(i),f_size,conv_depth=branches[i],
pad=pad,nonlinearity=nonlinearity,dropout=dropout)
horizontal_pass.append(net)
net = Pool2DLayer(net,pool_size=2)
# Bottleneck
net = Conv2DLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu'))
net = Conv2DLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu'))
net = Deconv2DLayer(net, no_f_base*2**(len(branches)-1), 2, 2)
# Moving upwards the U-shape:
for i in xrange(len(branches)):
net = PadLayer(net,1)
net = ConcatLayer([net,horizontal_pass[-(i+1)]],cropping=(None,None,"center","center"))
if i==len(branches)-1:
net = conv_pool_up(net,no_f_base*2**(len(branches)-1-i),f_size,
pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=True)
else:
net = conv_pool_up(net,no_f_base*2**(len(branches)-1-i),f_size,
pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=False)
# Class layer: Work around standard softmax bc. it doesn't work with tensor4/3.
# Hence, we reshape and feed it to an external Nonlinearity layer.
# net["class_ns"] is the output in image-related shape.
imageout = net = Conv2DLayer(net, class_nums, 1, nonlinearity=linear,W=lasagne.init.HeNormal(gain='relu'))
net = DimshuffleLayer(net, (1, 0, 2, 3))
net = ReshapeLayer(net, (class_nums, -1))
net = DimshuffleLayer(net, (1, 0))
# Flattened output to be able to feed it to lasagne.objectives.categorical_crossentropy.
net = NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax)
return net,imageout
del net, imageout
def __init__(self, input_shape=(None, 1, 33, 33, 33)):
self.cubeSize = input_shape[-1]
# Theano variables
self.input_var = T.tensor5('input_var') # input image
self.target_var = T.ivector('target_var') # target
self.logger = logging.getLogger(__name__)
input_layer = InputLayer(input_shape, self.input_var)
self.logger.info('The shape of input layer is {}'.format(
get_output_shape(input_layer)))
hidden_layer1 = Conv3DLayer(incoming=input_layer,
num_filters=16,
filter_size=(3, 3, 3),
W=HeUniform(gain='relu'),
nonlinearity=rectify)
self.logger.info('The shape of first hidden layer is {}'.format(
get_output_shape(hidden_layer1)))
hidden_layer2 = Conv3DLayer(incoming=hidden_layer1,
num_filters=32,
filter_size=(3, 3, 3),
W=HeUniform(gain='relu'),
nonlinearity=rectify)
self.logger.info('The shape of second hidden layer is {}'.format(
get_output_shape(hidden_layer2)))
hidden_layer3 = Conv3DLayer(incoming=hidden_layer2,
num_filters=2,
filter_size=(1, 1, 1),
W=HeUniform(gain='relu'),
nonlinearity=rectify)
self.logger.info('The shape of third hidden layer is {}'.format(
get_output_shape(hidden_layer3)))
shuffledLayer = DimshuffleLayer(hidden_layer3, (0, 2, 3, 4, 1))
self.logger.info('The shape of shuffled layer is {}'.format(
get_output_shape(shuffledLayer)))
reshapedLayer = ReshapeLayer(shuffledLayer, ([0], -1))
self.logger.info('The shape of reshaped layer is {}'.format(
get_output_shape(reshapedLayer)))
self.output_layer = NonlinearityLayer(reshapedLayer, softmax)
self.logger.info('The shape of output layer is {}'.format(
get_output_shape(self.output_layer)))
