def output_path(net, incoming_layer, n_classes, filter_size, out_nonlin): ''' Build the output path (including last conv layer to have n_classes feature maps). Dimshuffle layers to fit with softmax implementation Parameters ---------- Same as above incoming_layer : string, name of last layer from bottleneck layers ''' #Final convolution (n_classes feature maps) with filter_size = 1 net['final_conv'] = ConvLayer(net[incoming_layer], n_classes, 1) #DimshuffleLayer and all this stuff is necessary to fit with softmax #implementation. In training, we specify layer = ['probs'] to have the #right layer but the 2 last reshape layers are necessary only to visualize #data. net['final_dimshuffle'] = DimshuffleLayer(net['final_conv'], (0, 2, 1)) laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape net['final_reshape'] = ReshapeLayer(net['final_dimshuffle'], (T.prod(laySize[0:2]), laySize[2])) net['probs'] = NonlinearityLayer(net['final_reshape'], nonlinearity=out_nonlin) net['probs_reshape'] = ReshapeLayer(net['probs'], (laySize[0], laySize[1], n_classes)) net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'], (0, 2, 1)) return net
def SoftmaxLayer(inputs, n_classes): """ Performs 1x1 convolution followed by softmax nonlinearity The output will have the shape (batch_size * n_rows * n_cols, n_classes) """ l = Conv2DLayer(inputs, n_classes, filter_size=1, nonlinearity=linear, W=HeUniform(gain='relu'), pad='same', flip_filters=False, stride=1) # We perform the softmax nonlinearity in 2 steps : # 1. Reshape from (batch_size, n_classes, n_rows, n_cols) to (batch_size * n_rows * n_cols, n_classes) # 2. Apply softmax l = DimshuffleLayer(l, (0, 2, 3, 1)) batch_size, n_rows, n_cols, _ = get_output(l).shape l = ReshapeLayer(l, (batch_size * n_rows * n_cols, n_classes)) l = NonlinearityLayer(l, softmax) l = ReshapeLayer(l, (batch_size, n_rows, n_cols, n_classes)) l = DimshuffleLayer(l, (0, 3, 1, 2)) l = ReshapeLayer(l, (batch_size, n_classes, n_rows, n_cols)) return l
def TransitionalNormalizeLayer(inputs, n_directions): """ Performs 1x1 convolution followed by softmax nonlinearity. The output will have the shape (batch_size * n_rows * n_cols, n_classes) """ l = Conv2DLayer(inputs, n_directions**2, filter_size=1, nonlinearity=linear, W=HeUniform(gain='relu'), pad='same', flip_filters=False, stride=1) # We perform the softmax nonlinearity in 2 steps : # 1. Reshape from (batch_size, n_classes, n_rows, n_cols) to (batch_size * n_rows * n_cols, n_classes) # 2. Apply softmax batch_size, n_channels, n_rows, n_cols = get_output(l).shape l = ReshapeLayer(l, (batch_size, n_directions, n_directions, n_rows, n_cols)) l = DimshuffleLayer(l, (0, 1, 3, 4, 2)) l = ReshapeLayer( l, (batch_size * n_directions * n_rows * n_cols, n_directions)) l = NormalizeLayer(l) l = ReshapeLayer(l, (batch_size, n_directions, n_rows, n_cols, n_directions)) l = DimshuffleLayer(l, (0, 1, 4, 2, 3)) l = ReshapeLayer(l, (batch_size, n_channels, n_rows, n_cols)) return l
def construct_unet_3D(channels=1, no_f_base=8, f_size=3, branches=[2,2,2,2],dropout=0.2,bs=None, class_nums=2, pad="same",nonlinearity=lasagne.nonlinearities.rectify, input_dim=[None,None,None],useups=False): net= InputLayer((bs, channels, input_dim[0], input_dim[1], input_dim[2])) # Moving downwards the U-shape: horizontal_pass=[] for i in xrange(len(branches)): net = conv_pool_down_3D(net,no_f_base*2**(i),f_size,conv_depth=branches[i], pad=pad,nonlinearity=nonlinearity,dropout=dropout) print "Down conv: ",net.output_shape horizontal_pass.append(net) net = MaxPool3DDNNLayer(net,pool_size=(2,2,2),stride=(2,2,2)) print "Down Pool: ",net.output_shape # Bottleneck net = Conv3DDNNLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu')) print "Bottleneck conv: ",net.output_shape net = Conv3DDNNLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu')) print "Bottleneck conv: ",net.output_shape #net = Conv3DDNNTransposeLayer(net, no_f_base*2**(len(branches)-1), 2, (2, 2, 2)) if not useups: net = TransposedConv3DLayer(net,no_f_base*2**(len(branches)-1),2,(2,2,2)) else: net = upscale_plus_conv_3D(net,no_f_base*2**(len(branches)-1),f_size,pad,nonlinearity) print "Bottleneck up: ",net.output_shape # Moving upwards the U-shape: for i in xrange(len(branches)): print "Pass before concat: ",horizontal_pass[-(i+1)].output_shape print "net before concat: ",net.output_shape if not useups: net = ConcatLayer([net,horizontal_pass[-(i+1)]],cropping=(None,None,"center","center","center")) else: net = ConcatLayer([net,horizontal_pass[-(i+1)]],cropping=(None,None,"center","center","center")) print "Shape after concat: ",net.output_shape if i==len(branches)-1: net = conv_pool_up_3D(net,bs,no_f_base*2**(len(branches)-1-i),f_size, pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=True,useups=False) else: net = conv_pool_up_3D(net,bs,no_f_base*2**(len(branches)-1-i),f_size, pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=False,useups=False) print "Conv up: ",net.output_shape # Class layer: Work around standard softmax bc. it doesn't work with tensor4/3. # Hence, we reshape and feed it to an external Nonlinearity layer. # net["class_ns"] is the output in image-related shape. imageout = net = Conv3DDNNLayer(net, class_nums, 1, nonlinearity=linear,W=lasagne.init.HeNormal(gain='relu')) print "imageout shape: ",net.output_shape net = DimshuffleLayer(net, (1, 0, 2, 3, 4)) print "After shuffle shape: ",net.output_shape net = ReshapeLayer(net, (class_nums, -1)) print "Reshape shape: ",net.output_shape net = DimshuffleLayer(net, (1, 0)) print "Dimshuffle shape: ",net.output_shape # Flattened output to be able to feed it to lasagne.objectives.categorical_crossentropy. net = NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax) #imout = NonlinearityLayer(imageout,nonlinearity=lasagne.nonlinearities.softmax) return net,imageout del net, imageout,imout
def get_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=L.nonlinearities.leaky_rectify, input_dim=(128, 128), base_n_filters=128): net = OrderedDict() net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1])) net['contr_1_1'] = batch_norm(ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)) net['contr_1_2'] = batch_norm(ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)) net['pool1'] = Pool2DLayer(net['contr_1_2'], 2) net['contr_2_1'] = batch_norm(ConvLayer(net['pool1'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad)) net['contr_2_2'] = batch_norm(ConvLayer(net['contr_2_1'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad)) net['pool2'] = Pool2DLayer(net['contr_2_2'], 2) net['contr_3_1'] = batch_norm(ConvLayer(net['pool2'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad)) net['contr_3_2'] = batch_norm(ConvLayer(net['contr_3_1'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad)) net['pool3'] = Pool2DLayer(net['contr_3_2'], 2) net['contr_4_1'] = batch_norm(ConvLayer(net['pool3'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad)) net['contr_4_2'] = batch_norm(ConvLayer(net['contr_4_1'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad)) l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) # the paper does not really describe where and how dropout is added. Feel free to try more options l = DropoutLayer(l, p=0.4) net['encode_1'] = batch_norm(ConvLayer(l, base_n_filters * 16, 3, nonlinearity=nonlinearity, pad=pad)) net['encode_2'] = batch_norm(ConvLayer(net['encode_1'], base_n_filters * 16, 3, nonlinearity=nonlinearity, pad=pad)) net['deconv1'] = Upscale2DLayer(net['encode_2'], 2) net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center")) net['expand_1_1'] = batch_norm(ConvLayer(net['concat1'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad)) net['expand_1_2'] = batch_norm(ConvLayer(net['expand_1_1'], base_n_filters * 8, 3, nonlinearity=nonlinearity, pad=pad)) net['deconv2'] = Upscale2DLayer(net['expand_1_2'], 2) net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center")) net['expand_2_1'] = batch_norm(ConvLayer(net['concat2'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad)) net['expand_2_2'] = batch_norm(ConvLayer(net['expand_2_1'], base_n_filters * 4, 3, nonlinearity=nonlinearity, pad=pad)) net['deconv3'] = Upscale2DLayer(net['expand_2_2'], 2) net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center")) net['expand_3_1'] = batch_norm(ConvLayer(net['concat3'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad)) net['expand_3_2'] = batch_norm(ConvLayer(net['expand_3_1'], base_n_filters * 2, 3, nonlinearity=nonlinearity, pad=pad)) net['deconv4'] = Upscale2DLayer(net['expand_3_2'], 2) net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center")) net['expand_4_1'] = batch_norm(ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)) net['expand_4_2'] = batch_norm(ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad)) net['conv_5'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) # (bs, nrClasses, x, y) net['dimshuffle'] = DimshuffleLayer(net['conv_5'], (1, 0, 2, 3)) # (nrClasses, bs, x, y) net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) # (nrClasses, bs*x*y) net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) # (bs*x*y, nrClasses) #Watch out: here is another nonlinearity -> do not use layers before this layer! net['output_flat'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=L.nonlinearities.sigmoid) # (bs*x*y, nrClasses) img_shape = net["conv_5"].output_shape net['output'] = ReshapeLayer(net['output_flat'], (-1, img_shape[2], img_shape[3], img_shape[1])) # (bs, x, y, nrClasses) return net
def SoftmaxLayer(inputs, n_classes): l = Conv2DLayer(inputs, n_classes, filter_size=1, nonlinearity=linear, W=HeUniform(gain='relu'), pad='same', flip_filters=False, stride=1) l = DimshuffleLayer(l,(1,0,2,3)) l = ReshapeLayer(l, (n_classes,-1)) l = DimshuffleLayer(l, (1,0)) l = NonlinearityLayer(l, nonlinearity=softmax) return l
def get_model(input_var, target_var, multiply_var): # input layer with unspecified batch size layer_input = InputLayer(shape=(None, 30, 80, 80), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var) layer_0 = DimshuffleLayer(layer_input, (0, 'x', 1, 2, 3)) # Z-score? # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_1 = batch_norm(Conv3DDNNLayer(incoming=layer_0, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_2 = batch_norm(Conv3DDNNLayer(incoming=layer_1, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_3 = MaxPool3DDNNLayer(layer_2, pool_size=(2, 2, 2), stride=(2, 2, 2), pad=(1, 1, 1)) layer_4 = DropoutLayer(layer_3, p=0.25) # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_5 = batch_norm(Conv3DDNNLayer(incoming=layer_4, num_filters=32, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_6 = batch_norm(Conv3DDNNLayer(incoming=layer_5, num_filters=32, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_7 = MaxPool3DDNNLayer(layer_6, pool_size=(2, 2, 2), stride=(2, 2, 2), pad=(1, 1, 1)) layer_8 = DropoutLayer(layer_7, p=0.25) # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_5 = batch_norm(Conv3DDNNLayer(incoming=layer_8, num_filters=64, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_6 = batch_norm(Conv3DDNNLayer(incoming=layer_5, num_filters=64, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_7 = batch_norm(Conv3DDNNLayer(incoming=layer_6, num_filters=64, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=leaky_rectify)) layer_8 = MaxPool3DDNNLayer(layer_7, pool_size=(2, 2, 2), stride=(2, 2, 2), pad=(1, 1, 1)) layer_9 = DropoutLayer(layer_8, p=0.25) # LSTM layer = DimshuffleLayer(layer_9, (0,2,1,3,4)) # layer_prediction = LSTMLayer(layer, num_units=2, only_return_final=True, learn_init=True, cell=Gate(linear)) layer = LSTMLayer(layer, num_units=2, only_return_final=True, learn_init=True) layer_prediction = DenseLayer(layer, 2, nonlinearity=linear) # Output Layer # layer_hidden = DenseLayer(layer_flatten, 500, nonlinearity=linear) # layer_prediction = DenseLayer(layer_hidden, 2, nonlinearity=linear) # Loss prediction = get_output(layer_prediction) / multiply_var**2 loss = T.abs_(prediction - target_var) loss = loss.mean() #Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum params = get_all_params(layer_prediction, trainable=True) # Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, disabling dropout layers. test_prediction = get_output(layer_prediction, deterministic=True) / multiply_var**2 test_loss = T.abs_(test_prediction - target_var) test_loss = test_loss.mean() # crps estimate crps = T.abs_(test_prediction - target_var).mean()/600 return test_prediction, crps, loss, params
def cnn_fn(self): l_in = InputLayer((None, self.max_length, self.vocab_size)) l_in_T = DimshuffleLayer(l_in, (0, 2, 1)) l_causal_conv = DilatedConv1DLayer( l_in_T, num_filters=self.nn_residual_channels, dilation=1, nonlinearity=None) l_prev = l_causal_conv skip_layers = [] for h in range(len(self.nn_dilations)): l_filter = DilatedConv1DLayer( l_prev, num_filters=self.nn_dilation_channels, dilation=self.nn_dilations[h], nonlinearity=tanh) l_gate = DilatedConv1DLayer(l_prev, num_filters=self.nn_dilation_channels, dilation=self.nn_dilations[h], nonlinearity=sigmoid) l_merge = ElemwiseMergeLayer([l_filter, l_gate], merge_function=T.mul) l_dense = Conv1DLayer(l_merge, num_filters=self.nn_residual_channels, filter_size=1, nonlinearity=None) l_residual = ElemwiseSumLayer([l_prev, l_dense]) l_skip = Conv1DLayer(l_merge, num_filters=self.nn_residual_channels, filter_size=1, nonlinearity=None) skip_layers.append(l_skip) l_prev = l_residual l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers), nonlinearity=elu) l_final = DimshuffleLayer(l_skip_sum, (0, 2, 1)) return l_final
def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(128, 128), base_n_filters=64, do_dropout=False): net = OrderedDict() net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1])) net['contr_1_1'] = ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['contr_1_2'] = ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['pool1'] = Pool2DLayer(net['contr_1_2'], 2) net['contr_2_1'] = ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) net['contr_2_2'] = ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) net['pool2'] = Pool2DLayer(net['contr_2_2'], 2) net['contr_3_1'] = ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) net['contr_3_2'] = ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) net['pool3'] = Pool2DLayer(net['contr_3_2'], 2) net['contr_4_1'] = ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) net['contr_4_2'] = ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) # the paper does not really describe where and how dropout is added. Feel free to try more options if do_dropout: l = DropoutLayer(l, p=0.4) net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) net['deconv1'] = Upscale2DLayer(net['encode_2'], 2) net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center")) net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) net['deconv2'] = Upscale2DLayer(net['expand_1_2'], 2) net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center")) net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) net['deconv3'] = Upscale2DLayer(net['expand_2_2'], 2) net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center")) net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) net['deconv4'] = Upscale2DLayer(net['expand_3_2'], 2) net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center")) net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3)) net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) net['output_flattened'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax) return net
def build_rnn_network(rnnmodel,X_sym,hid_init_sym): net = {} net['input0'] = InputLayer((batch_size, seq_len),X_sym) net['input']=lasagne.layers.EmbeddingLayer(net['input0'],outputclass,units[0])#,W=lasagne.init.Uniform(inial_scale) net['rnn0']=DimshuffleLayer(net['input'],(1,0,2)) #change to (time, batch_size,hidden_units) if use_bn_embed: net['rnn0']=BatchNorm_step_timefirst_Layer(net['rnn0'],axes=(0,1),epsilon=args.epsilon ) for l in range(1, num_layers+1): net['hiddeninput%d'%l] = InputLayer((batch_size, units[l-1]),hid_init_sym[:,acc_units[l-1]:acc_units[l]]) net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (batch_size* seq_len, -1)) net['rnn%d'%(l-1)]=DenseLayer(net['rnn%d'%(l-1)],units[l-1],W=ini_W,b=lasagne.init.Constant(args.ini_b),nonlinearity=None) #W=Uniform(ini_rernn_in_to_hid), # net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (seq_len, batch_size, -1)) if args.use_residual and l>args.residual_layers and (l-1)%args.residual_layers==0:# and l!=num_layers if units[l - 1]!=units[l - 1 - args.residual_layers]: net['leftbranch%d' % (l - 1)] = ReshapeLayer(net['sum%d'%(l-args.residual_layers)], (batch_size * seq_len, -1)) net['leftbranch%d' % (l - 1)] = DenseLayer(net['leftbranch%d' % (l - 1)], units[l - 1], W=ini_W, nonlinearity=None) net['leftbranch%d' % (l - 1)] = ReshapeLayer(net['leftbranch%d' % (l - 1)], (seq_len, batch_size, -1)) net['leftbranch%d' % (l - 1)] = BatchNorm_step_timefirst_Layer(net['leftbranch%d' % (l - 1)], axes=(0, 1), epsilon=args.epsilon) print('left branch') else: net['leftbranch%d' % (l - 1)] = net['sum%d'%(l-args.residual_layers)] net['sum%d'%l]=ElemwiseSumLayer((net['rnn%d'%(l-1)],net['leftbranch%d' % (l - 1)])) else: net['sum%d'%l]=net['rnn%d'%(l-1)] net['rnn%d'%l]=net['sum%d'%l] if not args.use_bn_afterrnn: net['rnn%d'%l]=BatchNorm_step_timefirst_Layer(net['rnn%d'%l],axes= (0,1),beta=lasagne.init.Constant(args.ini_b),epsilon=args.epsilon) ini_hid_start=0 if act==tanh: ini_hid_start=-1*U_bound net['rnn%d'%l]=rnnmodel(net['rnn%d'%l],units[l-1],hid_init=net['hiddeninput%d'%l],W_hid_to_hid=Uniform(range=(ini_hid_start,U_bound)),nonlinearity=act,only_return_final=False, grad_clipping=args.gradclipvalue) net['last_state%d'%l]=SliceLayer(net['rnn%d'%l],-1, axis=0) if l==1: net['hid_out']=net['last_state%d'%l] else: net['hid_out']=ConcatLayer([net['hid_out'], net['last_state%d'%l]],axis=1) if use_dropout and l%droplayers==0 and not args.bn_drop: net['rnn%d'%l]=lasagne.layers.DropoutLayer(net['rnn%d'%l], p=droprate, shared_axes=taxdrop) if args.use_bn_afterrnn: net['rnn%d'%l]=BatchNorm_step_timefirst_Layer(net['rnn%d'%l],axes= (0,1),epsilon=args.epsilon) net['rnn%d'%num_layers]=DimshuffleLayer(net['rnn%d'%num_layers],(1,0,2)) net['reshape_rnn']=ReshapeLayer(net['rnn%d'%num_layers],(-1,units[num_layers-1])) net['out']=DenseLayer(net['reshape_rnn'],outputclass,nonlinearity=softmax)#lasagne.init.HeNormal(gain='relu'))#,W=Uniform(inial_scale) return net
def nn_fn(self): l_in_z = InputLayer((None, self.z_dim)) l_in_x = InputLayer((None, self.max_length, self.emb_dim)) l_in_z_reshape = ReshapeLayer(l_in_z, ( [0], [1], 1, )) l_in_z_rep = TileLayer(l_in_z_reshape, (1, 1, self.max_length)) l_x_pre_pad = SliceLayer(PadLayer(l_in_x, [(1, 0), (0, 0)], batch_ndim=1), indices=slice(0, -1), axis=1) l_x_pre_pad = DimshuffleLayer(l_x_pre_pad, (0, 2, 1)) l_x_pre_pad_drop = DropoutLayer(l_x_pre_pad, self.nn_word_drop, shared_axes=(1, )) l_concat = ConcatLayer((l_in_z_rep, l_x_pre_pad_drop), axis=1) l_in_d = Conv1DLayer(l_concat, num_filters=self.nn_channels_external, pad='same', filter_size=1, nonlinearity=None) for d in self.nn_dilations: l_cnn1 = Conv1DLayer(l_in_d, filter_size=1, num_filters=self.nn_channels_internal) l_dcnn = DilatedConv1DLayer(l_cnn1, filter_size=self.nn_filter_size, num_filters=self.nn_channels_internal, dilation=d) l_cnn2 = Conv1DLayer(l_dcnn, filter_size=1, num_filters=self.nn_channels_external) l_in_d = ElemwiseSumLayer([l_in_d, l_cnn2]) l_final = Conv1DLayer(l_in_d, filter_size=1, num_filters=self.emb_dim, nonlinearity=None) l_out = DimshuffleLayer(l_final, (0, 2, 1)) return (l_in_z, l_in_x), l_out
def q_network(state): input_state = InputLayer(input_var=state, shape=(None, self.state_dimension[0], self.state_dimension[1], self.state_dimension[2])) input_state = DimshuffleLayer(input_state, pattern=(0, 3, 1, 2)) conv = Conv2DLayer(input_state, num_filters=32, filter_size=(8, 8), stride=(4, 4), nonlinearity=rectify) conv = Conv2DLayer(conv, num_filters=64, filter_size=(4, 4), stride=(2, 2), nonlinearity=rectify) conv = Conv2DLayer(conv, num_filters=64, filter_size=(3, 3), stride=(1, 1), nonlinearity=rectify) flatten = FlattenLayer(conv) dense = DenseLayer(flatten, num_units=512, nonlinearity=rectify) q_values = DenseLayer(dense, num_units=self.action_dimension, nonlinearity=linear) return q_values
def smooth_convolution(prediction, n_classes): from lasagne.layers import Conv1DLayer as ConvLayer from lasagne.layers import DimshuffleLayer, ReshapeLayer prediction = ReshapeLayer(prediction, (-1, 200, n_classes)) # channels first prediction = DimshuffleLayer(prediction, (0, 2, 1)) input_size = lasagne.layers.get_output(prediction).shape # reshape to put each channel in the batch dimensions, to filter each # channel independently prediction = ReshapeLayer(prediction, (T.prod(input_size[0:2]), 1, input_size[2])) trans_filter = np.tile(np.array([0, -1., 1.]).astype('float32'), (1, 1, 1)) convolved = ConvLayer(prediction, num_filters=1, filter_size=3, stride=1, b=None, nonlinearity=None, W=trans_filter, pad='same') # reshape back convolved = ReshapeLayer(convolved, input_size) return convolved
def _create_decoder(self, decoder_input, decoder_input_hid, max_len, n_features_context, resetgate=None, updategate=None, hiddengate=None): #### Decoder (forwards) # decoder_input_orig keeps the value of l_encoder, which is given to this method decoder_input_orig = decoder_input # decoder_input then becomes a repeated l_encoder for _ in np.arange(max_len - 1): decoder_input = ConcatLayer([decoder_input, decoder_input_orig]) print(f"- Decoder input before reshape. shape: {lasagne.layers.get_output_shape(decoder_input)}") decoder_input = ReshapeLayer(decoder_input, (self.batch_size, n_features_context, max_len), name="reshape_enc_dec") print(f"- Decoder input after reshape. shape: {lasagne.layers.get_output_shape(decoder_input)}") decoder_input = DimshuffleLayer(decoder_input, (0, 2, 1), name="dimshuf_enc_dec") # Use standard gates, if no weight sharing gates are supplied if resetgate is None: resetgate = lasagne.layers.Gate(W_cell=None) if updategate is None: updategate = lasagne.layers.Gate(W_cell=None) if hiddengate is None: hiddengate = lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh) decoder_output = gated_layer(decoder_input, self.n_hidden, grad_clipping=self.grad_clip, only_return_final=False, backwards=False, cell_init=self.init, hid_init=decoder_input_hid, gated_layer_type=self.gated_layer_type, resetgate=resetgate, updategate=updategate, hidden_update=hiddengate, name="dec") if self.dropout > 0.0: decoder_output = DropoutLayer(decoder_output, p=self.dropout, rescale=False) l_decoder = decoder_output return l_decoder
def get_model(input_var, target_var, multiply_var): # input layer with unspecified batch size layer = InputLayer(shape=(None, 30, 64, 64), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var) layer = DimshuffleLayer(layer, (0, 'x', 1, 2, 3)) # Z-score? # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=1, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer_prediction = layer # Loss prediction = get_output(layer_prediction) loss = categorical_crossentropy(prediction.flatten(), target_var.flatten()) #Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum params = get_all_params(layer_prediction, trainable=True) # Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, disabling dropout layers. test_prediction = get_output(layer_prediction, deterministic=True) test_loss = categorical_crossentropy(test_prediction.flatten(), target_var.flatten()) return test_prediction, prediction, loss, params
def build_convpool_conv1d(input_vars, nb_classes, imsize=32, n_colors=3, n_timewin=3): """ Builds the complete network with 1D-conv layer to integrate time from sequences of EEG images. :param input_vars: list of EEG images (one image per time window) :param nb_classes: number of classes :param imsize: size of the input image (assumes a square input) :param n_colors: number of color channels in the image :param n_timewin: number of time windows in the snippet :return: a pointer to the output of last layer """ convnets = [] w_init = None # Build 7 parallel CNNs with shared weights for i in range(n_timewin): if i == 0: convnet, w_init = build_cnn(input_vars[i], imsize=imsize, n_colors=n_colors) else: convnet, _ = build_cnn(input_vars[i], w_init=w_init, imsize=imsize, n_colors=n_colors) convnets.append(FlattenLayer(convnet)) # at this point convnets shape is [numTimeWin][n_samples, features] # we want the shape to be [n_samples, features, numTimeWin] convpool = ConcatLayer(convnets) convpool = ReshapeLayer(convpool, ([0], n_timewin, get_output_shape(convnets[0])[1])) convpool = DimshuffleLayer(convpool, (0, 2, 1)) # input to 1D convlayer should be in (batch_size, num_input_channels, input_length) convpool = Conv1DLayer(convpool, 64, 3) # A fully-connected layer of 512 units with 50% dropout on its inputs: convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5), num_units=512, nonlinearity=lasagne.nonlinearities.rectify) # And, finally, the output layer with 50% dropout on its inputs: convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5), num_units=nb_classes, nonlinearity=lasagne.nonlinearities.softmax) return convpool
def createConvtScaleSpaceLayer(input_layer_4d, resize_ratio_list, name=None): input_layer_shape = get_output_shape(input_layer_4d) batch_size = input_layer_shape[0] # patch_size = input_layer_shape[2] # orig_c = (float(input_layer_shape[2]) - 1.0) * 0.5 scale_space_layer_list = [] for resize_ratio in resize_ratio_list: rf = resize_ratio c = 0 # the implementation already works on 0-center coordinate system rescaleA = np.tile( np.asarray([[rf, 0], [0, rf], [c, c]], dtype=floatX).T.reshape([1, 2, 3]), [batch_size, 1, 1]).reshape([-1, 6]) param_layer = InputLayer((batch_size, 6), input_var=theano.shared(rescaleA)) resize_layer = TransformerLayer(input_layer_4d, param_layer, input_layer_shape[2], input_layer_shape[3]) scale_space_layer_list += [ # ReshapeLayer(resize_layer, # tuple([v for v in input_layer_shape] + [1])) DimshuffleLayer(resize_layer, (0, 1, 2, 3, 'x')) ] scale_space_layer = ConcatLayer(scale_space_layer_list, axis=4, name=name) return scale_space_layer
def build_res_rnn_network(rnnmodel): net = {} net['input'] = InputLayer((batch_size, seq_len, feature_size)) net['rnn0']=DimshuffleLayer(net['input'],(1,0,2)) for l in range(1, num_layers+1): hidini=0 if l==num_layers: hidini=U_lowbound net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (batch_size* seq_len, -1)) net['rnn%d'%(l-1)]=DenseLayer(net['rnn%d'%(l-1)],hidden_units,W=ini_W,b=Uniform(range=(0,args.ini_b)),nonlinearity=None) #W=Uniform(ini_rernn_in_to_hid), # net['rnn%d'%(l-1)]=ReshapeLayer(net['rnn%d'%(l-1)], (seq_len, batch_size, -1)) net['rnn%d'%l]=net['rnn%d'%(l-1)] if not args.use_bn_afterrnn: net['rnn%d'%l]=BatchNormLayer(net['rnn%d'%l],axes= (0,1),beta=Uniform(range=(0,args.ini_b))) net['rnn%d'%l]=rnnmodel(net['rnn%d'%l],hidden_units,W_hid_to_hid=Uniform(range=(hidini,U_bound)),nonlinearity=act,only_return_final=False, grad_clipping=args.gradclipvalue) if args.use_bn_afterrnn: net['rnn%d'%l]=BatchNormLayer(net['rnn%d'%l],axes= (0,1)) if l==num_layers: net['rnn%d'%num_layers]=lasagne.layers.SliceLayer(net['rnn%d'%num_layers],indices=-1, axis=0) net['out']=DenseLayer(net['rnn%d'%num_layers],outputclass,nonlinearity=softmax) return net
def build_convpool_mix(input_vars, nb_classes, grad_clip=110, imsize=32, n_colors=3, n_timewin=7): """ Builds the complete network with LSTM and 1D-conv layers combined :param input_vars: list of EEG images (one image per time window) :param nb_classes: number of classes :param grad_clip: the gradient messages are clipped to the given value during the backward pass. :param imsize: size of the input image (assumes a square input) :param n_colors: number of color channels in the image :param n_timewin: number of time windows in the snippet :return: a pointer to the output of last layer """ convnets = [] w_init = None # Build 7 parallel CNNs with shared weights for i in range(n_timewin): if i == 0: convnet, w_init = build_cnn(input_vars[i], imsize=imsize, n_colors=n_colors) else: convnet, _ = build_cnn(input_vars[i], w_init=w_init, imsize=imsize, n_colors=n_colors) convnets.append(FlattenLayer(convnet)) # at this point convnets shape is [numTimeWin][n_samples, features] # we want the shape to be [n_samples, features, numTimeWin] convpool = ConcatLayer(convnets) convpool = ReshapeLayer(convpool, ([0], n_timewin, get_output_shape(convnets[0])[1])) reformConvpool = DimshuffleLayer(convpool, (0, 2, 1)) # input to 1D convlayer should be in (batch_size, num_input_channels, input_length) conv_out = Conv1DLayer(reformConvpool, 64, 3) conv_out = FlattenLayer(conv_out) # Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features) lstm = LSTMLayer(convpool, num_units=128, grad_clipping=grad_clip, nonlinearity=lasagne.nonlinearities.tanh) lstm_out = SliceLayer(lstm, -1, 1) # Merge 1D-Conv and LSTM outputs dense_input = ConcatLayer([conv_out, lstm_out]) # A fully-connected layer of 256 units with 50% dropout on its inputs: convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5), num_units=512, nonlinearity=lasagne.nonlinearities.rectify) # And, finally, the 10-unit output layer with 50% dropout on its inputs: convpool = DenseLayer(convpool, num_units=nb_classes, nonlinearity=lasagne.nonlinearities.softmax) return convpool
def _combine_encoder_steps(self, encoder): #print(lasagne.layers.get_output_shape(encoder)) shuffled_encoder = DimshuffleLayer(encoder, (0, 2, 1)) #print(lasagne.layers.get_output_shape(shuffled_encoder)) encoder = lasagne.layers.DenseLayer(shuffled_encoder, num_units=1, num_leading_axes=2) #print(lasagne.layers.get_output_shape(encoder)) encoder = ReshapeLayer(encoder, shape=(self.batch_size, self.n_hidden)) #print(lasagne.layers.get_output_shape(encoder)) return encoder
def construct_tiramisu(channels=1, no_f_base=45, f_size_base=3, bs=None, class_nums=2, k=16, denseblocks=[4,5,7,10,12], blockbottom=[15], dropout=0, input_var=None, pad="same",c_nonlinearity=lasagne.nonlinearities.rectify, f_nonlinearity=lasagne.nonlinearities.rectify, input_dim=[112,112]): #Network start net = InputLayer((bs, channels, input_dim[0], input_dim[1]), input_var) net = Conv2DLayer(net, no_f_base, f_size_base, pad=pad, W=lasagne.init.HeUniform(gain='relu'), flip_filters=False) #Downward block building horizontal_pass=[] for blocks in xrange(len(denseblocks)): net_preblock = net net = tiramisu_denseblock(net, denseblocks[blocks], k, drop_p=dropout) net = ConcatLayer([net_preblock, net], axis=1) #Connection around Denseblock print "Input shape: {}, after concat: {}".format(net_preblock.output_shape,net.output_shape) horizontal_pass.append(net) net = tiramisu_transistion_down(net, drop_p=dropout) print "After transition down: ",net.output_shape print "---Down done---" #Bottom dense block for bottom in xrange(len(blockbottom)): net = tiramisu_denseblock(net, blockbottom[bottom], k, drop_p=dropout) if bottom < len(blockbottom)-1: net = tiramisu_transition_bottom(net, drop_p=dropout) print "Bottom: ",net.output_shape print "---Bottom done---" #Up dense block for block in xrange(len(denseblocks)): print "Before concat size: ",net.output_shape net = tiramisu_transistion_up(net, f_size_base, 2) net = ConcatLayer([net, horizontal_pass[-(block+1)]], axis=1,cropping=[None, None, "center", "center"]) print "After concat size: ",net.output_shape net = tiramisu_denseblock(net, denseblocks[-(block+1)], k, drop_p=dropout) print "Denseblock size: ",net.output_shape print "---Up done---" #Out block image_out = net = Conv2DLayer(net, class_nums, 1, pad=pad, W=lasagne.init.HeUniform(gain='relu'), nonlinearity=linear, flip_filters=False) net = DimshuffleLayer(net,(1,0,2,3)) net = ReshapeLayer(net, (class_nums,-1)) net = DimshuffleLayer(net, (1,0)) net = NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax) return net,image_out net = None image_out=None
def sliding_window_input(input_layer): window_size = 5 sub_input = [] for i in xrange(window_size): indices = slice(window_size - i - 1, -i if i > 0 else None) network = DimshuffleLayer(SliceLayer(input_layer, indices, axis=-1), (0, 1, 'x')) sub_input.append(network) network = ConcatLayer(sub_input, -1) return network
def build_lstm(input_vars, input_shape=None): ''' 1) InputLayer 2) ReshapeLayer 3) LSTM Layer 1 4) LSTM Layer 2 5) Slice Layer 6) Fully Connected Layer 1 w/ dropout tanh 7) Fully Connected Layer 2 w/ dropout softmax ''' # Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features) network = InputLayer(shape=(input_shape[0], None, num_input_channels, input_shape[-3], input_shape[-2], input_shape[-1]), input_var=input_vars) network = ReshapeLayer(network, ([0], [1], -1)) network = DimshuffleLayer(network, (1, 0, 2)) #network = ReshapeLayer(network, (-1, 128)) #l_inp = InputLayer((None, None, num_inputs)) l_lstm1 = LSTMLayer(network, num_units=128, grad_clipping=grad_clip, nonlinearity=lasagne.nonlinearities.tanh) #New LSTM l_lstm2 = LSTMLayer(l_lstm1, num_units=128, grad_clipping=grad_clip, nonlinearity=lasagne.nonlinearities.tanh) #end of insertion # After LSTM layer you either need to reshape or slice it (depending on whether you # want to keep all predictions or just the last prediction. # http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html # https://github.com/Lasagne/Recipes/blob/master/examples/lstm_text_generation.py l_lstm_slice = SliceLayer(l_lstm2, -1, 1) # Selecting the last prediction # A fully-connected layer of 256 units with 50% dropout on its inputs: l_dense = DenseLayer(lasagne.layers.dropout(l_lstm_slice, p=.5), num_units=256, nonlinearity=lasagne.nonlinearities.rectify) # We only need the final prediction, we isolate that quantity and feed it # to the next layer. # And, finally, the output layer with 50% dropout on its inputs: l_dense = DenseLayer(lasagne.layers.dropout(l_dense, p=.5), num_units=num_classes, nonlinearity=lasagne.nonlinearities.softmax) return l_dense
def build_indrnn_network(X_sym): net = {} net['input0'] = InputLayer((batch_size, seq_len, indim, 3), X_sym) net['input'] = ReshapeLayer(net['input0'], (batch_size, seq_len, indim * 3)) net['rnn0'] = DimshuffleLayer(net['input'], (1, 0, 2)) for l in range(1, num_layers + 1): hidini = 0 if l == num_layers: hidini = U_lowbound net['rnn%d' % (l - 1)] = ReshapeLayer(net['rnn%d' % (l - 1)], (batch_size * seq_len, -1)) net['rnn%d' % (l - 1)] = DenseLayer(net['rnn%d' % (l - 1)], hidden_units, W=ini_W, b=lasagne.init.Constant( args.ini_b), nonlinearity=None) # net['rnn%d' % (l - 1)] = ReshapeLayer(net['rnn%d' % (l - 1)], (seq_len, batch_size, -1)) if args.conv_drop: net['rnn%d' % (l - 1)] = DropoutLayer(net['rnn%d' % (l - 1)], p=droprate, shared_axes=(0, )) net['rnn%d' % l] = net['rnn%d' % (l - 1)] if not args.use_bn_afterrnn: net['rnn%d' % l] = BatchNormLayer(net['rnn%d' % l], beta=lasagne.init.Constant( args.ini_b), axes=(0, 1)) net['rnn%d' % l] = rnnmodel(net['rnn%d' % l], hidden_units, W_hid_to_hid=Uniform(range=(hidini, U_bound)), nonlinearity=act, only_return_final=False, grad_clipping=gradclipvalue) if args.use_bn_afterrnn: net['rnn%d' % l] = BatchNormLayer(net['rnn%d' % l], axes=(0, 1)) if args.use_dropout and l % args.drop_layers == 0: net['rnn%d' % l] = DropoutLayer(net['rnn%d' % l], p=droprate, shared_axes=(0, )) net['rnn%d' % num_layers] = lasagne.layers.SliceLayer(net['rnn%d' % num_layers], indices=-1, axis=0) net['out'] = DenseLayer(net['rnn%d' % num_layers], outputclass, nonlinearity=softmax) return net
def build_cnn(input_layer): # Add a channel axis for convolutional nets network = DimshuffleLayer(input_layer, (0, 'x', 1)) network = Conv1DLayer(network, num_filters=4, filter_size=5, nonlinearity=rectify) network = MaxPool1DLayer(network, pool_size=2) network = Conv1DLayer(network, num_filters=4, filter_size=5, nonlinearity=rectify) network = MaxPool1DLayer(network, pool_size=2) network = DropoutLayer(network, p=.5) network = DenseLayer(network, num_units=256, nonlinearity=rectify) return network
def build_convpool_mix(input_vars, input_shape=None): """ Builds the complete network with LSTM and 1D-conv layers combined to integrate time from sequences of EEG images. :param input_vars: list of EEG images (one image per time window) :return: a pointer to the output of last layer """ convnets = [] W_init = None # Build 7 parallel CNNs with shared weights for i in range(input_shape[0]): if i == 0: convnet, W_init = build_cnn(input_vars[i], input_shape) else: convnet, _ = build_cnn(input_vars[i], input_shape, W_init) convnets.append(FlattenLayer(convnet)) # at this point convnets shape is [numTimeWin][n_samples, features] # we want the shape to be [n_samples, features, numTimeWin] convpool = ConcatLayer(convnets) # convpool = ReshapeLayer(convpool, ([0], -1, numTimeWin)) convpool = ReshapeLayer( convpool, ([0], input_shape[0], get_output_shape(convnets[0])[1])) reformConvpool = DimshuffleLayer(convpool, (0, 2, 1)) # input to 1D convlayer should be in (batch_size, num_input_channels, input_length) conv_out = Conv1DLayer(reformConvpool, 64, 3) conv_out = FlattenLayer(conv_out) # Input to LSTM should have the shape as (batch size, SEQ_LENGTH, num_features) lstm = LSTMLayer(convpool, num_units=128, grad_clipping=grad_clip, nonlinearity=lasagne.nonlinearities.tanh) # After LSTM layer you either need to reshape or slice it (depending on whether you # want to keep all predictions or just the last prediction. # http://lasagne.readthedocs.org/en/latest/modules/layers/recurrent.html # https://github.com/Lasagne/Recipes/blob/master/examples/lstm_text_generation.py lstm_out = SliceLayer(lstm, -1, 1) # Merge 1D-Conv and LSTM outputs dense_input = ConcatLayer([conv_out, lstm_out]) # A fully-connected layer of 256 units with 50% dropout on its inputs: convpool = DenseLayer(lasagne.layers.dropout(dense_input, p=.5), num_units=512, nonlinearity=lasagne.nonlinearities.rectify) # We only need the final prediction, we isolate that quantity and feed it # to the next layer. # And, finally, the 10-unit output layer with 50% dropout on its inputs: convpool = DenseLayer(convpool, num_units=num_classes, nonlinearity=lasagne.nonlinearities.softmax) return convpool
def build_model(self): # reshape to [batch, color, x, y] to allow for convolution layers to work correctly observation_reshape = DimshuffleLayer(self.observation_layer, (0, 3, 1, 2)) observation_reshape = Pool2DLayer(observation_reshape, pool_size=(2, 2)) # memory window_size = 5 # prev state input prev_window = InputLayer( (None, window_size) + tuple(observation_reshape.output_shape[1:]), name="previous window state") # our window memory_layer = WindowAugmentation(observation_reshape, prev_window, name="new window state") memory_dict = {memory_layer: prev_window} # pixel-wise maximum over the temporal window (to avoid flickering) memory_layer = ExpressionLayer(memory_layer, lambda a: a.max(axis=1), output_shape=(None, ) + memory_layer.output_shape[2:]) # neural network body nn = batch_norm( lasagne.layers.Conv2DLayer(memory_layer, num_filters=16, filter_size=(8, 8), stride=(4, 4))) nn = batch_norm( lasagne.layers.Conv2DLayer(nn, num_filters=32, filter_size=(4, 4), stride=(2, 2))) nn = batch_norm(lasagne.layers.DenseLayer(nn, num_units=256)) # q_eval policy_layer = DenseLayer(nn, num_units=self.n_actions, nonlinearity=lasagne.nonlinearities.linear, name="QEvaluator") # resolver resolver = EpsilonGreedyResolver(policy_layer, name="resolver") # all together agent = Agent(self.observation_layer, memory_dict, policy_layer, resolver) return resolver, agent
def build_lstm(input_layer): #network = sliding_window_input(input_layer) network = DimshuffleLayer(input_layer, (0, 1, 'x')) n_hidden = 50 grad_clipping = 20 network = LSTMLayer(network, num_units=n_hidden, grad_clipping=grad_clipping, nonlinearity=tanh) network = LSTMLayer(network, num_units=n_hidden, grad_clipping=grad_clipping, nonlinearity=tanh) network = SliceLayer(network, indices=-1, axis=1) #network = DenseLayer(network, num_units=256, nonlinearity=rectify) return network
def construct_unet_recursive(channels=1, no_f_base=8, f_size=3, branches=[2,2,2,2],dropout=0.2,bs=None, class_nums=2, pad="same",nonlinearity=lasagne.nonlinearities.rectify, input_dim=[512,512]): net= InputLayer((bs, channels, input_dim[0], input_dim[1])) # Moving downwards the U-shape: horizontal_pass=[] for i in xrange(len(branches)): net = conv_pool_down(net,no_f_base*2**(i),f_size,conv_depth=branches[i], pad=pad,nonlinearity=nonlinearity,dropout=dropout) horizontal_pass.append(net) net = Pool2DLayer(net,pool_size=2) # Bottleneck net = Conv2DLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu')) net = Conv2DLayer(net,no_f_base*2**len(branches),f_size,pad=pad,nonlinearity=nonlinearity,W=lasagne.init.HeNormal(gain='relu')) net = Deconv2DLayer(net, no_f_base*2**(len(branches)-1), 2, 2) # Moving upwards the U-shape: for i in xrange(len(branches)): net = PadLayer(net,1) net = ConcatLayer([net,horizontal_pass[-(i+1)]],cropping=(None,None,"center","center")) if i==len(branches)-1: net = conv_pool_up(net,no_f_base*2**(len(branches)-1-i),f_size, pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=True) else: net = conv_pool_up(net,no_f_base*2**(len(branches)-1-i),f_size, pad=pad,nonlinearity=nonlinearity,conv_depth=branches[i],halt=False) # Class layer: Work around standard softmax bc. it doesn't work with tensor4/3. # Hence, we reshape and feed it to an external Nonlinearity layer. # net["class_ns"] is the output in image-related shape. imageout = net = Conv2DLayer(net, class_nums, 1, nonlinearity=linear,W=lasagne.init.HeNormal(gain='relu')) net = DimshuffleLayer(net, (1, 0, 2, 3)) net = ReshapeLayer(net, (class_nums, -1)) net = DimshuffleLayer(net, (1, 0)) # Flattened output to be able to feed it to lasagne.objectives.categorical_crossentropy. net = NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax) return net,imageout del net, imageout
def __init__(self, input_shape=(None, 1, 33, 33, 33)): self.cubeSize = input_shape[-1] # Theano variables self.input_var = T.tensor5('input_var') # input image self.target_var = T.ivector('target_var') # target self.logger = logging.getLogger(__name__) input_layer = InputLayer(input_shape, self.input_var) self.logger.info('The shape of input layer is {}'.format( get_output_shape(input_layer))) hidden_layer1 = Conv3DLayer(incoming=input_layer, num_filters=16, filter_size=(3, 3, 3), W=HeUniform(gain='relu'), nonlinearity=rectify) self.logger.info('The shape of first hidden layer is {}'.format( get_output_shape(hidden_layer1))) hidden_layer2 = Conv3DLayer(incoming=hidden_layer1, num_filters=32, filter_size=(3, 3, 3), W=HeUniform(gain='relu'), nonlinearity=rectify) self.logger.info('The shape of second hidden layer is {}'.format( get_output_shape(hidden_layer2))) hidden_layer3 = Conv3DLayer(incoming=hidden_layer2, num_filters=2, filter_size=(1, 1, 1), W=HeUniform(gain='relu'), nonlinearity=rectify) self.logger.info('The shape of third hidden layer is {}'.format( get_output_shape(hidden_layer3))) shuffledLayer = DimshuffleLayer(hidden_layer3, (0, 2, 3, 4, 1)) self.logger.info('The shape of shuffled layer is {}'.format( get_output_shape(shuffledLayer))) reshapedLayer = ReshapeLayer(shuffledLayer, ([0], -1)) self.logger.info('The shape of reshaped layer is {}'.format( get_output_shape(reshapedLayer))) self.output_layer = NonlinearityLayer(reshapedLayer, softmax) self.logger.info('The shape of output layer is {}'.format( get_output_shape(self.output_layer)))