def initNetwork2(self, input_dim, F1, F2, num_classes): dropout = self.dropout C,H,W = input_dim batch_norm_dropout = lambda x:lasagne.layers.DropoutLayer(batch_norm(x), p=dropout) net = batch_norm(lasagne.layers.InputLayer(shape=(None, C, H, W), input_var=self.xvar)) self.penaltyL1 = lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2)*0 if True: net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F1, filter_size=7, stride=3, pad=0, W=self.Wnorm())) self.penaltyL1 = lasagne.regularization.regularize_network_params(net, lasagne.regularization.l2)*self.regL1 net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=5, stride=1, pad=0, W=self.Wnorm())) elif False: net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F1, filter_size=7, stride=3, pad=0, W=self.Wnorm())) net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=1, pad=0, W=self.Wnorm())) net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=1, pad=0, W=self.Wnorm())) elif False: net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F1, filter_size=3, stride=3, pad=1, W=self.Wnorm())) for _ in xrange(3): net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=1, pad=0, W=self.Wnorm())) else: #net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F1, filter_size=3, stride=1, pad=0, W=self.Wnorm())) #net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=1, pad=0, W=self.Wnorm())) net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F1, filter_size=5, stride=1, pad=0, W=self.Wnorm())) net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=3, pad=0, W=self.Wnorm())) net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=1, pad=0, W=self.Wnorm())) net = batch_norm_dropout(lasagne.layers.Conv2DLayer(net, num_filters=F2, filter_size=3, stride=1, pad=0, W=self.Wnorm())) net = lasagne.layers.MaxPool2DLayer(net, pool_size=2, stride=2, pad=0) net = lasagne.layers.Conv2DLayer(net, num_filters=num_classes, filter_size=1, stride=1, pad=0, nonlinearity=None, W=self.Wnorm()) net = lasagne.layers.DimshuffleLayer(net, (0,2,3,1)) net = lasagne.layers.ReshapeLayer(net, (-1, num_classes)) net = lasagne.layers.NonlinearityLayer(net, nonlinearity=lasagne.nonlinearities.softmax) return net
def build_network(self, input_var = None, batch_size = None): print "build_network in VideoClassifier executed.." print "inputs are : " , self.sinputs if not input_var is None: self.sinputs = input_var if not batch_size is None: self.batch_size = batch_size # Merge or fuse or concatenate incoming layers self.network['ConcatLayer'] = lasagne.layers.ConcatLayer([self.right_network['FC_2'], self.left_network['FC_2']], axis=1, cropping=None) self.network['FC_3'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['ConcatLayer'], p=self.dropout_rates[0]), num_units=84, nonlinearity=lasagne.nonlinearities.tanh)) self.network['prob'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['FC_3'], p=self.dropout_rates[2]), num_units=self.fc_layers[2], nonlinearity=lasagne.nonlinearities.softmax)) return self.network
def lasagne_model(): l_in = InputLayer(shape=(None, 1)+speechSize) l_conv1 = Conv2DLayer(l_in, num_filters = 128, filter_size=(3,3), nonlinearity=rectify) l_conv1b = Conv2DLayer(l_conv1, num_filters = 128, filter_size=(3,3), nonlinearity=rectify) l_conv1b = batch_norm(l_conv1b) l_pool1 = MaxPool2DLayer(l_conv1b, pool_size=(2,2)) # l_pool1 = DropoutLayer(l_pool1, p=0.2) l_conv2 = Conv2DLayer(l_pool1, num_filters = 256, filter_size=(3,3), nonlinearity=rectify) l_conv2b = Conv2DLayer(l_conv2, num_filters = 256, filter_size=(3,3), nonlinearity=rectify) l_conv2b = batch_norm(l_conv2b) l_pool2 = MaxPool2DLayer(l_conv2b, pool_size=(2,2)) # l_pool2 = DropoutLayer(l_pool2, p=0.2) # TODO:tanh这个要研究啊! l_hidden3 = DenseLayer(l_pool2, num_units = h_dimension, nonlinearity=tanh) l_hidden3 = batch_norm(l_hidden3) # l_hidden3 = DropoutLayer(l_hidden3, p=0.3) l_hidden4 = DenseLayer(l_hidden3, num_units = h_dimension, nonlinearity=tanh) # l_hidden4 = DropoutLayer(l_hidden4, p=0.5) l_out = DenseLayer(l_hidden4, num_units=num_labels_train, nonlinearity=softmax) return l_out,l_hidden4
def build_network(self, input_var=None, batch_size = None): print "build_network() in SkeletonClassifier invoked" print self.sinputs if not input_var is None: self.sinputs = input_var if batch_size: self.batch_size = batch_size if not input_var is None: self.sinputs = input_var if not batch_size is None: self.batch_size = batch_size self.network['input'] = lasagne.layers.InputLayer(shape=(self.batch_size,self.nframes,1,self.dlength), input_var=self.sinputs[0]) self.network['FC_1'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['input'], p=self.dropout_rates[1]), num_units=self.fc_layers[0],nonlinearity=lasagne.nonlinearities.tanh)) self.network['FC_2'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['FC_1'], p=self.dropout_rates[2]), num_units=self.fc_layers[1], nonlinearity=lasagne.nonlinearities.tanh)) self.network['FC_3'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['FC_2'], p=self.dropout_rates[3]), num_units=self.fc_layers[2], nonlinearity=lasagne.nonlinearities.tanh)) self.network['prob'] = lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['FC_3'], p=.2), num_units=self.nclasses, nonlinearity=lasagne.nonlinearities.softmax) return self.network
def test_batch_norm_macro(): from lasagne.layers import (Layer, BatchNormLayer, batch_norm, NonlinearityLayer) from lasagne.nonlinearities import identity input_shape = (2, 3) obj = object() # check if it steals the nonlinearity layer = Mock(Layer, output_shape=input_shape, nonlinearity=obj) bnstack = batch_norm(layer) assert isinstance(bnstack, NonlinearityLayer) assert isinstance(bnstack.input_layer, BatchNormLayer) assert layer.nonlinearity is identity assert bnstack.nonlinearity is obj # check if it removes the bias layer = Mock(Layer, output_shape=input_shape, b=obj, params={obj: set()}) bnstack = batch_norm(layer) assert isinstance(bnstack, BatchNormLayer) assert layer.b is None assert obj not in layer.params # check if it can handle an unset bias layer = Mock(Layer, output_shape=input_shape, b=None, params={obj: set()}) bnstack = batch_norm(layer) assert isinstance(bnstack, BatchNormLayer) assert layer.b is None # check if it passes on kwargs layer = Mock(Layer, output_shape=input_shape) bnstack = batch_norm(layer, name='foo') assert isinstance(bnstack, BatchNormLayer) assert bnstack.name == 'foo'
def build_contract_level(incoming, num_filters, nonlin, W_init=lasagne.init.GlorotUniform(), b_init=lasagne.init.Constant(0.01), filter_size=3): """Builds a Conv-Conv-Pool block of the U-Net encoder.""" network = nn.Conv2DLayer(incoming, num_filters, filter_size, pad='same', W=W_init, b=b_init, nonlinearity=nonlin) network = nn.batch_norm(network) network = nn.Conv2DLayer(network, num_filters, filter_size, pad='same', W=W_init, b=b_init, nonlinearity=nonlin) network = nn.batch_norm(network) return network, nn.MaxPool2DLayer(network, 2)
def pons_cnn(params): """""" layers = L.InputLayer((None, 1, params['dur'], 128)) print layers.output_shape sclr = joblib.load(params['scaler']) layers = L.standardize(layers, sclr.mean_.astype(np.float32), sclr.scale_.astype(np.float32), shared_axes=(0, 1, 2)) print layers.output_shape layers_timbre = L.GlobalPoolLayer( L.batch_norm(L.Conv2DLayer(layers, 64, (1, 96)))) layers_rhythm = L.GlobalPoolLayer( L.batch_norm(L.Conv2DLayer(layers, 64, (params['dur'] - 10, 1)))) layers = L.ConcatLayer([layers_rhythm, layers_timbre], axis=-1) layers = L.DenseLayer(layers, 64, nonlinearity=nl.rectify) print layers.output_shape layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax) print layers.output_shape return layers
def createDiscriminator2(input_var=None): _ = InputLayer(shape=(None, 3, 64, 64), input_var=input_var) _ = batch_norm(Conv2DDNNLayer(_, 64, 3, pad='same')) _ = batch_norm(Conv2DDNNLayer(_, 64, 3, pad='same')) _ = MaxPool2DDNNLayer(_, 2) _ = batch_norm(Conv2DDNNLayer(_, 64, 3, pad='same')) _ = MaxPool2DDNNLayer(_, 2) _ = batch_norm(Conv2DDNNLayer(_, 128, 3, pad='same')) _ = batch_norm(Conv2DDNNLayer(_, 128, 3, pad='same')) _ = FlattenLayer(_) _ = DenseLayer(_, num_units=1000, nonlinearity=lasagne.nonlinearities.rectify) l_discriminator = DenseLayer(_, num_units=1, nonlinearity=lasagne.nonlinearities.sigmoid) print('--------------------') print('Discriminator architecture: \n') #get all layers allLayers=lasagne.layers.get_all_layers(l_discriminator) #for each layer print its shape information for l in allLayers: print(lasagne.layers.get_output_shape(l)) print ("Discriminator output:", l_discriminator.output_shape) return l_discriminator
def normal(ilayer,fmaps,activation,t='enc',ltype='normal'): if t == 'enc': x = batch_norm(lasagne.layers.Conv2DLayer( ilayer, num_filters=fmaps[0],filter_size=(3,3), nonlinearity=None,pad=1, W=initf )) else: x = batch_norm(lasagne.layers.Conv2DLayer( ilayer, num_filters=fmaps[0],filter_size=(3,3), nonlinearity=activation,pad=1, W=initf )) if ltype == 'normal': x = batch_norm(lasagne.layers.Conv2DLayer( x, num_filters=fmaps[1],filter_size=(3,3), nonlinearity=activation,pad=1, W=initf )) elif ltype == 'residual': x = batch_norm(lasagne.layers.Conv2DLayer( x, num_filters=fmaps[1],filter_size=(3,3), nonlinearity=None,pad=1, W=initf )) y = lasagne.layers.Conv2DLayer( ilayer, num_filters=fmaps[1],filter_size=(1,1), nonlinearity=None,pad='same', W=initf) x = ElemwiseSumLayer([x, y]) x = NonlinearityLayer(x,nonlinearity=activation) return x
def build_discriminator(input_var=None, convs=0): from lasagne.layers import (InputLayer, DenseLayer, batch_norm) from lasagne.layers.dnn import Conv2DDNNLayer as Conv2DLayer # override from lasagne.nonlinearities import LeakyRectify, sigmoid lrelu = LeakyRectify(0.2) if convs == 0: # input: (None, 1, 64, 64) layer = InputLayer(shape=(None, 1, 64, 64), input_var=input_var) # two convolutions layer = batch_norm( Conv2DLayer(layer, 64, 5, stride=2, pad=2, nonlinearity=lrelu)) layer = batch_norm( Conv2DLayer(layer, 128, 5, stride=2, pad=2, nonlinearity=lrelu)) else: # input: (None, 1, 128, 128) layer = InputLayer(shape=(None, 1, 128, 128), input_var=input_var) # two convolutions layer = batch_norm( Conv2DLayer(layer, 64, 5, stride=2, pad=2, nonlinearity=lrelu)) layer = batch_norm( Conv2DLayer(layer, 128, 5, stride=2, pad=2, nonlinearity=lrelu)) # fully-connected layer layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu)) # output layer layer = DenseLayer(layer, 1, nonlinearity=sigmoid) print("Discriminator output:", layer.output_shape) return layer
def get_model(input_var, target_var, multiply_var): # input layer with unspecified batch size layer = InputLayer(shape=(None, 12, 64, 64), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var) layer = DimshuffleLayer(layer, (0, 'x', 1, 2, 3)) # Z-score? # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer = Conv3DDNNLayer(incoming=layer, num_filters=1, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=sigmoid) layer_prediction = layer # Loss prediction = get_output(layer_prediction) loss = binary_crossentropy(prediction[:,0,:,:,:], target_var).mean() #Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum params = get_all_params(layer_prediction, trainable=True) # Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, disabling dropout layers. test_prediction = get_output(layer_prediction, deterministic=True) test_loss = binary_crossentropy(test_prediction[:,0,:,:,:], target_var).mean() return test_prediction, prediction, loss, params
def discriminator_3D(input_var=None, num_units=512, seq_length=4): discriminator = [] lrelu = lasagne.nonlinearities.LeakyRectify(0.2) discriminator.append( ll.InputLayer(shape=(None, seq_length, 3, 80, 160), input_var=input_var)) # lasagne documentations requires shape : # (batch_size, num_input_channels, input_depth, input_rows, input_columns) # so we need to change dimension ordering discriminator.append(ll.DimshuffleLayer(discriminator[-1], (0, 2, 1, 3, 4))) discriminator.append( ll.Conv3DLayer(discriminator[-1], num_filters=num_units / 8, filter_size=5, stride=2, pad=2, nonlinearity=lrelu)) discriminator.append( ll.batch_norm( ll.Conv3DLayer(discriminator[-1], num_filters=num_units / 4, filter_size=5, stride=2, pad=2, nonlinearity=lrelu))) discriminator.append( ll.batch_norm( ll.Conv3DLayer(discriminator[-1], num_filters=num_units / 2, filter_size=5, stride=2, pad=2, nonlinearity=lrelu))) discriminator.append( ll.batch_norm( ll.Conv3DLayer(discriminator[-1], num_filters=num_units, filter_size=5, stride=2, pad=2, nonlinearity=lrelu))) discriminator.append(ll.FlattenLayer(discriminator[-1])) discriminator.append( ll.DenseLayer(discriminator[-1], num_units=1, nonlinearity=None)) for layer in discriminator: print layer.output_shape print "" return discriminator
def build_discriminator_32(image=None, ndf=128): lrelu = LeakyRectify(0.2) # input: images InputImg = InputLayer(shape=(None, 3, 32, 32), input_var=image) print("Dis Img_input:", InputImg.output_shape) # Conv Layer dis1 = Conv2DLayer(InputImg, ndf, (4, 4), (2, 2), pad=1, W=Normal(0.02), nonlinearity=lrelu) print("Dis conv1:", dis1.output_shape) # Conv Layer dis2 = batch_norm( Conv2DLayer(dis1, ndf * 2, (4, 4), (2, 2), pad=1, W=Normal(0.02), nonlinearity=lrelu)) print("Dis conv2:", dis2.output_shape) # Conv Layer dis3 = batch_norm( Conv2DLayer(dis2, ndf * 4, (4, 4), (2, 2), pad=1, W=Normal(0.02), nonlinearity=lrelu)) print("Dis conv3:", dis3.output_shape) # Conv Layer dis4 = DenseLayer(dis3, 1, W=Normal(0.02), nonlinearity=sigmoid) print("Dis output:", dis4.output_shape) return dis4
def build_model(nActions): net = OrderedDict() net['input'] = InputLayer((None, 4, 84, 84)) net['conv1'] = batch_norm( ConvLayer(net['input'], num_filters=32, filter_size=8, stride=4, pad='valid', nonlinearity=ReLU)) net['conv2'] = batch_norm( ConvLayer(net['conv1'], num_filters=64, filter_size=4, stride=2, pad='valid', nonlinearity=ReLU)) net['conv3'] = batch_norm( ConvLayer(net['conv2'], num_filters=64, filter_size=3, stride=1, pad='valid', nonlinearity=ReLU)) net['fc4'] = batch_norm( DenseLayer(net['conv3'], num_units=512, nonlinearity=ReLU)) net['fc5'] = DenseLayer(net['fc4'], num_units=nActions, nonlinearity=linear) return net
def build_nn(self, l_obs): dense_layer = batch_norm(lasagne.layers.DenseLayer(l_obs, num_units=64, nonlinearity=lasagne.nonlinearities.tanh)) dense_layer = DropoutLayer(dense_layer) output_layer = batch_norm(lasagne.layers.DenseLayer(dense_layer, num_units=self.y_test.shape[1], nonlinearity=lasagne.nonlinearities.sigmoid)) return output_layer
def residual_block(l, increase_dim=False, projection=False): input_num_filters = l.output_shape[1] if increase_dim: first_stride = (2,2) out_num_filters = input_num_filters*2 else: first_stride = (1,1) out_num_filters = input_num_filters stack_1 = batch_norm(ConvLayer(l, num_filters=out_num_filters, filter_size=(3,3), stride=first_stride, nonlinearity=rectify, pad='same', W=lasagne.init.HeNormal(gain='relu'), flip_filters=False)) stack_2 = batch_norm(ConvLayer(stack_1, num_filters=out_num_filters, filter_size=(3,3), stride=(1,1), nonlinearity=None, pad='same', W=lasagne.init.HeNormal(gain='relu'), flip_filters=False)) # add shortcut connections if increase_dim: if projection: # projection shortcut, as option B in paper projection = batch_norm(ConvLayer(l, num_filters=out_num_filters, filter_size=(1,1), stride=(2,2), nonlinearity=None, pad='same', b=None, flip_filters=False)) block = NonlinearityLayer(ElemwiseSumLayer([stack_2, projection]),nonlinearity=rectify) else: # identity shortcut, as option A in paper identity = ExpressionLayer(l, lambda X: X[:, :, ::2, ::2], lambda s: (s[0], s[1], s[2]//2, s[3]//2)) padding = PadLayer(identity, [out_num_filters//4,0,0], batch_ndim=1) block = NonlinearityLayer(ElemwiseSumLayer([stack_2, padding]),nonlinearity=rectify) else: block = NonlinearityLayer(ElemwiseSumLayer([stack_2, l]),nonlinearity=rectify) return block
def encoder(z_dim=100, input_var=None, num_units=512, vae=True): encoder = [] lrelu = lasagne.nonlinearities.LeakyRectify(0.2) encoder.append(ll.InputLayer(shape=(None, 3, 80, 160), input_var=input_var)) encoder.append( ll.Conv2DLayer(encoder[-1], num_filters=num_units / 8, filter_size=(5, 5), stride=2, pad=2, nonlinearity=lrelu)) encoder.append( ll.batch_norm( ll.Conv2DLayer(encoder[-1], num_filters=num_units / 4, filter_size=(5, 5), stride=2, pad=2, nonlinearity=lrelu))) encoder.append( ll.batch_norm( ll.Conv2DLayer(encoder[-1], num_filters=num_units / 2, filter_size=(5, 5), stride=2, pad=2, nonlinearity=lrelu))) encoder.append( ll.batch_norm( ll.Conv2DLayer(encoder[-1], num_filters=num_units, filter_size=(5, 5), stride=2, pad=2, nonlinearity=lrelu))) encoder.append(ll.FlattenLayer(encoder[-1])) if vae: enc_mu = ll.DenseLayer(encoder[-1], num_units=z_dim, nonlinearity=None) enc_logsigma = ll.DenseLayer(encoder[-1], num_units=z_dim, nonlinearity=None) l_z = GaussianSampleLayer(enc_mu, enc_logsigma, name='Z layer') encoder += [enc_mu, enc_logsigma, l_z] for layer in encoder: print layer.output_shape print "" return encoder
def get_model(input_var, target_var, multiply_var): # input layer with unspecified batch size layer = InputLayer(shape=(None, 30, 64, 64), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var) layer = DimshuffleLayer(layer, (0, 'x', 1, 2, 3)) # Z-score? # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=16, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer = batch_norm(Conv3DDNNLayer(incoming=layer, num_filters=1, filter_size=(3,3,3), stride=(1,1,1), pad='same', nonlinearity=rectify)) layer_prediction = layer # Loss prediction = get_output(layer_prediction) loss = categorical_crossentropy(prediction.flatten(), target_var.flatten()) #Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum params = get_all_params(layer_prediction, trainable=True) # Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, disabling dropout layers. test_prediction = get_output(layer_prediction, deterministic=True) test_loss = categorical_crossentropy(test_prediction.flatten(), target_var.flatten()) return test_prediction, prediction, loss, params
def build_cnn(config, use_noise=True, use_bn=True): # NOTE: Neither Conv2DDNNLayer nor Conv2DMMLayer will not work # with T.Rop operation, which used for the Fisher-vector product. l_input = L.InputLayer((None, 1, config['height'], config['width'])) l_out = L.Conv2DLayer(l_input, num_filters=config['cnn_f1'], filter_size=(6,6), stride=2, nonlinearity=relu, W=LI.HeUniform('relu'), b=LI.Constant(0.) ) # https://arxiv.org/pdf/1602.01407v2.pdf # QUOTE: KFC-pre and BN can be combined synergistically. if use_bn: l_out = L.batch_norm(l_out, beta=None, gamma=None) l_out = L.Conv2DLayer(l_out, num_filters=config['cnn_f2'], filter_size=(4,4), stride=2, nonlinearity=relu, W=LI.HeUniform('relu'), b=LI.Constant(0.) ) if use_bn: l_out = L.batch_norm(l_out, beta=None, gamma=None) if use_noise: l_out = L.dropout(l_out) l_out = L.Conv2DLayer(l_out, num_filters=config['cnn_f3'], filter_size=(4,4), stride=2, nonlinearity=relu, W=LI.HeUniform('relu'), b=LI.Constant(0.) ) if use_bn: l_out = L.batch_norm(l_out, beta=None, gamma=None) if use_noise: l_out = L.dropout(l_out) return l_input, l_out
def __build_48_net__(self): network = layers.InputLayer((None, 3, 48, 48), input_var=self.__input_var__) network = layers.Conv2DLayer(network, num_filters=64, filter_size=(5, 5), stride=1, nonlinearity=relu) network = layers.MaxPool2DLayer(network, pool_size=(3, 3), stride=2) network = layers.batch_norm(network) network = layers.Conv2DLayer(network, num_filters=64, filter_size=(5, 5), stride=1, nonlinearity=relu) network = layers.batch_norm(network) network = layers.MaxPool2DLayer(network, pool_size=(3, 3), stride=2) network = layers.Conv2DLayer(network, num_filters=64, filter_size=(3, 3), stride=1, nonlinearity=relu) network = layers.batch_norm(network) network = layers.MaxPool2DLayer(network, pool_size=(3, 3), stride=2) network = layers.DenseLayer(network, num_units=256, nonlinearity=relu) network = layers.DenseLayer(network, num_units=2, nonlinearity=softmax) return network
def define_net(): net = {} # net['input_img'] = ll.InputLayer(shape=(None, 1, 28, 28), input_var=input_img) # net['input_noise'] = ll.InputLayer(shape=(None, 1, 28, 28), input_var=noise) # ll.ConcatLayer([net['input_img'], net['input_noise']], axis=1) net['input'] = ll.InputLayer(shape=(None, 2, 28, 28)) net['conv_1'] = ll.batch_norm( ll.Conv2DLayer(net['input'], num_filters=32, stride=(2, 2), filter_size=(5, 5), pad='same')) net['conv_2'] = ll.batch_norm( ll.Conv2DLayer(net['conv_1'], num_filters=64, stride=(2, 2), filter_size=(5, 5), pad='same')) net['unconv_3'] = ll.batch_norm( ll.TransposedConv2DLayer(net['conv_2'], filter_size=(5, 5), num_filters=32, stride=(2, 2), crop=(2, 2))) net['out'] = ll.batch_norm( ll.TransposedConv2DLayer(net['unconv_3'], filter_size=(4, 4), num_filters=1, stride=(2, 2), crop=(0, 0), nonlinearity=lasagne.nonlinearities.sigmoid)) return net
def lasagne_model(): l_in = InputLayer(shape=(None, 1,PIXELS , PIXELS)) # l_in =lasagne.layers.NonlinearityLayer(l_in,lasagne.nonlinearities.tanh) l_conv1 = Conv2DLayer(l_in, num_filters = 128, filter_size=(3,3), nonlinearity=rectify) l_conv1b = Conv2DLayer(l_conv1, num_filters = 128, filter_size=(3,3), nonlinearity=rectify) l_conv1b =batch_norm(l_conv1b) l_pool1 = MaxPool2DLayer(l_conv1b, pool_size=(2,2)) # l_dropout1 = DropoutLayer(l_pool1, p=0.2) l_conv2 = Conv2DLayer(l_pool1, num_filters = 256, filter_size=(3,3), nonlinearity=rectify) l_conv2b = Conv2DLayer(l_conv2, num_filters = 256, filter_size=(3,3), nonlinearity=rectify) l_conv2b =batch_norm(l_conv2b) l_pool2 = MaxPool2DLayer(l_conv2b, pool_size=(2,2)) # l_dropout2 = DropoutLayer(l_pool2, p=0.2) l_hidden3 = DenseLayer(l_pool2, num_units = h_dimension, nonlinearity=rectify) l_hidden3 =batch_norm(l_hidden3) # l_dropout3 = DropoutLayer(l_hidden3, p=0.5) l_hidden4 = DenseLayer(l_hidden3, num_units = h_dimension, nonlinearity=rectify) # l_dropout4 = DropoutLayer(l_hidden4, p=0.5) l_out = DenseLayer(l_hidden4, num_units=num_labels_train, nonlinearity=softmax) return l_out,l_hidden4
def __init__(self, dims, nonlinearities=None, dropouts=None, update_fn=None, batch_norm=False, loss_type='cosine_margin', margin=0.8): """Initialize a Siamese neural network Parameters: ----------- update_fn: theano function with 2 arguments (loss, params) Update scheme, default to adadelta batch_norm: bool Do batch normalisation on first layer, default to false """ assert len(dims) >= 3, 'Not enough dimmensions' if dropouts != None: dropouts = copy.copy(dropouts) assert len(dropouts) == len(dims) - 1 dropouts.append(0) else: dropouts = [0] * len(dims) if nonlinearities==None: nonlinearities = [nl.sigmoid] * (len(dims) -1) else: assert len(nonlinearities) == len(dims) - 1 if update_fn == None: update_fn = lasagne.updates.adadelta self.input_var1 = T.matrix('inputs1') self.input_var2 = T.matrix('inputs2') self.target_var = T.ivector('targets') # input layer network1 = layers.InputLayer((None, dims[0]), input_var=self.input_var1) network2 = layers.InputLayer((None, dims[0]), input_var=self.input_var2) if dropouts[0]: network1 = layers.DropoutLayer(network1, p=dropouts[0]) network2 = layers.DropoutLayer(network2, p=dropouts[0]) # hidden layers for dim, dropout, nonlin in zip(dims[1:], dropouts[1:], nonlinearities): network1 = layers.DenseLayer(network1, num_units=dim, W=lasagne.init.GlorotUniform(), nonlinearity=nonlin) network2 = layers.DenseLayer(network2, num_units=dim, W=network1.W, b=network1.b, nonlinearity=nonlin) if batch_norm: network1 = layers.batch_norm(network1) network2 = layers.batch_norm(network2) if dropout: network1 = layers.DropoutLayer(network1, p=dropout) network2 = layers.DropoutLayer(network2, p=dropout) self.network = [network1, network2] self.params = layers.get_all_params(network1, trainable=True) # util functions, completely stolen from Lasagne example self.prediction1 = layers.get_output(network1) self.prediction2 = layers.get_output(network2) # if non-determnistic: self.test_prediction1 = layers.get_output(network1, deterministic=True) self.test_prediction2 = layers.get_output(network2, deterministic=True) self.change_loss(loss_type, margin) self.change_update(update_fn)
def build_generator(parameter, input_var=None): from lasagne.layers import InputLayer, ReshapeLayer, DenseLayer, batch_norm from lasagne.nonlinearities import tanh, sigmoid layer = InputLayer(shape=(None, 100), input_var=input_var) # fully-connected layer layer = DenseLayer(layer, 256 * 4 * 4 * 4) parameter['W1'] = layer.W parameter['b1'] = layer.b layer = batch_norm(layer, beta=parameter['beta1'], gamma=parameter['gamma1'], mean=parameter['mean1'], inv_std=parameter['inv_std1']) layer = ReshapeLayer(layer, ([0], 256 * 4, 4, 4)) layer = Deconv2DLayer(layer, 256 * 2, 5, stride=2, pad=2) parameter['W2'] = layer.W parameter['b2'] = layer.b layer = batch_norm(layer, beta=parameter['beta2'], gamma=parameter['gamma2'], mean=parameter['mean2'], inv_std=parameter['inv_std2']) layer = Deconv2DLayer(layer, 256, 5, stride=2, pad=2) parameter['W3'] = layer.W parameter['b3'] = layer.b layer = batch_norm(layer, beta=parameter['beta3'], gamma=parameter['gamma3'], mean=parameter['mean3'], inv_std=parameter['inv_std3']) layer = Deconv2DLayer(layer, 4, 5, stride=2, pad=2, nonlinearity=sigmoid) parameter['W4'] = layer.W parameter['b4'] = layer.b # shape=(batch,1,28,28) print("Generator output:", layer.output_shape) return layer
def build_generator(input_var=None): #D_inp = T.tensor4('ds') G = l.InputLayer(shape=(None, 1, noise_H, noise_W), input_var=input_var) G = batch_norm( l.DenseLayer(G, num_units=(noise_H * noise_W * 256), nonlinearity=reLU)) G = l.ReshapeLayer(G, shape=([0], 256, noise_H, noise_W)) #4 G = l.TransposedConv2DLayer(G, 1, filter_size=(2, 2), stride=(2, 2), output_size=8) #8 G = batch_norm(l.Conv2DLayer(G, 40, (3, 3), nonlinearity=reLU, pad='full')) #10 G = l.TransposedConv2DLayer(G, 1, filter_size=(2, 2), stride=(2, 2), output_size=20) #20 G = batch_norm(l.Conv2DLayer(G, 20, (3, 3), nonlinearity=reLU, pad='full')) #22 G = batch_norm(l.Conv2DLayer(G, 20, (5, 5), nonlinearity=reLU, pad='full')) #26 G = batch_norm(l.Conv2DLayer(G, 1, (3, 3), nonlinearity=reLU, pad='full')) #28 return G
def build_generator(input_noise=None, input_text=None): from lasagne.layers import InputLayer, ReshapeLayer, DenseLayer, batch_norm, ConcatLayer from lasagne.nonlinearities import sigmoid # input: 100dim layer = InputLayer(shape=(None, noise_dim), input_var=input_noise) layer2 = InputLayer(shape=(None,1,300), input_var=input_text) layer2 = ReshapeLayer(layer2, ([0], 1*300)) layer = ConcatLayer([layer, layer2], axis=1) #increasing order of fc-layer for i in range(len(fclayer_list)): layer = batch_norm(DenseLayer(layer, fclayer_list[i])) newPS = 28 if stride!=1: newPS = 28/(2**len(layer_list)) layer = batch_norm(DenseLayer(layer, layer_list[0]*newPS*newPS)) layer = ReshapeLayer(layer, ([0], layer_list[0], newPS, newPS)) for i in range(1,len(layer_list)): layer = batch_norm(Deconv2DLayer(layer, layer_list[i], filter_sz, stride=stride, pad=(filter_sz-1)/2)) layer = Deconv2DLayer(layer, 1, filter_sz, stride=stride, pad=(filter_sz-1)/2, nonlinearity=sigmoid) print ("Generator output:", layer.output_shape) return layer
def discriminator(input_var=None, configs=None): lrelu = LeakyRectify(0.2) network = InputLayer(shape=(None, 1, configs['img_rows'], configs['img_cols']), input_var=input_var) network = batch_norm( Conv2DLayer(network, num_filters=64, filter_size=(5, 5), stride=2, nonlinearity=lrelu, W=lasagne.init.GlorotUniform())) # network = lasagne.layers.MaxPool2DLayer(network, pool_size=(2,2)) network = batch_norm( lasagne.layers.Conv2DLayer(network, num_filters=128, filter_size=5, stride=2, nonlinearity=lrelu)) # network = lasagne.layers.MaxPool2DLayer(network, pool_size=(2,2)) network = batch_norm( DenseLayer(incoming=lasagne.layers.dropout(network, p=0.25), num_units=1024, nonlinearity=lrelu)) network = DenseLayer( incoming=network, num_units=1, nonlinearity=sigmoid, ) network = lasagne.layers.ReshapeLayer(network, (-1, nb_classes)) return network
def classificationBranch(net, kernel_size): # Post Convolution branch = l.batch_norm(l.Conv2DLayer(net, num_filters=int(FILTERS[-1] * RESNET_K), filter_size=kernel_size, nonlinearity=nl.rectify)) #log.p(("\t\tPOST CONV SHAPE:", l.get_output_shape(branch), "LAYER:", len(l.get_all_layers(branch)) - 1)) # Dropout Layer branch = l.DropoutLayer(branch) # Dense Convolution branch = l.batch_norm(l.Conv2DLayer(branch, num_filters=int(FILTERS[-1] * RESNET_K * 2), filter_size=1, nonlinearity=nl.rectify)) #log.p(("\t\tDENSE CONV SHAPE:", l.get_output_shape(branch), "LAYER:", len(l.get_all_layers(branch)) - 1)) # Dropout Layer branch = l.DropoutLayer(branch) # Class Convolution branch = l.Conv2DLayer(branch, num_filters=len(cfg.CLASSES), filter_size=1, nonlinearity=None) return branch
def build_discriminator(input_img=None, input_text=None): from lasagne.layers import (InputLayer, Conv2DLayer, ReshapeLayer, DenseLayer, batch_norm, ConcatLayer) from lasagne.nonlinearities import LeakyRectify, sigmoid lrelu = LeakyRectify(0.1) # input: (None, 1, 28, 28) layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_img) layer2 = InputLayer(shape=(None,1,300), input_var=input_text) layer2 = ReshapeLayer(layer2, ([0], 1*300)) for i in reversed(range(len(layer_list))): layer = batch_norm(Conv2DLayer(layer, layer_list[i], filter_sz, stride=stride, pad=(filter_sz-1)/2, nonlinearity=lrelu)) newPS = 28 if stride!=1: newPS = 28/(2**len(layer_list)) layer = ReshapeLayer(layer, ([0], layer_list[0]*newPS*newPS)) layer = ConcatLayer([layer, layer2], axis=1) for i in reversed(range(len(fclayer_list))): layer = batch_norm(DenseLayer(layer, fclayer_list[i], nonlinearity=lrelu)) layer = DenseLayer(layer, 1, nonlinearity=None, b=None) print ("Discriminator output:", layer.output_shape) return layer
def build_network(self, input_var=None): if not input_var is None: self.sinputs = input_var self.network['input'] = lasagne.layers.InputLayer(shape=(self.batch_size, 1, self.input_size['audio'][0],self.input_size['audio'][1]), input_var=self.sinputs[0]) self.network['Conv2D_1'] = batch_norm(lasagne.layers.Conv2DLayer( lasagne.layers.dropout(self.network['input'], p=self.dropout_rates[0]) , num_filters=25, filter_size=(5, 5), nonlinearity=lasagne.nonlinearities.tanh, W=lasagne.init.GlorotUniform())) self.network['MaxPool2D_1'] = lasagne.layers.MaxPool2DLayer(self.network['Conv2D_1'], pool_size=(1, 1)) self.network['FC_1'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['MaxPool2D_1'], p=self.dropout_rates[1]), num_units=self.fc_layers[0], nonlinearity=lasagne.nonlinearities.tanh)) self.network['FC_N'] = batch_norm(lasagne.layers.DenseLayer(lasagne.layers.dropout(self.network['FC_1'], p=self.dropout_rates[2]), num_units=self.fc_layers[1], nonlinearity=lasagne.nonlinearities.tanh)) self.network['prob'] = batch_norm(lasagne.layers.DenseLayer( lasagne.layers.dropout(self.network['FC_N'], p=self.dropout_rates[3]), num_units=self.nclasses, nonlinearity=lasagne.nonlinearities.softmax)) return self.network
def build_generator(input_var=None, noise_size=100, nfilters=[512, 256, 128, 64, 3]): ############################### # Build Network Configuration # ############################### print('... Building the generator') # Input of the network : shape = (batch_size, 100) network = layers.InputLayer(shape=(None, noise_size), input_var=input_var) # Reshape layer : shape = (batch_size, 100, 1, 1) network = layers.ReshapeLayer(network, (-1, noise_size, 1, 1)) # Tranposed conv layer : shape = (batch_size, 512, 4, 4) network = layers.batch_norm( layers.TransposedConv2DLayer(network, num_filters=nfilters[0], filter_size=(4, 4), stride=(1, 1))) # Tranposed conv layer : shape = (batch_size, 256, 8, 8) network = layers.batch_norm( layers.TransposedConv2DLayer(network, num_filters=nfilters[1], filter_size=(5, 5), stride=(2, 2), crop=2, output_size=8)) # Tranposed conv layer : shape = (batch_size, 128, 16, 16) network = layers.batch_norm( layers.TransposedConv2DLayer(network, num_filters=nfilters[2], filter_size=(5, 5), stride=(2, 2), crop=2, output_size=16)) # Tranposed conv layer : shape = (batch_size, 64, 32, 32) network = layers.batch_norm( layers.TransposedConv2DLayer(network, num_filters=nfilters[3], filter_size=(5, 5), stride=(2, 2), crop=2, output_size=32)) # Tranposed conv layer : shape = (batch_size, 3, 64, 64) network = layers.TransposedConv2DLayer(network, num_filters=nfilters[4], filter_size=5, stride=2, crop=2, output_size=64, nonlinearity=nonlinearities.sigmoid) return network
def inc_dim_layer(l_in, num_filters): """ Increase the dimension of filter number decrease image size Args: incoming: Returns: """ l = batch_norm( ConvLayer(l_in, num_filters=num_filters, filter_size=(3, 3), stride=(2, 2), nonlinearity=rectify, pad='same', W=lasagne.init.HeNormal(gain='relu')) ) # 128 x 16 x 16 (1 highway block) (2 conv layers) l = batch_norm( ConvLayer(l, num_filters=num_filters, filter_size=(3, 3), stride=(1, 1), nonlinearity=rectify, pad='same', W=lasagne.init.HeNormal(gain='relu'))) return l
def build_discriminator(input_var=None, nfilters=[64, 128, 256, 512], input_channels=3): ############################### # Build Network Configuration # ############################### print('... Building the discriminator') leaky = nonlinearities.LeakyRectify(0.2) # Input of the network : shape = (batch_size, 3, 64, 64) network = layers.InputLayer(shape=(None, input_channels, 64, 64), input_var=input_var) # Conv layer : shape = (batch_size, 64, 32, 32) network = layers.Conv2DLayer(network, num_filters=nfilters[0], filter_size=(5, 5), stride=2, pad=2, nonlinearity=leaky) # Conv layer : shape = (batch_size, 128, 16, 16) network = layers.batch_norm( lasagne.layers.Conv2DLayer(network, num_filters=nfilters[1], filter_size=(5, 5), stride=2, pad=2, nonlinearity=leaky)) # Conv layer : shape = (batch_size, 256, 8, 8) network = layers.batch_norm( lasagne.layers.Conv2DLayer(network, num_filters=nfilters[2], filter_size=(5, 5), stride=2, pad=2, nonlinearity=leaky)) # Conv layer : shape = (batch_size, 512, 4, 4) network = layers.batch_norm( lasagne.layers.Conv2DLayer(network, num_filters=nfilters[3], filter_size=(5, 5), stride=2, pad=2, nonlinearity=leaky)) # Flatten layer :shape = (batch_size, 8192) network = lasagne.layers.FlattenLayer(network) # Dense layer :shape = (batch_size, 1) network = lasagne.layers.DenseLayer( network, 1, nonlinearity=lasagne.nonlinearities.sigmoid) return network
def init_discriminator(self, first_layer, input_var=None): """ Initialize the DCGAN discriminator network using lasagne Returns the network """ lrelu = nonlinearities.LeakyRectify(0.2) layers = [] l_in = lyr.InputLayer((None, 3, 64, 64), input_var) layers.append(l_in) l_1 = lyr.Conv2DLayer(incoming=l_in, num_filters=first_layer, filter_size=5, stride=2, pad=2, nonlinearity=lrelu) layers.append(l_1) l_2 = lyr.batch_norm( lyr.Conv2DLayer(incoming=l_1, num_filters=first_layer * 2, filter_size=5, stride=2, pad=2, nonlinearity=lrelu)) layers.append(l_2) l_3 = lyr.batch_norm( lyr.Conv2DLayer(incoming=l_2, num_filters=first_layer * 4, filter_size=5, stride=2, pad=2, nonlinearity=lrelu)) layers.append(l_3) l_4 = lyr.batch_norm( lyr.Conv2DLayer(incoming=l_3, num_filters=first_layer * 8, filter_size=5, stride=2, pad=2, nonlinearity=lrelu)) l_4 = lyr.FlattenLayer(l_4) layers.append(l_4) l_out = lyr.DenseLayer(incoming=l_4, num_units=1, nonlinearity=nonlinearities.sigmoid) layers.append(l_out) if self.verbose: for i, layer in enumerate(layers): print 'dicriminator layer %s output shape:' % i, layer.output_shape return l_out
def architecture_upconv_mp3(input_var, input_shape, n_conv_layers, n_conv_filters): net = {} kwargs = dict(nonlinearity=lasagne.nonlinearities.elu, W=lasagne.init.HeNormal()) net['data'] = InputLayer(input_shape, input_var) print("\rLayer output shapes") print(net['data'].output_shape) # Bunch of 3 x 3 convolution layers: experimentally we found that, adding 3 conv layers in start than in middle is better: but why? i = 'data' j = 'c1' for idx in range(n_conv_layers): print("Conv layer index: %d" % (idx + 1)) net[j] = batch_norm( Conv2DLayer(net[i], num_filters=n_conv_filters, filter_size=3, stride=1, pad=1, **kwargs)) print(net[j].output_shape) # renaming for next iteration i = j j = j[:-1] + str(idx + 2) # Bunch of transposed convolution layers net['uc1'] = batch_norm( TransposedConv2DLayer(net[i], num_filters=n_conv_filters / 2, filter_size=4, stride=2, crop=1, **kwargs)) print(net['uc1'].output_shape) net['uc2'] = batch_norm( TransposedConv2DLayer(net['uc1'], num_filters=1, filter_size=4, stride=2, crop=1, **kwargs)) print(net['uc2'].output_shape) # slicing the output to 115 x 80 size net['s1'] = lasagne.layers.SliceLayer(net['uc2'], slice(0, 115), axis=-2) print(net['s1'].output_shape) net['out'] = lasagne.layers.SliceLayer(net['s1'], slice(0, 80), axis=-1) print(net['out'].output_shape) print("Number of parameter to be learned: %d" % (lasagne.layers.count_params(net['out']))) return net['out']
def __build_48_calib_net__(self): network = layers.InputLayer((None, 3, 48, 48), input_var=self.__input_var__) network = layers.Conv2DLayer(network,num_filters=64,filter_size=(5,5),stride=1,nonlinearity=relu) network = layers.batch_norm(layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2)) network = layers.Conv2DLayer(network,num_filters=64,filter_size=(5,5),stride=1,nonlinearity=relu) network = layers.batch_norm(layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2)) network = layers.DenseLayer(network,num_units = 256,nonlinearity = relu) network = layers.DenseLayer(network,num_units = 45, nonlinearity = softmax) return network
def build_generator(input_var=None, dim_z=100): layer = InputLayer(shape=(None, dim_z), input_var=input_var) layer = batch_norm(DenseLayer(layer, 1024)) layer = batch_norm(DenseLayer(layer, 128 * 7 * 7)) layer = ReshapeLayer(layer, ([0], 128, 7, 7)) layer = batch_norm(Deconv2DLayer(layer, 64, 5, stride=2, pad=2)) layer = Deconv2DLayer(layer, 1, 5, stride=2, pad=2, nonlinearity=sigmoid) logger.debug('Generator output: {}'.format(layer.output_shape)) return layer
def build_synth(input_dist=None): from lasagne.layers import (InputLayer, DenseLayer, batch_norm, ReshapeLayer) from lasagne.nonlinearities import LeakyRectify, rectify lrelu = LeakyRectify(0.2) layer = InputLayer(shape=(None, 1, 28, 28), input_var=input_dist) layer = batch_norm(DenseLayer(layer, 1024, nonlinearity=lrelu)) layer = batch_norm(DenseLayer(layer, 1 * 28 * 28))#, nonlinearity=lrelu) layer = ReshapeLayer(layer, ([0], 1, 28, 28)) return layer
def build_treatment_model(self, n_vars, **kwargs): input_vars = TT.matrix() instrument_vars = TT.matrix() targets = TT.vector() inputs = layers.InputLayer((None, n_vars), input_vars) inputs = layers.DropoutLayer(inputs, p=0.2) dense_layer = layers.DenseLayer(inputs, 2 * kwargs['dense_size'], nonlinearity=nonlinearities.rectify) dense_layer = layers.batch_norm(dense_layer) dense_layer= layers.DropoutLayer(dense_layer, p=0.2) for _ in xrange(kwargs['n_dense_layers'] - 1): dense_layer = layers.DenseLayer(dense_layer, kwargs['dense_size'], nonlinearity=nonlinearities.rectify) dense_layer = layers.batch_norm(dense_layer) self.treatment_output = layers.DenseLayer(dense_layer, 1, nonlinearity=nonlinearities.linear) init_params = layers.get_all_param_values(self.treatment_output) prediction = layers.get_output(self.treatment_output, deterministic=False) test_prediction = layers.get_output(self.treatment_output, deterministic=True) l2_cost = regularization.regularize_network_params(self.treatment_output, regularization.l2) loss = gmm_loss(prediction, targets, instrument_vars) + 1e-4 * l2_cost params = layers.get_all_params(self.treatment_output, trainable=True) param_updates = updates.adadelta(loss, params) self._train_fn = theano.function( [ input_vars, targets, instrument_vars, ], loss, updates=param_updates ) self._loss_fn = theano.function( [ input_vars, targets, instrument_vars, ], loss, ) self._output_fn = theano.function( [ input_vars, ], test_prediction, ) return init_params
def build_net(): net = OrderedDict() net['input'] = InputLayer((BATCH_SIZE, 1, 128, 128)) net['conv_1_1'] = batch_norm(ConvLayer(net['input'], 12, 7, pad='same', stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_1_1_do'] = DropoutLayer(net['conv_1_1'], p=0.1) net['conv_1_2'] = batch_norm(ConvLayer(net['conv_1_1'], 12, 5, pad='same', stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_1_2_do'] = DropoutLayer(net['conv_1_2'], p=0.1) net['maxPool_1_1'] = Pool2DLayer(net['conv_1_2'], 2, mode='max') net['conv_2_1'] = batch_norm(ConvLayer(net['maxPool_1_1'], 24, 3, pad='same', stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_2_1_do'] = DropoutLayer(net['conv_2_1'], p=0.2) net['conv_2_2'] = batch_norm(ConvLayer(net['conv_2_1'], 24, 3, pad='same', stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_2_2_do'] = DropoutLayer(net['conv_2_2'], p=0.2) net['maxPool_2_1'] = Pool2DLayer(net['conv_2_2'], 2, mode='max') net['conv_3_1'] = batch_norm(ConvLayer(net['maxPool_2_1'], 48, 3, pad=1, stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_3_1_do'] = DropoutLayer(net['conv_3_1'], p=0.3) net['maxPool_3_1'] = Pool2DLayer(net['conv_3_1'], 2, mode='max') net['conv_3_2'] = batch_norm(ConvLayer(net['maxPool_3_1'], 48, 3, pad=1, stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_3_2_do'] = DropoutLayer(net['conv_3_2'], p=0.3) net['maxPool_3_2'] = Pool2DLayer(net['conv_3_2'], 2, mode='max') net['conv_3_3'] = batch_norm(ConvLayer(net['maxPool_3_2'], 48, 3, pad=1, stride=1, nonlinearity=lasagne.nonlinearities.elu)) # net['conv_3_3_do'] = DropoutLayer(net['conv_3_3'], p=0.3) net['maxPool_3_3'] = Pool2DLayer(net['conv_3_3'], 2, mode='max') net['fc_4'] = batch_norm(DenseLayer(net['maxPool_3_3'], 200, nonlinearity=lasagne.nonlinearities.elu)) # net['fc_4_dropOut'] = DropoutLayer(net['fc_4'], p=0.5) net['prob'] = batch_norm(DenseLayer(net['fc_4'], 2, nonlinearity=lasagne.nonlinearities.softmax)) return net
def mlp_network(dim=784,num_hidden_layers=1,num_hidden_nodes_per_layer=10,dropout=0.5): net={} net['l_in']=lasagne.layers.InputLayer((None,dim)) net['l_d']=lasagne.layers.DropoutLayer(net['l_in'],p=.1) for i in range(num_hidden_layers): if i==0: net[i]=batch_norm(lasagne.layers.DenseLayer(net['l_d'],num_units=num_hidden_nodes_per_layer,nonlinearity=lasagne.nonlinearities.rectify)) else: net[i]=batch_norm(lasagne.layers.DenseLayer(net[i-1],num_units=num_hidden_nodes_per_layer,nonlinearity=lasagne.nonlinearities.rectify)) net[i+1]=lasagne.layers.DropoutLayer(net[i],p=dropout) net['l_out']=lasagne.layers.DenseLayer(net[2*num_hidden_layers-1],num_units=10,nonlinearity=lasagne.nonlinearities.softmax) return net
def setup_discriminator(self): c = args.discriminator_size self.make_layer('disc1.1', batch_norm(self.network['conv1_2']), 1*c, filter_size=(5,5), stride=(2,2), pad=(2,2)) self.make_layer('disc1.2', self.last_layer(), 1*c, filter_size=(5,5), stride=(2,2), pad=(2,2)) self.make_layer('disc2', batch_norm(self.network['conv2_2']), 2*c, filter_size=(5,5), stride=(2,2), pad=(2,2)) self.make_layer('disc3', batch_norm(self.network['conv3_2']), 3*c, filter_size=(3,3), stride=(1,1), pad=(1,1)) hypercolumn = ConcatLayer([self.network['disc1.2>'], self.network['disc2>'], self.network['disc3>']]) self.make_layer('disc4', hypercolumn, 4*c, filter_size=(1,1), stride=(1,1), pad=(0,0)) self.make_layer('disc5', self.last_layer(), 3*c, filter_size=(3,3), stride=(2,2)) self.make_layer('disc6', self.last_layer(), 2*c, filter_size=(1,1), stride=(1,1), pad=(0,0)) self.network['disc'] = batch_norm(ConvLayer(self.last_layer(), 1, filter_size=(1,1), nonlinearity=lasagne.nonlinearities.linear))
def __build_24_net__(self): network = layers.InputLayer((None, 3, 24, 24), input_var=self.__input_var__) network = layers.dropout(network, p=0.1) network = layers.Conv2DLayer(network,num_filters=64,filter_size=(5,5),stride=1,nonlinearity=relu) network = layers.batch_norm(network) network = layers.MaxPool2DLayer(network, pool_size = (3,3),stride = 2) network = layers.DropoutLayer(network,p=0.5) network = layers.batch_norm(network) network = layers.DenseLayer(network,num_units = 64,nonlinearity = relu) network = layers.DropoutLayer(network,p=0.5) network = layers.DenseLayer(network,num_units = 2, nonlinearity = softmax) return network
def resfuse_block(l, residual=1, projection=True): """ residual: a hyperparameter of how much to leak in (should be small, or loss will explode: when it = 1, training loss: 2217594.131540) (such effect will be even higher with deeper layers) If we don't do projection, then the loss will just explode Every resfuse block is made of 2 resblock and top connect to bottom We simply won't allow dimension increase in a simple resfuse_block, dimension increase should be performed by a single resnet block layer! Args: increase_dim: only affect the first resnet block excessive: whether we try to connect more, or less """ input_num_filters = l.output_shape[1] stack1 = residual_block(l) stack2 = residual_block(stack1) block = None if projection: block = batch_norm( ConvLayer(l, num_filters=input_num_filters, filter_size=(1, 1), stride=(1, 1), nonlinearity=None, pad='same', b=None, name="resfuse_projection")) assert block.output_shape == stack2.output_shape block = NonlinearityLayer(ElemwiseSumLayer([block, stack2]), nonlinearity=None) else: # block = NonlinearityLayer(ElemwiseSumLayer([stack2, l], coeffs=residual), nonlinearity=None) block = stack2 # no summation, just regular resnet block return block
def get_convolutional_block( incoming, convo_size=3, num_filters=32, pool_size=2, drop=0.5, padding='valid', counter=itertools.count(), sufix='' ): index = counter.next() convolution = Conv3DLayer( incoming=incoming, name='\033[34mconv_%s%d\033[0m' % (sufix, index), num_filters=num_filters, filter_size=convo_size, pad=padding ) normalisation = batch_norm( layer=convolution, name='norm_%s%d' % (sufix, index) ) dropout = DropoutLayer( incoming=normalisation, name='drop_%s%d' % (sufix, index), p=drop ) pool = Pool3DLayer( incoming=dropout, name='\033[31mavg_pool_%s%d\033[0m' % (sufix, index), pool_size=pool_size, mode='average_inc_pad' ) return pool
def build_resfuse_net(input_var=None, n=5, execessive=False): # Building the network l_in = InputLayer(shape=(None, 3, 64, 64), input_var=input_var) # first layer, output is 16 x 64 x 64 l = batch_norm(ConvLayer(l_in, num_filters=16, filter_size=(3, 3), stride=(1, 1), nonlinearity=rectify, pad='same', W=lasagne.init.HeNormal(gain='relu'))) # first stack of residual blocks, output is 16 x 64 x 64 l = resfuse_block(l) # 2 resfuse blocks l = resfuse_super_block(l, excessive=execessive) # second stack of residual blocks, output is 32 x 32 x 32 l = residual_block(l, increase_dim=True) l = resfuse_super_block(l, excessive=execessive) # 4 res-blocks # third stack of residual blocks, output is 64 x 16 x 16 l = residual_block(l, increase_dim=True) l = resfuse_super_block(l, excessive=execessive) # 4 res-blocks # average pooling l = GlobalPoolLayer(l) # fully connected layer network = DenseLayer( l, num_units=100, W=lasagne.init.HeNormal(), nonlinearity=softmax) return network
def build_CNN_nopool(in_shape, num_filter, fil_size, strides, num_out, nlin_func=rectify, in_var=None): # build a CNN net = InputLayer(input_var=in_var, shape=in_shape) for i in xrange(len(fil_size)): net = batch_norm(ConvLayer(net, num_filters=num_filter[i], filter_size=fil_size[i], stride=strides[i], pad=1, nonlinearity=nlin_func, flip_filters=False)) net = DenseLayer(incoming=net, num_units=num_out, nonlinearity=identity) return net
def residual_block(l, increase_dim=False, projection=True, first=False): input_num_filters = l.output_shape[1] if increase_dim: first_stride = (2,2) out_num_filters = input_num_filters*2 else: first_stride = (1,1) out_num_filters = input_num_filters if first: # hacky solution to keep layers correct bn_pre_relu = l else: # contains the BN -> ReLU portion, steps 1 to 2 bn_pre_conv = BatchNormLayer(l) bn_pre_relu = NonlinearityLayer(bn_pre_conv, rectify) # contains the weight -> BN -> ReLU portion, steps 3 to 5 conv_1 = batch_norm(ConvLayer(bn_pre_relu, num_filters=out_num_filters, filter_size=(3,3), stride=first_stride, nonlinearity=rectify, pad='same', W=he_norm)) # contains the last weight portion, step 6 conv_2 = ConvLayer(conv_1, num_filters=out_num_filters, filter_size=(3,3), stride=(1,1), nonlinearity=None, pad='same', W=he_norm) # add shortcut connections if increase_dim: # projection shortcut, as option B in paper projection = ConvLayer(l, num_filters=out_num_filters, filter_size=(1,1), stride=(2,2), nonlinearity=None, pad='same', b=None) block = ElemwiseSumLayer([conv_2, projection]) else: block = ElemwiseSumLayer([conv_2, l]) return block
def build_residual_layer(network, num_filters, num_elements): original = network #like an 8x8 but with about half the parameters... maybe... for i in xrange(num_elements): network = Conv2DLayer(network, num_filters, 7, stride = 1, pad='same') network = batch_norm(network) return ElemwiseSumLayer([original, network])
def build_cnn(input_var=None, n=5): # Building the network l_in = InputLayer(shape=(None, 3, 64, 64), input_var=input_var) # first layer, output is 16 x 64 x 64 l = batch_norm(ConvLayer(l_in, num_filters=16, filter_size=(3, 3), stride=(1, 1), nonlinearity=rectify, pad='same', W=lasagne.init.HeNormal(gain='relu'))) # we could pool aggressively here, but we don't have to # CIFAR-10 doesn't aggressively pool, and ImageNet 128x128 aggressively pools # first stack of residual blocks, output is 16 x 64 x 64 for _ in range(n): l = residual_block(l) # second stack of residual blocks, output is 32 x 32 x 32 l = residual_block(l, increase_dim=True) for _ in range(1, n): l = residual_block(l) # third stack of residual blocks, output is 64 x 16 x 16 l = residual_block(l, increase_dim=True) for _ in range(1, n): l = residual_block(l) # average pooling l = GlobalPoolLayer(l) # fully connected layer network = DenseLayer( l, num_units=100, W=lasagne.init.HeNormal(), nonlinearity=softmax) return network
def processLayer(self, network, layer_definition): ''' Create a lasagne layer corresponding to the "layer definition" ''' if (layer_definition["type"] == "Input"): if self.network_type == 'CAE': network = lasagne.layers.InputLayer(shape=tuple([None] + layer_definition['output_shape']), input_var=self.t_input) elif self.network_type == 'AE': network = lasagne.layers.InputLayer(shape=(None, layer_definition['output_shape'][2]), input_var=self.t_input) elif (layer_definition['type'] == 'Dense'): network = lasagne.layers.DenseLayer(network, num_units=layer_definition['num_units'], nonlinearity=self.getNonLinearity(layer_definition['non_linearity']), name=self.getLayerName(layer_definition),W=self.getInitializationFct()) elif (layer_definition['type'] == 'Conv2D'): network = lasagne.layers.Conv2DLayer(network, num_filters=layer_definition['num_filters'], filter_size=tuple(layer_definition["filter_size"]), pad=layer_definition['conv_mode'], nonlinearity=self.getNonLinearity(layer_definition['non_linearity']), name=self.getLayerName(layer_definition),W=self.getInitializationFct()) elif (layer_definition['type'] == 'MaxPool2D' or layer_definition['type'] == 'MaxPool2D*'): network = lasagne.layers.MaxPool2DLayer(network, pool_size=tuple(layer_definition["filter_size"]), name=self.getLayerName(layer_definition)) elif (layer_definition['type'] == 'InverseMaxPool2D'): network = lasagne.layers.InverseLayer(network, self.layer_list[layer_definition['layer_index']], name=self.getLayerName(layer_definition)) elif (layer_definition['type'] == 'Unpool2D'): network = Unpool2DLayer(network, tuple(layer_definition['filter_size']), name=self.getLayerName(layer_definition)) elif (layer_definition['type'] == 'Reshape'): network = lasagne.layers.ReshapeLayer(network, shape=tuple([-1] + layer_definition["output_shape"]), name=self.getLayerName(layer_definition)) elif (layer_definition['type'] == 'Deconv2D'): network = lasagne.layers.Deconv2DLayer(network, num_filters=layer_definition['num_filters'], filter_size=tuple(layer_definition['filter_size']), crop=layer_definition['conv_mode'], nonlinearity=self.getNonLinearity(layer_definition['non_linearity']), name=self.getLayerName(layer_definition)) self.layer_list.append(network) # Batch normalization on all convolutional layers except if at output if (self.batch_norm and (not layer_definition["is_output"]) and layer_definition['type'] in ("Conv2D", "Deconv2D")): network = batch_norm(network) # Save the encode layer separately if (layer_definition['is_encode']): self.encode_layer = lasagne.layers.flatten(network, name='fl') self.encode_size = layer_definition['output_shape'][0] * layer_definition['output_shape'][1] * layer_definition['output_shape'][2] return network
def cnn_network(dim=784,num_filters=3,filter_size=3): #Defining a simple one layer CNN network net={} net['l_in']=lasagne.layers.InputLayer((None,dim)) net[0]=lasagne.layers.ReshapeLayer(net['l_in'],(-1,1,28,28)) net[1]=batch_norm(lasagne.layers.Conv2DLayer(net[0],num_filters,filter_size,pad=1,nonlinearity=lasagne.nonlinearities.rectify)) net['l_out']=lasagne.layers.DenseLayer(net[1],num_units=10,nonlinearity=lasagne.nonlinearities.softmax) return net
def get_model(input_var, target_var, multiply_var): # input layer with unspecified batch size layer_input = InputLayer(shape=(None, 30, 80, 80), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var) layer_0 = DimshuffleLayer(layer_input, (0, 'x', 1, 2, 3)) # Z-score? # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_1 = batch_norm(Conv3DDNNLayer(incoming=layer_0, num_filters=64, filter_size=(3,3,3), stride=(1,3,3), pad='same', nonlinearity=leaky_rectify)) layer_2 = MaxPool3DDNNLayer(layer_1, pool_size=(1, 2, 2), stride=(1, 2, 2), pad=(0, 1, 1)) layer_3 = DropoutLayer(layer_2, p=0.25) # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_4 = batch_norm(Conv3DDNNLayer(incoming=layer_3, num_filters=128, filter_size=(3,3,3), stride=(1,3,3), pad='same', nonlinearity=leaky_rectify)) layer_5 = MaxPool3DDNNLayer(layer_4, pool_size=(1, 2, 2), stride=(1, 2, 2), pad=(0, 1, 1)) layer_6 = DropoutLayer(layer_5, p=0.25) # Recurrent layer layer_7 = DimshuffleLayer(layer_6, (0,2,1,3,4)) layer_8 = LSTMLayer(layer_7, num_units=612, only_return_final=True) layer_9 = DropoutLayer(layer_8, p=0.25) # Output Layer layer_hidden = DenseLayer(layer_9, 500, nonlinearity=sigmoid) layer_prediction = DenseLayer(layer_hidden, 2, nonlinearity=linear) # Loss prediction = get_output(layer_prediction) / multiply_var loss = squared_error(prediction, target_var) loss = loss.mean() #Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum params = get_all_params(layer_prediction, trainable=True) # Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, disabling dropout layers. test_prediction = get_output(layer_prediction, deterministic=True) / multiply_var test_loss = squared_error(test_prediction, target_var) test_loss = test_loss.mean() # crps estimate crps = T.abs_(test_prediction - target_var).mean()/600 return test_prediction, crps, loss, params
def pop(input_stack, nlayers = 5): pop_net = {} prev_layer = input_stack for i in range(nlayers): next_layer = "l%i" % i prev_layer = pop_net[next_layer] = batch_norm(DenseLayer(prev_layer, nstack_reals+nitem_reals, nonlinearity = lasagne.nonlinearities.rectify, W=lasagne.init.HeNormal(gain='relu'))) return pop_net, prev_layer
def highway_layer(incoming, filter_size=(3, 3), increase_dim=False, **kwargs): num_filters = incoming.output_shape[1] # regular layer l_h = batch_norm(lasagne.layers.Conv2DLayer(incoming, num_filters=num_filters, filter_size=filter_size, pad='same', stride=(1, 1), W=lasagne.init.HeNormal(gain='relu'), nonlinearity=rectify)) # gate layer l_t = batch_norm(lasagne.layers.Conv2DLayer(incoming, num_filters=num_filters, filter_size=filter_size, pad='same', stride=(1, 1), W=lasagne.init.HeNormal(), nonlinearity=T.nnet.sigmoid)) return MultiplicativeGatingLayer(gate=l_t, input1=l_h, input2=incoming)