def __init__(self, input_shape, output_dim, hidden_dim, hidden_nonlinearity=LN.rectify, output_nonlinearity=None, output_W_init=LI.GlorotUniform(), name=None, input_var=None, input_layer=None, trunc_steps=20, output_gain=1): if input_layer is None: l_in = L.InputLayer(shape=(None, None) + input_shape, input_var=input_var, name="input") else: l_in = input_layer l_step_input = L.InputLayer(shape=(None, ) + input_shape) l_step_prev_hidden = L.InputLayer(shape=(None, hidden_dim)) l_gru = GRULayer(l_in, num_units=hidden_dim, hidden_nonlinearity=hidden_nonlinearity, hidden_init_trainable=False, trunc_steps=trunc_steps) l_gru_flat = L.ReshapeLayer(l_gru, shape=(-1, hidden_dim)) l_output_flat = L.DenseLayer( l_gru_flat, num_units=output_dim, nonlinearity=output_nonlinearity, W=output_W_init, ) l_output = OpLayer( l_output_flat, op=lambda flat_output, l_input: flat_output.reshape( (l_input.shape[0], l_input.shape[1], -1)), shape_op=lambda flat_output_shape, l_input_shape: (l_input_shape[0], l_input_shape[1], flat_output_shape[-1]), extras=[l_in]) l_step_hidden = l_gru.get_step_layer(l_step_input, l_step_prev_hidden) l_step_output = L.DenseLayer( l_step_hidden, num_units=output_dim, nonlinearity=output_nonlinearity, W=l_output_flat.W, b=l_output_flat.b, ) self._l_in = l_in self._hid_init_param = l_gru.h0 self._l_gru = l_gru self._l_out = l_output self._l_step_input = l_step_input self._l_step_prev_hidden = l_step_prev_hidden self._l_step_hidden = l_step_hidden self._l_step_output = l_step_output
def __init__(self, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=LI.GlorotUniform(), hidden_b_init=LI.Constant(0.), output_W_init=LI.GlorotUniform(), output_b_init=LI.Constant(0.), name=None, input_var=None, input_layer=None, input_shape=None, batch_norm=False): Serializable.quick_init(self, locals()) if name is None: prefix = "" else: prefix = name + "_" if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var) else: l_in = input_layer self._layers = [l_in] l_hid = l_in for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer( l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="%shidden_%d" % (prefix, idx), W=hidden_W_init, b=hidden_b_init, ) if batch_norm: l_hid = L.batch_norm(l_hid) self._layers.append(l_hid) l_out = L.DenseLayer( l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="%soutput" % (prefix, ), W=output_W_init, b=output_b_init, ) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out # self._input_var = l_in.input_var self._output = L.get_output(l_out) LasagnePowered.__init__(self, [l_out])
def _build_network(self, input_var): """Builds actor network, scaling outputs based on action bounds.""" tanh = lasagne.nonlinearities.ScaledTanH(1, scale_out=self.action_scale) network = nn.InputLayer((None, ) + self.state_shape, input_var) network = nn.DenseLayer(network, 30, W_init, b_init, relu) network = nn.DenseLayer(network, 30, W_init, b_init, relu) return nn.DenseLayer(network, self.num_actions, W_out, b_out, tanh)
def build_pi_model(): log.i('BUILDING RASBPERRY PI MODEL...') # Random Seed lasagne_random.set_rng(cfg.getRandomState()) # Input layer for images net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0])) # Convolutinal layer groups for i in range(len(cfg.FILTERS)): # 3x3 Convolution + Stride net = batch_norm( l.Conv2DLayer(net, num_filters=cfg.FILTERS[i], filter_size=cfg.KERNEL_SIZES[i], num_groups=cfg.NUM_OF_GROUPS[i], pad='same', stride=2, W=initialization(cfg.NONLINEARITY), nonlinearity=nonlinearity(cfg.NONLINEARITY))) log.i(('\tGROUP', i + 1, 'OUT SHAPE:', l.get_output_shape(net))) # Fully connected layers + dropout layers net = l.DenseLayer(net, cfg.DENSE_UNITS, nonlinearity=nonlinearity(cfg.NONLINEARITY), W=initialization(cfg.NONLINEARITY)) net = l.DropoutLayer(net, p=cfg.DROPOUT) net = l.DenseLayer(net, cfg.DENSE_UNITS, nonlinearity=nonlinearity(cfg.NONLINEARITY), W=initialization(cfg.NONLINEARITY)) net = l.DropoutLayer(net, p=cfg.DROPOUT) # Classification Layer (Softmax) net = l.DenseLayer(net, len(cfg.CLASSES), nonlinearity=nonlinearity('softmax'), W=initialization('softmax')) log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net))) log.i("...DONE!") # Model stats log.i(("MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS")) log.i(("MODEL HAS", l.count_params(net), "PARAMS")) return net
def __init__(self, incomings, num_units_hidden_common, dim_z, beta): ''' params: incomings: input layers, [image, label] num_units_hidden_common: num_units_hidden for all BasicLayers. dim_z: the dimension of z and num_units_output for encoders BaiscLayer ''' super(SemiVAE, self).__init__(incomings) self.mrg_srng = MRG_RandomStreams() # random generator self.incomings = incomings self.num_classes = incomings[1].output_shape[1] self.num_units_hidden_common = num_units_hidden_common self.dim_z = dim_z self.beta = beta self.concat_xy = layers.ConcatLayer(self.incomings, axis=1) self.encoder = BasicLayer( self.concat_xy, num_units_hidden=self.num_units_hidden_common, ) self.encoder_mu = layers.DenseLayer( self.encoder, self.dim_z, nonlinearity=nonlinearities.identity) self.encoder_log_var = layers.DenseLayer( self.encoder, self.dim_z, nonlinearity=nonlinearities.identity) [image_input, label_input] = self.incomings self.dim_image = image_input.output_shape[1] print('dim_image: ', self.dim_image) # merge encoder_mu and encoder_log_var to get z. self.sampler = SamplerLayer([self.encoder_mu, self.encoder_log_var]) self.concat_yz = layers.ConcatLayer([label_input, self.sampler], axis=1) self.decoder = BasicLayer( self.concat_yz, num_units_hidden=self.num_units_hidden_common) self.decoder_x = layers.DenseLayer(self.decoder, num_units=self.dim_image, nonlinearity=nonlinearities.sigmoid) self.classifier_helper = BasicLayer( self.incomings[0], num_units_hidden=self.num_units_hidden_common) self.classifier = layers.DenseLayer( self.classifier_helper, num_units=self.num_classes, nonlinearity=nonlinearities.softmax, )
def arch_class_00(dim_desc, dim_labels, param_arch, logger): logger.info('Architecture:') # input layers desc = LL.InputLayer(shape=(None, dim_desc)) patch_op = LL.InputLayer(input_var=Tsp.csc_fmatrix('patch_op'), shape=(None, None)) logger.info(' input : dim = %d' % dim_desc) # layer 1: dimensionality reduction to 16 n_dim = 16 net = LL.DenseLayer(desc, n_dim) logger.info(' layer 1: FC%d' % n_dim) # layer 2: anisotropic convolution layer with 16 filters n_filters = 16 net = CL.GCNNLayer([net, patch_op], n_filters, nrings=5, nrays=16) string = ' layer 2: IC%d' % n_filters if param_arch['flag_batchnorm'] is True: net = LL.batch_norm(net) string = string + ' + batch normalization' logger.info(string) # layer 3: anisotropic convolution layer with 32 filters n_filters = 32 net = CL.GCNNLayer([net, patch_op], n_filters, nrings=5, nrays=16) string = ' layer 3: IC%d' % n_filters if param_arch['flag_batchnorm'] is True: net = LL.batch_norm(net) string = string + ' + batch normalization' logger.info(string) # layer 4: anisotropic convolution layer with 64 filters n_filters = 64 net = CL.GCNNLayer([net, patch_op], n_filters, nrings=5, nrays=16) string = ' layer 4: IC%d' % n_filters if param_arch['flag_batchnorm'] is True: net = LL.batch_norm(net) string = string + ' + batch normalization' logger.info(string) # layer 5: fully connected layer with 256 filters n_dim = 256 net = LL.DenseLayer(net, n_dim) string = ' layer 5: FC%d' % n_dim if param_arch['flag_batchnorm'] is True: net = LL.batch_norm(net) string = string + ' + batch normalization' logger.info(string) # layer 6: softmax layer producing a probability on the labels if param_arch['flag_nonlinearity'] == 'softmax': cla = LL.DenseLayer(net, dim_labels, nonlinearity=LN.softmax) string = ' layer 6: softmax' elif param_arch['flag_nonlinearity'] == 'log_softmax': cla = LL.DenseLayer(net, dim_labels, nonlinearity=log_softmax) string = ' layer 6: log-softmax' else: raise Exception('[e] the chosen non-linearity is not supported!') logger.info(string) # outputs return desc, patch_op, cla, net, logger
def fcn_transfer(params): """""" assert 'inputs' in params layers = L.InputLayer((None, 256 * len(params['inputs']))) layers = L.dropout(layers) layers = L.DenseLayer(layers, 1024, nonlinearity=nl.elu) layers = L.dropout(layers) layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax) return layers
def build_instrument_model(self, n_vars, **kwargs): targets = TT.vector() instrument_vars = TT.matrix() instruments = layers.InputLayer((None, n_vars), instrument_vars) instruments = layers.DropoutLayer(instruments, p=0.2) dense_layer = layers.DenseLayer(instruments, kwargs['dense_size'], nonlinearity=nonlinearities.tanh) dense_layer = layers.DropoutLayer(dense_layer, p=0.2) for _ in xrange(kwargs['n_dense_layers'] - 1): dense_layer = layers.DenseLayer(dense_layer, kwargs['dense_size'], nonlinearity=nonlinearities.tanh) dense_layer = layers.DropoutLayer(dense_layer, p=0.5) self.instrument_output = layers.DenseLayer( dense_layer, 1, nonlinearity=nonlinearities.linear) init_params = layers.get_all_param_values(self.instrument_output) prediction = layers.get_output(self.instrument_output, deterministic=False) test_prediction = layers.get_output(self.instrument_output, deterministic=True) # flexible here, endog variable can be categorical, continuous, etc. l2_cost = regularization.regularize_network_params( self.instrument_output, regularization.l2) loss = objectives.squared_error( prediction.flatten(), targets.flatten()).mean() + 1e-4 * l2_cost loss_total = objectives.squared_error(prediction.flatten(), targets.flatten()).mean() params = layers.get_all_params(self.instrument_output, trainable=True) param_updates = updates.adadelta(loss, params) self._instrument_train_fn = theano.function([ targets, instrument_vars, ], loss, updates=param_updates) self._instrument_loss_fn = theano.function([ targets, instrument_vars, ], loss_total) self._instrument_output_fn = theano.function([instrument_vars], test_prediction) return init_params
def buildModel(): print "BUILDING MODEL TYPE..." #default settings filters = 16 first_stride = 2 last_filter_multiplier = 4 #input layer net = l.InputLayer((None, IM_DIM, IM_SIZE[1], IM_SIZE[0])) #conv layers net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters , filter_size=7, pad='same', stride=first_stride, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 2 , filter_size=5, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 4 , filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.MaxPool2DLayer(net, pool_size=2) net = l.DropoutLayer(net, DROPOUT) #net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 8 , filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) #net = l.MaxPool2DLayer(net, pool_size=2) #net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 16 , filter_size=3, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) #net = l.MaxPool2DLayer(net, pool_size=2) #net = l.batch_norm(l.Conv2DLayer(net, num_filters=filters * 32 , filter_size=7, pad='same', stride=1, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) #net = l.MaxPool2DLayer(net, pool_size=2) #print "\tFINAL POOL OUT SHAPE:", l.get_output_shape(net) #dense layers net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.DropoutLayer(net, DROPOUT) net = l.batch_norm(l.DenseLayer(net, 512, W=init.HeNormal(gain=INIT_GAIN), nonlinearity=NONLINEARITY)) net = l.DropoutLayer(net, DROPOUT) #Classification Layer if MULTI_LABEL: net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.sigmoid, W=init.HeNormal(gain=1)) else: net = l.DenseLayer(net, NUM_CLASSES, nonlinearity=nonlinearities.softmax, W=init.HeNormal(gain=1)) print "...DONE!" #model stats print "MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS" print "MODEL HAS", l.count_params(net), "PARAMS" return net
def get_model(inp, patch_op): icnn1 = batch_norm(utils_lasagne.GCNNLayer([inp, patch_op], 16, nrings=5, nrays=16)) ffn1 = LL.DenseLayer(icnn1, 512) icnn2 = batch_norm(utils_lasagne.GCNNLayer([icnn1, patch_op], 32, nrings=5, nrays=16)) ffn2 = LL.DenseLayer(icnn2, 512) icnn3 = batch_norm(utils_lasagne.GCNNLayer([icnn2, patch_op], 64, nrings=5, nrays=16)) ffn3 = LL.DenseLayer(icnn3, 512) ffn4 = LL.ConcatLayer([inp,ffn1,ffn2,ffn3],axis=1, cropping=None); ffn = LL.DenseLayer(ffn4, nclasses, nonlinearity=utils_lasagne.log_softmax) return ffn
def __init__(self, args): self.args = args rng = np.random.RandomState(self.args.seed) # fixed random seeds theano_rng = MRG_RandomStreams(rng.randint(2 ** 15)) lasagne.random.set_rng(np.random.RandomState(rng.randint(2 ** 15))) data_rng = np.random.RandomState(self.args.seed_data) ''' specify pre-trained generator E ''' self.enc_layers = [LL.InputLayer(shape=(None, 3, 32, 32), input_var=None)] enc_layer_conv1 = dnn.Conv2DDNNLayer(self.enc_layers[-1], 64, (5,5), pad=0, stride=1, W=Normal(0.01), nonlinearity=nn.relu) self.enc_layers.append(enc_layer_conv1) enc_layer_pool1 = LL.MaxPool2DLayer(self.enc_layers[-1], pool_size=(2, 2)) self.enc_layers.append(enc_layer_pool1) enc_layer_conv2 = dnn.Conv2DDNNLayer(self.enc_layers[-1], 128, (5,5), pad=0, stride=1, W=Normal(0.01), nonlinearity=nn.relu) self.enc_layers.append(enc_layer_conv2) enc_layer_pool2 = LL.MaxPool2DLayer(self.enc_layers[-1], pool_size=(2, 2)) self.enc_layers.append(enc_layer_pool2) self.enc_layer_fc3 = LL.DenseLayer(self.enc_layers[-1], num_units=256, nonlinearity=T.nnet.relu) self.enc_layers.append(self.enc_layer_fc3) self.enc_layer_fc4 = LL.DenseLayer(self.enc_layers[-1], num_units=10, nonlinearity=T.nnet.softmax) self.enc_layers.append(self.enc_layer_fc4) ''' load pretrained weights for encoder ''' weights_toload = np.load('pretrained/encoder.npz') weights_list_toload = [weights_toload['arr_{}'.format(k)] for k in range(len(weights_toload.files))] LL.set_all_param_values(self.enc_layers[-1], weights_list_toload) ''' input tensor variables ''' #self.G_weights #self.D_weights self.dummy_input = T.scalar() self.G_layers = [] self.z = theano_rng.uniform(size=(self.args.batch_size, self.args.z0dim)) self.x = T.tensor4() self.meanx = T.tensor3() self.Gen_x = T.tensor4() self.D_layers = [] self.D_layer_adv = [] self.D_layer_z_recon = [] self.gen_lr = T.scalar() # learning rate self.disc_lr = T.scalar() # learning rate self.y = T.ivector() self.y_1hot = T.matrix() self.Gen_x_list = [] self.y_recon_list = [] self.mincost = T.scalar() #self.enc_layer_fc3 = self.get_enc_layer_fc3() self.real_fc3 = LL.get_output(self.enc_layer_fc3, self.x, deterministic=True)
def build_network(input_var, image_size=28, output_dim=10): nonlin = lasagne.nonlinearities.rectify W_init = lasagne.init.GlorotUniform() b_init = lasagne.init.Constant(0.) input_shape = (None, 1, image_size, image_size) network = nn.InputLayer(input_shape, input_var) network = nn.Conv2DLayer(network, num_filters=64, filter_size=(3, 3), nonlinearity=nonlin, W=W_init, b=b_init) network = nn.Conv2DLayer(network, num_filters=64, filter_size=(3, 3), nonlinearity=nonlin, W=W_init, b=b_init) network = nn.MaxPool2DLayer(network, pool_size=(2, 2)) network = nn.Conv2DLayer(network, num_filters=128, filter_size=(3, 3), W=W_init, b=b_init, nonlinearity=nonlin) network = nn.Conv2DLayer(network, num_filters=128, filter_size=(3, 3), W=W_init, b=b_init, nonlinearity=nonlin) network = nn.MaxPool2DLayer(network, pool_size=(2, 2)) network = nn.dropout(network, p=0.5) network = nn.DenseLayer(network, num_units=256, W=W_init, b=b_init, nonlinearity=nonlin) network = nn.dropout(network, p=0.5) network = nn.DenseLayer(network, num_units=output_dim, W=W_init, b=b_init, nonlinearity=None) return network
def test_stochastic_layer_network(): learning_rate = 0.1 momentum = 0.9 num_epoch = 1000 input = T.fmatrix('input') output = T.fmatrix('output') print 'FF-Layer: (Batch_size, n_features)' print 'Building stochastic layer model' l_in = L.InputLayer(shape=(1, 10), input_var=input) l_2 = L.DenseLayer(l_in, num_units=10, nonlinearity=lasagne.nonlinearities.softmax, W=lasagne.init.Constant(0.)) print 'Input Layer shape: ', L.get_output_shape(l_in) print 'Dense Layer shape: ', L.get_output_shape(l_2) l_stochastic_layer = StochasticLayer(l_2, estimator='ST') print 'Stochastic Layer shape: ', L.get_output_shape(l_stochastic_layer) l_out = L.DenseLayer(l_stochastic_layer, num_units=10, b=lasagne.init.Constant(0.)) print 'Final Dense Layer shape: ', L.get_output_shape(l_out) network_output = L.get_output(l_out) print 'Building loss function...' loss = lasagne.objectives.squared_error(network_output, output) loss = loss.mean() params = L.get_all_params(l_out, trainable=True) updates = nesterov_momentum(loss, params, learning_rate, momentum) train = theano.function([input, output], loss, updates=updates, allow_input_downcast=True) output_fn = theano.function([input], network_output, allow_input_downcast=True) test_X = np.ones((1, 10)) test_Y = np.ones((1, 10)) losses = [] mean_losses = [] for epoch in range(num_epoch): print 'Epoch number: ', epoch losses.append(train(test_X, test_Y)) print 'epoch {} mean loss {}'.format(epoch, np.mean(losses)) print 'Current Output: ', output_fn(test_X) mean_losses.append(np.mean(losses)) plt.title("Mean loss") plt.xlabel("Training examples") plt.ylabel("Loss") plt.plot(mean_losses, label="train") plt.grid() plt.legend() plt.draw()
def get_model(inp, patch_op): icnn = LL.DenseLayer(inp, 16) icnn = batch_norm( utils_lasagne.GCNNLayer([icnn, patch_op], 16, nrings=4, nrays=8)) icnn = batch_norm( utils_lasagne.GCNNLayer([icnn, patch_op], 32, nrings=4, nrays=8)) icnn = batch_norm( utils_lasagne.GCNNLayer([icnn, patch_op], 64, nrings=4, nrays=8)) ffn = batch_norm(LL.DenseLayer(icnn, 512)) ffn = LL.DenseLayer(icnn, nclasses, nonlinearity=utils_lasagne.log_softmax) return ffn
def dec_net(_incoming, output_channels, nonlinearity=None): _fc1 = L.DenseLayer(_incoming, 4 * _incoming.output_shape[1], W=I.Normal(0.02), b=None, nonlinearity=NL.rectify) _fc2 = L.DenseLayer(_fc1, output_channels, W=I.Normal(0.02), b=I.Constant(0), nonlinearity=nonlinearity) return _fc2
def Network2(input_var=None,output_num=None,l1 = None,l2=None,l3=None,input_size = None): l_in = L.InputLayer(shape=(None,1,1,input_size),input_var=input_var) l_hid1 = L.DenseLayer(l_in,num_units=l1,nonlinearity=lasagne.nonlinearities.tanh) l_hid2 = L.DenseLayer(l_hid1,num_units = l2,nonlinearity = lasagne.nonlinearities.tanh) l_hid3 = L.DenseLayer(l_hid2,num_units = l3,nonlinearity = lasagne.nonlinearities.tanh) l_out = L.DenseLayer(l_hid3,num_units = output_num,nonlinearity=None) return l_out
def __init__(self, env_spec, hidden_sizes=(32, 32), hidden_nonlinearity=NL.rectify, hidden_w_init=LI.HeUniform(), hidden_b_init=LI.Constant(0.), output_nonlinearity=NL.tanh, output_w_init=LI.Uniform(-3e-3, 3e-3), output_b_init=LI.Uniform(-3e-3, 3e-3), bn=False): assert isinstance(env_spec.action_space, Box) Serializable.quick_init(self, locals()) l_obs = L.InputLayer(shape=(None, env_spec.observation_space.flat_dim)) l_hidden = l_obs if bn: l_hidden = batch_norm(l_hidden) for idx, size in enumerate(hidden_sizes): l_hidden = L.DenseLayer( l_hidden, num_units=size, W=hidden_w_init, b=hidden_b_init, nonlinearity=hidden_nonlinearity, name="h%d" % idx) if bn: l_hidden = batch_norm(l_hidden) l_output = L.DenseLayer( l_hidden, num_units=env_spec.action_space.flat_dim, W=output_w_init, b=output_b_init, nonlinearity=output_nonlinearity, name="output") # Note the deterministic=True argument. It makes sure that when getting # actions from single observations, we do not update params in the # batch normalization layers action_var = L.get_output(l_output, deterministic=True) self._output_layer = l_output self._f_actions = tensor_utils.compile_function([l_obs.input_var], action_var) super(DeterministicMLPPolicy, self).__init__(env_spec) LasagnePowered.__init__(self, [l_output])
def build_network(self): l_char1_in = L.InputLayer(shape=(None, None, self.max_word_len), input_var=self.inps[0]) l_char2_in = L.InputLayer(shape=(None, None, self.max_word_len), input_var=self.inps[1]) l_mask1_in = L.InputLayer(shape=(None, None, self.max_word_len), input_var=self.inps[2]) l_mask2_in = L.InputLayer(shape=(None, None, self.max_word_len), input_var=self.inps[3]) l_char_in = L.ConcatLayer([l_char1_in, l_char2_in], axis=1) # B x (ND+NQ) x L l_char_mask = L.ConcatLayer([l_mask1_in, l_mask2_in], axis=1) shp = (self.inps[0].shape[0], self.inps[0].shape[1] + self.inps[1].shape[1], self.inps[1].shape[2]) l_index_reshaped = L.ReshapeLayer(l_char_in, (shp[0] * shp[1], shp[2])) # BN x L l_mask_reshaped = L.ReshapeLayer(l_char_mask, (shp[0] * shp[1], shp[2])) # BN x L l_lookup = L.EmbeddingLayer(l_index_reshaped, self.num_chars, self.char_dim) # BN x L x D l_fgru = L.GRULayer(l_lookup, 2 * self.char_dim, grad_clipping=10, gradient_steps=-1, precompute_input=True, only_return_final=True, mask_input=l_mask_reshaped) l_bgru = L.GRULayer(l_lookup, 2 * self.char_dim, grad_clipping=10, gradient_steps=-1, precompute_input=True, backwards=True, only_return_final=True, mask_input=l_mask_reshaped) # BN x 2D l_fwdembed = L.DenseLayer(l_fgru, self.embed_dim / 2, nonlinearity=None) # BN x DE l_bckembed = L.DenseLayer(l_bgru, self.embed_dim / 2, nonlinearity=None) # BN x DE l_embed = L.ElemwiseSumLayer([l_fwdembed, l_bckembed], coeffs=1) l_char_embed = L.ReshapeLayer(l_embed, (shp[0], shp[1], self.embed_dim / 2)) l_embed1 = L.SliceLayer(l_char_embed, slice(0, self.inps[0].shape[1]), axis=1) l_embed2 = L.SliceLayer(l_char_embed, slice(-self.inps[1].shape[1], None), axis=1) return l_embed1, l_embed2
def BatchNormRecurrentLayer(incoming, num_units, nonlinearity=None, gradient_steps=-1, grad_clipping=0, layer_type=layers.CustomRecurrentLayer, name='', **kwargs): """ Helper method to define a Vanilla Recurrent Layer with batch normalization """ input_shape = incoming.output_shape # Define input to hidden connections in_to_hid_rf = layers.InputLayer((None, ) + input_shape[2:]) in_to_hid_rf = layers.DenseLayer(in_to_hid_rf, num_units, b=None, nonlinearity=None, name='ith_{0}'.format(name)) in_to_hid_rf_W = in_to_hid_rf.W # Use batch normalization in the input to hidden connections in_to_hid_rf = layers.BatchNormLayer(in_to_hid_rf, name='ith_bn_{0}'.format(name)) # Define hidden to hidden connections hid_to_hid_rf = layers.InputLayer((None, num_units)) hid_to_hid_rf = layers.DenseLayer(hid_to_hid_rf, num_units, b=None, nonlinearity=None, name='hth_{0}'.format(name)) l_r_f = layer_type(incoming, input_to_hidden=in_to_hid_rf, hidden_to_hidden=hid_to_hid_rf, gradient_steps=gradient_steps, grad_clipping=grad_clipping, nonlinearity=nonlinearity, name='l_r_{0}'.format(name), **kwargs) # Make layer parameters intuitively accessible l_r_f.W_in_to_hid = in_to_hid_rf_W l_r_f.W_hid_to_hid = hid_to_hid_rf.W l_r_f.beta = in_to_hid_rf.beta l_r_f.gamma = in_to_hid_rf.gamma l_r_f.mean = in_to_hid_rf.mean l_r_f.inv_std = in_to_hid_rf.inv_std l_r_f.hid_init = l_r_f.hid_init return l_r_f
def get_discriminator(self): ''' specify discriminator D0 ''' """ disc0_layers = [LL.InputLayer(shape=(self.args.batch_size, 3, 32, 32))] disc0_layers.append(LL.GaussianNoiseLayer(disc0_layers[-1], sigma=0.05)) disc0_layers.append(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu)) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 16x16 disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1)) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu))) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 8x8 disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1)) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=0, W=Normal(0.02), nonlinearity=nn.lrelu))) # 6x6 disc0_layer_shared = LL.NINLayer(disc0_layers[-1], num_units=192, W=Normal(0.02), nonlinearity=nn.lrelu) # 6x6 disc0_layers.append(disc0_layer_shared) disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared, num_units=50, W=Normal(0.02), nonlinearity=None) disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared)) disc0_layer_adv = LL.DenseLayer(disc0_layers[-1], num_units=10, W=Normal(0.02), nonlinearity=None) disc0_layers.append(disc0_layer_adv) return disc0_layers, disc0_layer_adv, disc0_layer_z_recon """ disc_x_layers = [LL.InputLayer(shape=(None, 3, 32, 32))] disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2)) disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu) disc_x_layers.append(disc_x_layers_shared) disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=self.args.z0dim, nonlinearity=None) disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x # disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100)) disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared)) disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None) disc_x_layers.append(disc_x_layer_adv) #output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False) #output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False) # temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True) # temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True) # init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])] return disc_x_layers, disc_x_layer_adv, disc_x_layer_z_recon
def build_network(self, ra_input_var, mc_input_var): print('Building raw network with parameters:') pp = pprint.PrettyPrinter(indent=4) pp.pprint(self.net_opts) ra_network_1 = layers.InputLayer((None, 1, None), ra_input_var) ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_1', dropout=False, pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_1') ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_2', pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_2') ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_3', pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_3') ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_4', pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_4') concat_list = [ra_network_1] mc_input = layers.InputLayer((None, 2, None), mc_input_var) concat_list.append(mc_input) network = layers.ConcatLayer(concat_list, axis=1, cropping=[None, None, 'center']) network = self.set_conv_layer(network, 'conv_1') network = self.set_pool_layer(network, 'pool_1') network = self.set_conv_layer(network, 'conv_2') network = self.set_pool_layer(network, 'pool_2') network = self.set_conv_layer(network, 'conv_3') network = layers.GlobalPoolLayer( network, getattr(T, self.net_opts['global_pool_func'])) # print(layers.get_output_shape(network)) # network = layers.DenseLayer(layers.dropout(network, p=self.net_opts['dropout_p']), # self.net_opts['dens_1'], # nonlinearity=lasagne.nonlinearities.rectify) network = layers.DenseLayer( layers.dropout(network, p=self.net_opts['dropout_p']), self.net_opts['dens_2'], nonlinearity=lasagne.nonlinearities.rectify) network = layers.DenseLayer( layers.dropout(network, p=self.net_opts['dropout_p']), self.net_opts['num_class'], nonlinearity=lasagne.nonlinearities.softmax) # print(layers.get_output_shape(network)) self.network = network return self.network
def build_architecture(self): l_out = layers.InputLayer(shape = tuple([None] + self.input_dim), input_var = self.input_var) l_out = layers.DenseLayer(l_out, num_units = 10, nonlinearity = lasagne.nonlinearities.sigmoid) self.l_out1 = l_out # 3 output classes # l_out = layers.DenseLayer(l_out, num_units = 3, nonlinearity = lasagne.nonlinearities.softmax) print self.b.get_value() l_out = layers.DenseLayer(incoming = l_out, num_units = 3, b = self.b, nonlinearity = lasagne.nonlinearities.softmax) # print type(self.b) self.l_out = l_out
def __build_24_calib_net__(self): network = layers.InputLayer((None, 3, 24, 24), input_var=self.__input_var__) network = layers.Conv2DLayer(network, num_filters=32, filter_size=(5, 5), stride=1, nonlinearity=relu) network = layers.MaxPool2DLayer(network, pool_size=(3, 3), stride=2) network = layers.DenseLayer(network, num_units=64, nonlinearity=relu) network = layers.DenseLayer(network, num_units=45, nonlinearity=softmax) return network
def __build_12_net__(self): network = layers.InputLayer((None, 3, 12, 12), input_var=self.__input_var__) network = layers.Conv2DLayer(network, num_filters=16, filter_size=(3, 3), stride=1, nonlinearity=relu) network = layers.MaxPool2DLayer(network, pool_size=(3, 3), stride=2) network = layers.DropoutLayer(network) network = layers.DenseLayer(network, num_units=16, nonlinearity=relu) #network = layers.Conv2DLayer(network,num_filters=16,filter_size=(1,1),stride=1,nonlinearity=relu) network = layers.DenseLayer(network, num_units=2, nonlinearity=softmax) return network
def build_network(state_shape, num_actions): """Builds a network with input size state_shape & num_actions output. As problem is discrete, we predict the Q-value for all possible actions. """ input_shape = (None, ) + state_shape W_init = lasagne.init.GlorotUniform() nonlin = lasagne.nonlinearities.rectify network = nn.InputLayer(input_shape, input_var=None) network = nn.DenseLayer(network,50, W=W_init, nonlinearity=nonlin) network = nn.DenseLayer(network, num_actions, W=W_init, nonlinearity=None) return network
def buildModel(): #this is our input layer with the inputs (None, dimensions, width, height) l_input = layers.InputLayer((None, 3, 32, 32)) #first convolutional layer, has l_input layer as incoming and is followed by a pooling layer l_conv1 = layers.Conv2DLayer(l_input, num_filters=32, filter_size=3, pad='same', nonlinearity=tanh) l_pool1 = layers.MaxPool2DLayer(l_conv1, pool_size=2) #second convolution (l_pool1 is incoming), let's increase the number of filters l_conv2 = layers.Conv2DLayer(l_pool1, num_filters=64, filter_size=3, pad='same', nonlinearity=tanh) l_pool2 = layers.MaxPool2DLayer(l_conv2, pool_size=2) #third convolution (l_pool2 is incoming), even more filters l_conv3 = layers.Conv2DLayer(l_pool2, num_filters=128, filter_size=3, pad='same', nonlinearity=tanh) l_pool3 = layers.MaxPool2DLayer(l_conv3, pool_size=2) #fourth and final convolution l_conv4 = layers.Conv2DLayer(l_pool3, num_filters=256, filter_size=3, pad='same', nonlinearity=tanh) l_pool4 = layers.MaxPool2DLayer(l_conv4, pool_size=2) #our cnn contains 3 dense layers, one of them is our output layer l_dense1 = layers.DenseLayer(l_pool4, num_units=128, nonlinearity=tanh) l_dense2 = layers.DenseLayer(l_dense1, num_units=128, nonlinearity=tanh) #the output layer has 5 units which is exactly the count of our class labels #it has a softmax activation function, its values represent class probabilities l_output = layers.DenseLayer(l_dense2, len(tagMap), nonlinearity=softmax) #let's see how many params our net has print("MODEL HAS", layers.count_params(l_output), "PARAMS") #we return the layer stack as our network by returning the last layer return l_output
def buildModel(): # Theinput layer with the inputs (None, dimensions, width, height) l_input = layers.InputLayer((None, 3, 32, 32)) # First convolutional layer, has l_input layer as incoming and is followed by a pooling layer, filters = 16 l_conv1 = layers.Conv2DLayer(l_input, num_filters=16, filter_size=3, pad='same', nonlinearity=tanh) l_pool1 = layers.MaxPool2DLayer(l_conv1, pool_size=2) # The second convolution (l_pool1 is incoming), filters = 32 l_conv2 = layers.Conv2DLayer(l_pool1, num_filters=32, filter_size=3, pad='same', nonlinearity=tanh) l_pool2 = layers.MaxPool2DLayer(l_conv2, pool_size=2) # The third convolution (l_pool2 is incoming), filters = 64 l_conv3 = layers.Conv2DLayer(l_pool2, num_filters=64, filter_size=3, pad='same', nonlinearity=tanh) l_pool3 = layers.MaxPool2DLayer(l_conv3, pool_size=2) # The fourth and final convolution, filters = 128 l_conv4 = layers.Conv2DLayer(l_pool3, num_filters=128, filter_size=3, pad='same', nonlinearity=tanh) l_pool4 = layers.MaxPool2DLayer(l_conv4, pool_size=2) # The CNN contains 3 dense layers, one of them is the output layer l_dense1 = layers.DenseLayer(l_pool4, num_units=64, nonlinearity=tanh) l_dense2 = layers.DenseLayer(l_dense1, num_units=64, nonlinearity=tanh) # The output layer has 43 units which is exactly the count of our class labels # It has a softmax activation function, its values represent class probabilities l_output = layers.DenseLayer(l_dense2, num_units=43, nonlinearity=softmax) #print "The CNN model has ", layers.count_params(l_output), "parameters" # Returning the layer stack by returning the last layer return l_output
def __init__(self, input_shape, output_dim, hidden_sizes, hidden_nonlinearity, output_nonlinearity, hidden_W_init=LI.GlorotUniform(), hidden_b_init=LI.Constant(0.), output_W_init=LI.GlorotUniform(), output_b_init=LI.Constant(0.), name=None, input_var=None, input_layer=None): if name is None: prefix = "" else: prefix = name + "_" if input_layer is None: l_in = L.InputLayer(shape=(None, ) + input_shape, input_var=input_var) else: l_in = input_layer self._layers = [l_in] l_hid = l_in for idx, hidden_size in enumerate(hidden_sizes): l_hid = L.DenseLayer( l_hid, num_units=hidden_size, nonlinearity=hidden_nonlinearity, name="%shidden_%d" % (prefix, idx), W=hidden_W_init, b=hidden_b_init, ) self._layers.append(l_hid) l_out = L.DenseLayer( l_hid, num_units=output_dim, nonlinearity=output_nonlinearity, name="%soutput" % (prefix, ), W=output_W_init, b=output_b_init, ) self._layers.append(l_out) self._l_in = l_in self._l_out = l_out self._input_var = l_in.input_var self._output = L.get_output(l_out)
def enc_net(_incoming, output_channels, drop_rate=0.3, nonlinearity=None): # #_noise = L.GaussianNoiseLayer(_incoming, sigma=0.1) _drop1 = L.DropoutLayer(_incoming, p=drop_rate, rescale=True) _fc1 = L.DenseLayer(_drop1, 4 * output_channels, W=I.Normal(0.02), b=I.Constant(0.1), nonlinearity=NL.rectify) _drop2 = L.DropoutLayer(_fc1, p=drop_rate, rescale=True) _fc2 = L.DenseLayer(_drop2, output_channels, W=I.Normal(0.02), b=I.Constant(0.1), nonlinearity=nonlinearity) return _fc2
def _forward(self): net = {} net['input'] = layers.InputLayer(shape=(None, 1, 28, 28), input_var=self.X) net['dense'] = layers.DenseLayer(net['input'], num_units=2048, W=init.GlorotUniform(), b=init.Constant(0.), nonlinearity=nonlinearities.rectify) net['out'] = layers.DenseLayer(net['dense'], num_units=10, W=init.GlorotUniform(), b=None, nonlinearity=nonlinearities.softmax) return net