def pons_cnn(params): """""" layers = L.InputLayer((None, 1, params['dur'], 128)) print layers.output_shape sclr = joblib.load(params['scaler']) layers = L.standardize(layers, sclr.mean_.astype(np.float32), sclr.scale_.astype(np.float32), shared_axes=(0, 1, 2)) print layers.output_shape layers_timbre = L.GlobalPoolLayer( L.batch_norm(L.Conv2DLayer(layers, 64, (1, 96)))) layers_rhythm = L.GlobalPoolLayer( L.batch_norm(L.Conv2DLayer(layers, 64, (params['dur'] - 10, 1)))) layers = L.ConcatLayer([layers_rhythm, layers_timbre], axis=-1) layers = L.DenseLayer(layers, 64, nonlinearity=nl.rectify) print layers.output_shape layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax) print layers.output_shape return layers
def get_feature_extractor_trans(model_D, model_bin, x_temp=T.tensor4()): features = ll.get_output(ll.GlobalPoolLayer(model_D), x_temp, deterministic=True) features = features / (T.abs_(features) + 0.001) features = ll.get_output(model_bin, features, deterministic=False) return th.function(inputs=[x_temp], outputs=features)
def output_block(net, config, non_lin, verbose=True): """ """ # output setting out_acts = [] for out_act in config.hyper_parameters.out_act: exec('from lasagne.nonlinearities import {}'.format(out_act)) out_acts.append(eval(out_act)) n_outs = config.hyper_parameters.n_out # Global Average Pooling last_conv_block_name = next(reversed(net)) net['gap'] = L.GlobalPoolLayer(net[last_conv_block_name], name='gap') net['gap.bn'] = L.BatchNormLayer(net['gap'], name='gap.bn') n_features = net['gap.bn'].output_shape[-1] # feature Layer net['fc'] = L.dropout(L.batch_norm( L.DenseLayer(net['gap.bn'], num_units=n_features, nonlinearity=non_lin, name='fc')), name='fc.bn.do') # output (prediction) # check whether the model if for MTL or STL # target is passed as list, regardless whether # it's MTL or STL (configuration checker checks it) targets = config.target out_layer_names = [] for target, n_out, out_act in zip(targets, n_outs, out_acts): out_layer_names.append('out.{}'.format(target)) if target == 'self': net[out_layer_names[-1]], inputs = build_siamese(net['fc']) else: net[out_layer_names[-1]] = L.DenseLayer(net['fc'], num_units=n_out, nonlinearity=out_act, name=out_layer_names[-1]) inputs = [net['input'].input_var] # make a concatation layer just for save/load purpose net['IO'] = L.ConcatLayer([ L.FlattenLayer(net[target_layer_name]) if target == 'self' else net[target_layer_name] for target_layer_name in out_layer_names ], name='IO') if verbose: print(net['gap.bn'].output_shape) print(net['fc'].output_shape) for target in targets: print(net['out.{}'.format(target)].output_shape) return net, inputs
def build_network(self, ra_input_var, mc_input_var): print('Building raw network with parameters:') pp = pprint.PrettyPrinter(indent=4) pp.pprint(self.net_opts) ra_network_1 = layers.InputLayer((None, 1, None), ra_input_var) ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_1', dropout=False, pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_1') ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_2', pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_2') ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_3', pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_3') ra_network_1 = self.set_conv_layer(ra_network_1, 'ra_conv_4', pad='same') ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_4') concat_list = [ra_network_1] mc_input = layers.InputLayer((None, 2, None), mc_input_var) concat_list.append(mc_input) network = layers.ConcatLayer(concat_list, axis=1, cropping=[None, None, 'center']) network = self.set_conv_layer(network, 'conv_1') network = self.set_pool_layer(network, 'pool_1') network = self.set_conv_layer(network, 'conv_2') network = self.set_pool_layer(network, 'pool_2') network = self.set_conv_layer(network, 'conv_3') network = layers.GlobalPoolLayer( network, getattr(T, self.net_opts['global_pool_func'])) # print(layers.get_output_shape(network)) # network = layers.DenseLayer(layers.dropout(network, p=self.net_opts['dropout_p']), # self.net_opts['dens_1'], # nonlinearity=lasagne.nonlinearities.rectify) network = layers.DenseLayer( layers.dropout(network, p=self.net_opts['dropout_p']), self.net_opts['dens_2'], nonlinearity=lasagne.nonlinearities.rectify) network = layers.DenseLayer( layers.dropout(network, p=self.net_opts['dropout_p']), self.net_opts['num_class'], nonlinearity=lasagne.nonlinearities.softmax) # print(layers.get_output_shape(network)) self.network = network return self.network
def get_discriminator(self): ''' specify discriminator D0 ''' """ disc0_layers = [LL.InputLayer(shape=(self.args.batch_size, 3, 32, 32))] disc0_layers.append(LL.GaussianNoiseLayer(disc0_layers[-1], sigma=0.05)) disc0_layers.append(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu)) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 16x16 disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1)) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu))) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 8x8 disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1)) disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=0, W=Normal(0.02), nonlinearity=nn.lrelu))) # 6x6 disc0_layer_shared = LL.NINLayer(disc0_layers[-1], num_units=192, W=Normal(0.02), nonlinearity=nn.lrelu) # 6x6 disc0_layers.append(disc0_layer_shared) disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared, num_units=50, W=Normal(0.02), nonlinearity=None) disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared)) disc0_layer_adv = LL.DenseLayer(disc0_layers[-1], num_units=10, W=Normal(0.02), nonlinearity=None) disc0_layers.append(disc0_layer_adv) return disc0_layers, disc0_layer_adv, disc0_layer_z_recon """ disc_x_layers = [LL.InputLayer(shape=(None, 3, 32, 32))] disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2)) disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu) disc_x_layers.append(disc_x_layers_shared) disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=self.args.z0dim, nonlinearity=None) disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x # disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100)) disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared)) disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None) disc_x_layers.append(disc_x_layer_adv) #output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False) #output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False) # temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True) # temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True) # init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])] return disc_x_layers, disc_x_layer_adv, disc_x_layer_z_recon
def build_dist_feat_fnc(net, target, conv_feat_locs=[5, 10, 12, 17, 19], fc_feat_locs=[24, 28]): """""" layers = L.get_all_layers(net[target]) assert len(layers) == 30 # only works for standard deep conv2d feat = [L.GlobalPoolLayer(layers[l]) for l in conv_feat_locs] feat += [layers[l] for l in fc_feat_locs] feat = L.ConcatLayer(feat, axis=1) f = L.get_output(feat, deterministic=True) f_feat = {target: {}} f_feat[target]['transform'] = theano.function([layers[0].input_var], f, allow_input_downcast=True) return f_feat
pad=0, W=Normal(0.02), nonlinearity=nn.lrelu))) # 6x6 disc0_layer_shared = LL.NINLayer(disc0_layers[-1], num_units=192, W=Normal(0.02), nonlinearity=nn.lrelu) # 6x6 disc0_layers.append(disc0_layer_shared) disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared, num_units=16, W=Normal(0.02), nonlinearity=None) disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared)) disc0_layer_adv = LL.DenseLayer(disc0_layers[-1], num_units=10, W=Normal(0.02), nonlinearity=None) disc0_layers.append(disc0_layer_adv) ''' forward pass ''' output_before_softmax_real0 = LL.get_output(disc0_layer_adv, x, deterministic=False) output_before_softmax_gen0, recon_z0 = LL.get_output( [disc0_layer_adv, disc0_layer_z_recon], gen_x, deterministic=False ) # discriminator's predicted probability that gen_x is real ''' loss for discriminator and Q '''
def pc_fcn(input_var, early_conv_dict, middle_conv_dict, pl_dict, late_conv_dict, dense_filter_size, final_pool_function=T.max, input_size=128, output_size=188, p_dropout=0.5): ''' early_conv_dict_list: list each element in the list is a dictionary containing the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' pl_dict: dict it contains the following keys: 'num_lambda', 'num_points', 'value_range', 'seg_size', 'seg_stride' late_conv_dict: dict it contains the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' dense_filter_size: int the filter size of the final dense-like conv layer pool_filter_list: list each element is an integer pool_stride_list: list each element is int or None ''' # early conv layers input_network = lasagne.layers.InputLayer(shape=(None, 1, None, input_size), input_var=input_var) total_stride = 1 network, total_stride = conv_layers(input_network, early_conv_dict, total_stride, init_input_size=input_size, p_dropout=0, base_name='early') # middle conv layers (dense layers) network_c, _ = conv_layers(network, middle_conv_dict, total_stride, init_input_size=1, p_dropout=0, base_name='middle_conv') # Persistence landscape network_p = network try: num_lambda = pl_dict['num_lambda'] except: num_lambda = pl_dict['n_f_db'] try: num_points = pl_dict['num_points'] except: num_points = pl_dict['n_points'] value_range = pl_dict['value_range'] seg_size = pl_dict['seg_size'] seg_step = pl_dict['seg_step'] patch_size = (seg_size, 1) patch_step = (seg_step, 1) network_p = cl.PersistenceFlatten2DLayer(network_p, num_lambda, num_points, value_range, patch_size, patch_step) # Convolution+Persistence network = layers.ConcatLayer([network_c, network_p], axis=1, cropping=[None, None, 'lower', None], name='Convolution+Persistence') # late conv layers (dense layers) network, total_stride = conv_layers(network, late_conv_dict, total_stride, init_input_size=1, p_dropout=p_dropout, base_name='late') # frame output layer. every frame has a value network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout), num_filters=output_size, filter_size=(dense_filter_size, 1), nonlinearity=lasagne.nonlinearities.sigmoid, W=lasagne.init.GlorotUniform()) # pool network = layers.GlobalPoolLayer(network, pool_function=final_pool_function) network = layers.ReshapeLayer(network, ([0], -1)) return network
def deep_cnn_2d(params): """""" nonlin = nl.elu layers = L.InputLayer((None, 1, params['dur'], 128)) print layers.output_shape sclr = joblib.load(params['scaler']) layers = L.standardize(layers, sclr.mean_.astype(np.float32), sclr.scale_.astype(np.float32), shared_axes=(0, 1, 2)) print layers.output_shape n_filter = [16, 32, 64, 64, 128, 256, 256] # l filter_sz = [(5, 5), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (1, 1)] # m if params['dur'] > 50: conv_strd = [(2, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] # c pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), (2, 2), None, None] # n else: conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] # c pool_sz = [(1, 2), (2, 2), (2, 2), (2, 2), (2, 2), None, None] # n pool_strd = [None, None, None, None, None, None, None] # s batch_norm = [False, True, False, True, False, False, False] # b dropout = [True, True, False, True, False, False, False] # d # added conv_spec = zip(n_filter, filter_sz, conv_strd, pool_sz, pool_strd, batch_norm, dropout) for l, m, c, n, s, b, d in conv_spec: if b: layers = L.batch_norm( L.Conv2DLayer(layers, l, m, stride=c, pad='same', nonlinearity=nonlin), ) else: layers = L.Conv2DLayer(layers, l, m, stride=c, pad='same', nonlinearity=nonlin) if n is not None: layers = L.MaxPool2DLayer(layers, pool_size=n, stride=s) if d: layers = L.dropout(layers, p=0.1) print layers.output_shape layers = L.batch_norm(L.GlobalPoolLayer(layers)) layers = L.dropout(layers) # added print layers.output_shape layers = L.batch_norm(L.DenseLayer(layers, 256, nonlinearity=nonlin)) layers = L.dropout(layers) print layers.output_shape layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax) print layers.output_shape return layers
dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-00')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-01')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 32, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-02'), name='dis-03')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-20')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 32, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-21'), name='dis-22')) dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-23')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-30')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 64, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-31'), name='dis-32')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-40')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 64, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-41'), name='dis-42')) dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-43')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-50')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 128, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-51'), name='dis-52')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-60')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 128, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-61'), name='dis-62')) dis_layers.append(ll.GlobalPoolLayer(dis_layers[-1], name='dis-63')) dis_layers.append(MLPConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-70')) dis_layers.append(nn.weight_norm(ll.DenseLayer(dis_layers[-1], num_units=1, W=Normal(0.05), nonlinearity=ln.sigmoid, name='dis-71'), train_g=True, init_stdv=0.1, name='dis-72')) # inference module inf_in_x = ll.InputLayer(shape=(None, in_channels) + dim_input) inf_layers = [inf_in_x] inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 64, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-02'), name='inf-03')) inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 128, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-11'), name='inf-12')) inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 256, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-21'), name='inf-22')) inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 512, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-31'), name='inf-32')) inf_layers.append(ll.DenseLayer(inf_layers[-1], num_units=n_z, W=Normal(0.05), nonlinearity=None, name='inf-4')) # discriminator xz disxz_in_x = ll.InputLayer(shape=(None, in_channels) + dim_input) disxz_in_z = ll.InputLayer(shape=(None, n_z))
# specify discriminative model disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))] disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1])) disc_layers.append(nn.weight_norm(ll.DenseLayer(disc_layers[-1], num_units=16, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1)) disc_params = ll.get_all_params(disc_layers, trainable=True) x_temp = T.tensor4() temp = ll.get_output(gen_layers[-1], deterministic=False, init=True) temp = ll.get_output(disc_layers[-1], x_temp, deterministic=False, init=True) init_updates = [u for l in gen_layers+disc_layers for u in getattr(l,'init_updates',[])] init_param = th.function(inputs=[x_temp], outputs=None, updates=init_updates) # costs labels = T.ivector() x_lab = T.tensor4() x_unl = T.tensor4()
def buildNet(): log.p('BUILDING BirdNET MODEL...', new_line=False) # Input layer for images net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0])) # Pre-processing stage #log.p(("\tPRE-PROCESSING STAGE:")) net = l.batch_norm(l.Conv2DLayer(net, num_filters=int(FILTERS[0] * RESNET_K), filter_size=(5, 5), pad='same', nonlinearity=nl.rectify)) #log.p(("\t\tFIRST CONV OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Max pooling net = l.MaxPool2DLayer(net, pool_size=(1, 2)) #log.p(("\t\tPRE-MAXPOOL OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Residual Stacks for i in range(1, len(FILTERS)): #log.p(("\tRES STACK", i, ':')) net = resblock(net, filters=int(FILTERS[i] * RESNET_K), kernel_size=KERNEL_SIZES[i], stride=2, preactivated=True, block_id=i, name='BLOCK ' + str(i) + '-1') for j in range(1, RESNET_N): net = resblock(net, filters=int(FILTERS[i] * RESNET_K), kernel_size=KERNEL_SIZES[i], preactivated=False, block_id=i+j, name='BLOCK ' + str(i) + '-' + str(j + 1)) # Post Activation net = l.batch_norm(net) net = l.NonlinearityLayer(net, nonlinearity=nl.rectify) # Classification branch #log.p(("\tCLASS BRANCH:")) net = classificationBranch(net, (4, 10)) #log.p(("\t\tBRANCH OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Pooling net = l.GlobalPoolLayer(net, pool_function=logmeanexp) #log.p(("\tGLOBAL POOLING SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Sigmoid output net = l.NonlinearityLayer(net, nonlinearity=nl.sigmoid) #log.p(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)))) log.p("DONE!") # Model stats #log.p(("MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS")) #log.p(("MODEL HAS", l.count_params(net), "PARAMS")) return net
def build_baseline_model(): log.i('BUILDING BASELINE MODEL...') # Random Seed lasagne_random.set_rng(cfg.getRandomState()) # Input layer for images net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0])) # Stride size (as an alternative to max pooling) if cfg.MAX_POOLING: s = 1 else: s = 2 # Convolutinal layer groups for i in range(len(cfg.FILTERS)): # 3x3 Convolution + Stride net = batch_norm( l.Conv2DLayer(net, num_filters=cfg.FILTERS[i], filter_size=cfg.KERNEL_SIZES[i], num_groups=cfg.NUM_OF_GROUPS[i], pad='same', stride=s, W=initialization(cfg.NONLINEARITY), nonlinearity=nonlinearity(cfg.NONLINEARITY))) # Pooling layer if cfg.MAX_POOLING: net = l.MaxPool2DLayer(net, pool_size=2) # Dropout Layer (we support different types of dropout) if cfg.DROPOUT_TYPE == 'channels' and cfg.DROPOUT > 0.0: net = l.dropout_channels(net, p=cfg.DROPOUT) elif cfg.DROPOUT_TYPE == 'location' and cfg.DROPOUT > 0.0: net = l.dropout_location(net, p=cfg.DROPOUT) elif cfg.DROPOUT > 0.0: net = l.DropoutLayer(net, p=cfg.DROPOUT) log.i(('\tGROUP', i + 1, 'OUT SHAPE:', l.get_output_shape(net))) # Final 1x1 Convolution net = batch_norm( l.Conv2DLayer(net, num_filters=cfg.FILTERS[i] * 2, filter_size=1, W=initialization('identity'), nonlinearity=nonlinearity('identity'))) log.i(('\tFINAL CONV OUT SHAPE:', l.get_output_shape(net))) # Global Pooling layer (default mode = average) net = l.GlobalPoolLayer(net) log.i(("\tFINAL POOLING SHAPE:", l.get_output_shape(net))) # Classification Layer (Softmax) net = l.DenseLayer(net, len(cfg.CLASSES), nonlinearity=nonlinearity('softmax'), W=initialization('softmax')) log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net))) log.i("...DONE!") # Model stats log.i(("MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS")) log.i(("MODEL HAS", l.count_params(net), "PARAMS")) return net
def dmr_regularizer(f_1, f2): x_bin = f_1 / (T.abs_(f_1) + args.gamma) x_bin_2 = T.sgn(f2) M_reg = (T.dot(x_bin_2, x_bin_2.T) - T.dot(x_bin_2, x_bin_2.T) * (T.eye(x_bin_2.shape[0]))) / x_bin_2.shape[1] M_true = (T.dot(x_bin, x_bin.T) - T.dot(x_bin, x_bin.T) * (T.eye(x_bin.shape[0]))) / x_bin.shape[1] l_reg = T.sum(T.abs_(M_reg - M_true)) / (x_bin.shape[0] * (x_bin.shape[0] - 1)) return l_reg if args.dataset_type == 'brown': global_pool = ll.GlobalPoolLayer(disc_layers[f_low_dim]) global_pool_2 = ll.ReshapeLayer(disc_layers[f_high_dim], ([0], -1)) else: global_pool = disc_layers[f_low_dim] global_pool_2 = ll.GlobalPoolLayer(disc_layers[f_high_dim]) features_1 = ll.get_output(global_pool, x_unl_1, deterministic=False) features_2 = ll.get_output(global_pool_2, x_unl_1, deterministic=False) loss_unl = loss_unl + args.lambda_dmr * dmr_regularizer(features_1, features_2) \ + args.lambda_bre * bre_regularizer(features_1, features_2) # Theano functions for training the disc net lr = T.scalar() disc_params = ll.get_all_params(disc_layers, trainable=True) disc_param_updates = nn.adam_updates(disc_params, loss_unl, lr=lr, mom1=0.5) disc_param_avg = [
def build(self): """""" # output setting out_acts = [] for out_act in self.config.hyper_parameters.out_act: # exec('from lasagne.nonlinearities import {}'.format(out_act)) out_acts.append(eval(out_act)) n_outs = self.config.hyper_parameters.n_out targets = self.config.target # apply size multiplier self.n_filters = map(lambda x: x * self.m, self.n_filters) # network dict & set input block self.net, self.variables['sigma'] = input_block(self.net, self.config, melspec=True) if self.branch_at == 'fc': # shares all layers except output layer for i in range(len(self.n_convs)): name = 'conv{:d}'.format(i + 1) self.net = conv_block(self.net, self.n_convs[i], self.n_filters[i], self.filter_sizes[i], self.strides[i], self.pool_sizes[i], self.non_lin, self.batch_norm, name, self.net.keys()[-1], self.verbose) # GAP self.net['gap'] = L.batch_norm( L.GlobalPoolLayer(self.net[next(reversed(self.net))], name='gap')) self.net['fc'] = L.dropout(L.batch_norm( L.DenseLayer(self.net['gap'], num_units=self.net['gap'].output_shape[-1], nonlinearity=self.non_lin, name='fc')), name='fc.bn.do') # Out block out_layer_names = [] for target, n_out, out_act in zip(targets, n_outs, out_acts): out_layer_names.append('{}.out'.format(target)) if target == 'self': self.net[out_layer_names[-1]], inputs = \ build_siamese(self.net['fc']) else: self.net[out_layer_names[-1]] = L.DenseLayer( self.net['fc'], num_units=n_out, nonlinearity=out_act, name=out_layer_names[-1]) inputs = [self.net['input'].input_var] self.variables['{}.inputs'.format(target)] = inputs else: # shares lower part of the network and branch out # shared conv blocks for i in range(self.branch_at - 1): name = 'conv{:d}'.format(i + 1) self.net = conv_block(self.net, self.n_convs[i], self.n_filters[i], self.filter_sizes[i], self.strides[i], self.pool_sizes[i], self.non_lin, self.batch_norm, name, self.net.keys()[-1], self.verbose) branch_point = self.net.keys()[-1] # branch out to each targets out_layer_names = [] for target, n_out, out_act in zip(targets, n_outs, out_acts): # first conv_block for each branch j = self.branch_at - 1 # branch_point_ix name = '{}.conv{:d}'.format(target, j + 1) self.net = conv_block(self.net, self.n_convs[j], self.n_filters[j], self.filter_sizes[j], self.strides[j], self.pool_sizes[j], self.non_lin, self.batch_norm, name, branch_point, self.verbose) for i in range(self.branch_at, len(self.n_convs)): name = '{}.conv{:d}'.format(target, i + 1) self.net = conv_block(self.net, self.n_convs[i], self.n_filters[i], self.filter_sizes[i], self.strides[i], self.pool_sizes[i], self.non_lin, self.batch_norm, name, self.net.keys()[-1], self.verbose) # GAP gap_name = '{}.gap'.format(target) self.net[gap_name] = L.batch_norm( L.GlobalPoolLayer(self.net[next(reversed(self.net))], name=gap_name)) # FC fc_name = '{}.fc'.format(target) self.net[fc_name] = L.dropout(L.batch_norm( L.DenseLayer(self.net[gap_name], num_units=self.net[gap_name].output_shape[-1], nonlinearity=self.non_lin, name=fc_name)), name=fc_name) # OUT out_layer_names.append('{}.out'.format(target)) if target == 'self': self.net[out_layer_names[-1]], inputs = \ build_siamese(self.net[fc_name]) else: self.net[out_layer_names[-1]] = L.DenseLayer( self.net[fc_name], num_units=n_out, nonlinearity=out_act, name=out_layer_names[-1]) inputs = [self.net['input'].input_var] self.variables['{}.inputs'.format(target)] = inputs # make a concatation layer just for save/load purpose self.net['IO'] = L.ConcatLayer([ L.FlattenLayer(self.net[target_layer_name]) if target == 'self' else self.net[target_layer_name] for target_layer_name in out_layer_names ], name='IO') if self.verbose: for target in targets: print(self.net['{}.out'.format(target)].output_shape) return self.net, self.variables
def fcn_multiscale( input_var_list, early_conv_dict_list, late_conv_dict, dense_filter_size, final_pool_function=T.max, input_size_list=[128], output_size=188, p_dropout=0.5 ): ''' early_conv_dict_list: list each element in the list is a dictionary containing the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' late_conv_dict: dict it contains the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' dense_filter_size: int the filter size of the final dense-like conv layer pool_filter_list: list each element is an integer pool_stride_list: list each element is int or None ''' assert(len(early_conv_dict_list) == len(input_var_list) == len(input_size_list)) # early conv layers conv_network_list = list() total_stride_list = list() for jj, [early_conv_dict, input_var, input_size] in enumerate(zip( early_conv_dict_list, input_var_list, input_size_list)): input_network = lasagne.layers.InputLayer( shape=(None, 1, None, input_size), input_var=input_var) total_stride = 1 network, total_stride = conv_layers(input_network, early_conv_dict, total_stride, init_input_size=input_size, p_dropout=0, base_name='early{}'.format(jj)) total_stride_list.append(total_stride) conv_network_list.append(network) # Concatenate network = layers.ConcatLayer(conv_network_list, axis=1, cropping=[None, None, 'lower', None], name='MultisourceConcatenate') # late conv layers (dense layers) network, total_stride = conv_layers(network, late_conv_dict, total_stride, init_input_size=1, p_dropout=p_dropout, base_name='late') # frame output layer. every frame has a value network = cl.Conv2DXLayer( lasagne.layers.dropout(network, p=p_dropout), num_filters=output_size, filter_size=(dense_filter_size, 1), nonlinearity=lasagne.nonlinearities.sigmoid, W=lasagne.init.GlorotUniform() ) # pool network = layers.GlobalPoolLayer(network, pool_function=final_pool_function) network = layers.ReshapeLayer(network, ([0], -1)) return network
def build_critic(self, version=1, extra_middle=False, only_middle=False): assert self.generator != None if version == 1: from lasagne.nonlinearities import sigmoid disc_layers = [ ll.InputLayer(shape=(self.batch_size, 3, 64, 64), input_var=self.input_c) ] # b_s x 3 x 64 x 64 --> b_s x 32 x 32 x 32 disc_layers.append( nn.batch_norm( ll.Conv2DLayer(disc_layers[-1], 128, (3, 3), pad=1, stride=2, W=Normal(0.03), nonlinearity=nn.lrelu))) #nn.weight_norm # disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) # b_s x 32 x 32 x 32 --> b_s x 64 x 16 x 16 disc_layers.append( nn.batch_norm( ll.Conv2DLayer(disc_layers[-1], 256, (3, 3), pad=1, stride=2, W=Normal(0.03), nonlinearity=nn.lrelu))) #nn.weight_norm # b_s x 64 x 16 x 16 --> b_s x 128 x 8 x 8 # note : this layer will be used to compare statistics (output of disc_layer[3]) disc_layers.append( nn.batch_norm( ll.Conv2DLayer(disc_layers[-1], 512, (3, 3), pad=1, stride=2, W=Normal(0.03), nonlinearity=nn.lrelu))) #nn.weight_norm # b_s x 128 x 8 x 8 --> b_s x 256 x 4 x 4 disc_layers.append( nn.batch_norm( ll.Conv2DLayer(disc_layers[-1], 1024, (3, 3), pad=1, stride=2, W=Normal(0.03), nonlinearity=nn.lrelu))) #nn.weight_norm disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1])) disc_layers.append( nn.MinibatchLayer(disc_layers[-1], num_kernels=100)) if extra_middle or only_middle: # goal : add mechanism that checks for mode collapse, but only for the middle part layer = nn.ExtractMiddleLayer(disc_layers[0], extra=2) # two convolutions print layer.output_shape layer = nn.batch_norm( ll.Conv2DLayer(layer, 128, 5, stride=2, pad='same', nonlinearity=nn.lrelu)) print layer.output_shape layer = nn.batch_norm( ll.Conv2DLayer(layer, 256, 5, stride=2, pad='same', nonlinearity=nn.lrelu)) print layer.output_shape layer = nn.batch_norm( ll.Conv2DLayer(layer, 512, 5, stride=2, pad='same', nonlinearity=nn.lrelu)) #layer = nn.batch_norm(ll.Conv2DLayer(layer, 1024, 5, stride=2, pad='same', # nonlinearity=nn.lrelu)) #disc_layers.append(layer) #print layer.output_shape layer = ll.GlobalPoolLayer(layer) #print layer.output_shape layer = nn.MinibatchLayer(layer, num_kernels=400) if only_middle: disc_layers = [] disc_layers.append((ll.DenseLayer(layer, num_units=1, W=Normal(0.03), nonlinearity=None))) return disc_layers disc_layers.append(ll.ConcatLayer([layer, disc_layers[-1]])) disc_layers.append((ll.DenseLayer(disc_layers[-1], num_units=1, W=Normal(0.03), nonlinearity=None))) # seeing how there is strong intra-batch normalization, I think it would be good to do minibatch discrimination solely on the center of the images. # I think that the statistics of the border are throwing for layer in disc_layers: print layer.output_shape print '' return disc_layers
def build_fcn_fixedgaussian_multiscale(num_scales): input_var_list = list() earlyconv_list = list() # A stack of convolution layers for each scale for ii in range(num_scales): # input tensor input_var = T.tensor4('input.{}'.format(ii)) # input layer network = lasagne.layers.InputLayer(shape=(None, 1, None, 128), input_var=input_var) # early conv layers network = cl.Conv2DXLayer(network, num_filters=32, filter_size=(8, 128), stride=(1, 1), nonlinearity=lasagne.nonlinearities.rectify, pad='strictsamex', name='early_conv.{}_1'.format(ii)) network = layers.MaxPool2DLayer(network, pool_size=(4, 1), stride=(4, 1), ignore_border=False, name='early_maxpool.{}_1'.format(ii)) network = cl.Conv2DXLayer(network, num_filters=32, filter_size=(8, 1), stride=(1, 1), nonlinearity=lasagne.nonlinearities.rectify, pad='strictsamex', name='early_conv.{}_2'.format(ii)) network = layers.MaxPool2DLayer(network, pool_size=(4, 1), stride=(4, 1), ignore_border=False, name='early_maxpool.{}_2'.format(ii)) input_var_list.append(input_var) earlyconv_list.append(network) network = layers.ConcatLayer(earlyconv_list, axis=1, cropping=[None, None, 'lower', None], name='multiscale_concat') # late conv layers network = layers.Conv2DLayer(layers.dropout(network, p=0.5), 512, (1, 1), (1, 1), name='late_conv.{}_1'.format(ii)) network = layers.Conv2DLayer(layers.dropout(network, p=0.5), 512, (1, 1), (1, 1), name='late_conv.{}_2'.format(ii)) network = layers.Conv2DLayer(layers.dropout(network, p=0.5), 188, (1, 1), (1, 1), nonlinearity=lasagne.nonlinearities.sigmoid, name='late_conv.{}_output'.format(ii)) # Gaussian scanning network = layers.ReshapeLayer(network, ([0], [1], [2])) nogaussian_layer = network network = cl.FixedGaussianScan1DLayer(network, filter_size=256, init_std=256 / 4**2, stride=1, pad='strictsame', name='fixed_gaussian_filter') gaussian_layer = network # pool network = layers.GlobalPoolLayer(network, pool_function=T.mean) return network, input_var_list, nogaussian_layer, gaussian_layer
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0.5, ps=2, n_kerns=32, d_kerns=(5, 5), pad='valid', stride=1, W=Normal(0.05), nonlinearity=ln.rectify, name='cla-1')) cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=64, d_kerns=(3, 3), pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify, name='cla-2')) cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0.5, ps=2, n_kerns=64, d_kerns=(3, 3), pad='valid', stride=1, W=Normal(0.05), nonlinearity=ln.rectify, name='cla-3')) cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=128, d_kerns=(3, 3), pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify, name='cla-4')) cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=128, d_kerns=(3, 3), pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify, name='cla-5')) cla_layers.append(ll.GlobalPoolLayer(cla_layers[-1])) cla_layers.append(ll.DenseLayer(cla_layers[-1], num_units=num_classes, W=lasagne.init.Normal(1e-2, 0), nonlinearity=ln.softmax, name='cla-6')) classifier = cla_layers[-1] # gen_in_z = ll.InputLayer(shape=(None, n_z)) gen_in_y = ll.InputLayer(shape=(None,)) gen_layers = [gen_in_z] gen_layers.append(MLPConcatLayer([gen_layers[-1], gen_in_y], num_classes, name='gen-1')) gen_layers.append(ll.batch_norm(ll.DenseLayer(gen_layers[-1], num_units=500, nonlinearity=ln.softplus, name='gen-2'), name='gen-3')) gen_layers.append(MLPConcatLayer([gen_layers[-1], gen_in_y], num_classes, name='gen-4'))
def get_discriminator_brown(num_feature=256): disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))] disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2)) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3, 3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 128, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 128, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 128, (3, 3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 128, (3, 3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( ll.NINLayer(disc_layers[-1], num_units=num_feature, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( ll.NINLayer(disc_layers[-1], num_units=128, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1])) disc_layers.append( nn.weight_norm(ll.DenseLayer(disc_layers[-1], num_units=2, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1)) #disc_layers.append(ll.ReshapeLayer(disc_layers[-4], ([0], -1))) #disc_layers.append(ll.GlobalPoolLayer(disc_layers[-4])) disc_layer_features_low_dim = -4 disc_layer_features_high_dim = -5 return disc_layers, disc_layer_features_low_dim, disc_layer_features_high_dim
def build_resnet_model(): log.i('BUILDING RESNET MODEL...') # Random Seed lasagne_random.set_rng(cfg.getRandomState()) # Input layer for images net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0])) # First Convolution net = l.Conv2DLayer(net, num_filters=cfg.FILTERS[0], filter_size=cfg.KERNEL_SIZES[0], pad='same', W=initialization(cfg.NONLINEARITY), nonlinearity=None) log.i(("\tFIRST CONV OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Residual Stacks for i in range(0, len(cfg.FILTERS)): net = resblock(net, filters=cfg.FILTERS[i] * cfg.RESNET_K, kernel_size=cfg.KERNEL_SIZES[i], stride=2, num_groups=cfg.NUM_OF_GROUPS[i]) for _ in range(1, cfg.RESNET_N): net = resblock(net, filters=cfg.FILTERS[i] * cfg.RESNET_K, kernel_size=cfg.KERNEL_SIZES[i], num_groups=cfg.NUM_OF_GROUPS[i], preactivated=False) log.i(("\tRES STACK", i + 1, "OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Post Activation net = batch_norm(net) net = l.NonlinearityLayer(net, nonlinearity=nonlinearity(cfg.NONLINEARITY)) # Pooling net = l.GlobalPoolLayer(net) log.i(("\tFINAL POOLING SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1)) # Classification Layer net = l.DenseLayer(net, len(cfg.CLASSES), nonlinearity=nonlinearity('identity'), W=initialization('identity')) net = l.NonlinearityLayer(net, nonlinearity=nonlinearity('softmax')) log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)))) log.i("...DONE!") # Model stats log.i(("MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS")) log.i(("MODEL HAS", l.count_params(net), "PARAMS")) return net
disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2)) disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu) disc_x_layers.append(disc_x_layers_shared) disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=50, nonlinearity=None) disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x # disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100)) disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared)) disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None) disc_x_layers.append(disc_x_layer_adv) #output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False) #output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False) # temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True) # temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True) # init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])] output_before_softmax_real = LL.get_output(disc_x_layer_adv, x, deterministic=False) output_before_softmax_gen, recon_z = LL.get_output([disc_x_layer_adv, disc_x_layer_z_recon], gen_x, deterministic=False) # discriminator's predicted probability that gen_x is real l_lab = output_before_softmax_real[T.arange(args.batch_size),y] l_unl = nn.log_sum_exp(output_before_softmax_real)
nn.batch_norm(dnn.Conv2DDNNLayer(genz_layers[-1], 512, (3, 3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='gz2'), g=None)) genz_layers.append( nn.batch_norm(ll.NINLayer(genz_layers[-1], num_units=512, W=Normal(0.05), nonlinearity=nn.lrelu, name='gz5'), g=None)) genz_layers.append(ll.GlobalPoolLayer(genz_layers[-1], name='gz6')) genz_layers.append( ll.DenseLayer(genz_layers[-1], num_units=100, W=Normal(0.05), nonlinearity=lasagne.nonlinearities.sigmoid, name='gz7')) # specify discriminative model # for z discz_layers = [z_input] discz_layers.append(ll.DropoutLayer(discz_layers[-1], p=0.2)) discz_layers.append( ll.DenseLayer(discz_layers[-1], 500,
stretch=True) img = plotting.plot_img(img_tile, title='') plotting.plt.savefig(args.data_name + "_patches_gen.png") img_bhwc = np.transpose(trainx[:100, ], (0, 2, 3, 1)) img_tile = plotting.img_tile(img_bhwc, aspect_ratio=1.0, border_color=1.0, stretch=True) img = plotting.plot_img(img_tile, title='') plotting.plt.savefig(args.data_name + "_patches_train.png") evens = [x for x in range(int(test_matched.shape[0] / 2)) if x % 2 == 0] odds = [x for x in range(int(test_matched.shape[0] / 2)) if x % 2 == 1] disc_layers.append(ll.GlobalPoolLayer(disc_layers[f_low_dim])) batch_size_test = 100 print('Extracting features from matched data...') features_matched = extract_features(disc_layers, test_matched, batch_size_test) print('Extracting features from unmatched data...') features_unmatched = extract_features(disc_layers, test_unmatched, batch_size_test) print('Number of features considered in the experiment: ' + str(features_matched[evens].shape[1])) features_matched[features_matched >= 0.0] = 1 features_matched[features_matched < 0.0] = 0 features_unmatched[features_unmatched >= 0.0] = 1
def cls_net(_incoming, noise_sigma=0.05, drop_ratio_conv=0.3, drop_ratio_fc=0.5): _noise = L.GaussianNoiseLayer(_incoming, sigma=noise_sigma) _conv1 = conv(_noise, num_filters=128, filter_size=4, stride=2, pad=1, W=I.Normal(0.02), b=I.Constant(0.), nonlinearity=NL.rectify) _lrn1 = L.LocalResponseNormalization2DLayer(_conv1, alpha=0.0001, k=2, beta=0.75, n=5) _drop2 = L.DropoutLayer(_lrn1, p=drop_ratio_conv, rescale=True) _conv2 = batch_norm_n( conv(_drop2, num_filters=256, filter_size=4, stride=2, pad=1, W=I.Normal(0.02), b=None, nonlinearity=NL.rectify)) _drop3 = L.DropoutLayer(_conv2, p=drop_ratio_conv, rescale=True) _conv3 = batch_norm_n( conv(_drop3, num_filters=384, filter_size=3, stride=1, pad=0, W=I.Normal(0.02), b=None, nonlinearity=NL.rectify)) _drop4 = L.DropoutLayer(_conv3, p=drop_ratio_conv, rescale=True) _conv4 = batch_norm_n( conv(_drop4, num_filters=512, filter_size=3, stride=1, pad=0, W=I.Normal(0.02), b=None, nonlinearity=NL.rectify)) _drop5 = L.DropoutLayer(_conv4, p=drop_ratio_fc, rescale=True) _conv5 = batch_norm_n( conv(_drop5, num_filters=1024, filter_size=1, stride=1, pad=0, W=I.Normal(0.02), b=None, nonlinearity=NL.rectify)) _pool6 = L.GlobalPoolLayer(_conv5, pool_function=theano.tensor.max) #mean _drop6 = L.DropoutLayer(_pool6, p=drop_ratio_fc, rescale=True) _fc6 = L.DenseLayer(_drop6, ny, W=I.Normal(0.02), b=I.Constant(0), nonlinearity=NL.softmax) _aux = [ _conv1, _conv2, _conv3, _conv4, _conv5, L.DimshuffleLayer(_fc6, (0, 1, 'x', 'x')) ] ### used to have a tanh() around everything except last return _aux, _fc6
def get_discriminator_binary(): disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))] disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2)) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3, 3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3, 3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3, 3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append( nn.weight_norm( dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3, 3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append( nn.weight_norm( ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1])) disc_layers.append( nn.weight_norm(ll.DenseLayer(disc_layers[-1], num_units=2, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1)) disc_layers.append( ll.DenseLayer(disc_layers[-2], num_units=128, W=Normal(0.05), nonlinearity=T.nnet.sigmoid)) return disc_layers
def fcn(input_var, early_conv_dict, late_conv_dict, dense_filter_size, final_pool_function=T.max, input_size=128, output_size=188, p_dropout=0.5): ''' early_conv_dict: dict it contains the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' pool_filter_list: list each element is an integer pool_stride_list: list each element is int or None late_conv_dict: dict it contains the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' dense_filter_size: int the filter size of the final dense-like conv layer ''' # early conv layers input_network = lasagne.layers.InputLayer(shape=(None, 1, None, input_size), input_var=input_var) total_stride = 1 network, total_stride = conv_layers(input_network, early_conv_dict, total_stride, init_input_size=input_size, p_dropout=0, base_name='early') # late conv layers (dense layers) network, total_stride = conv_layers(network, late_conv_dict, total_stride, init_input_size=1, p_dropout=p_dropout, base_name='late') # frame output layer. every frame has a value network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout), num_filters=output_size, filter_size=(dense_filter_size, 1), nonlinearity=lasagne.nonlinearities.sigmoid, W=lasagne.init.GlorotUniform()) # pool network = layers.GlobalPoolLayer(network, pool_function=final_pool_function) network = layers.ReshapeLayer(network, ([0], -1)) return network
def p_fcn(input_var, early_conv_dict, pl_dict, late_conv_dict, dense_filter_size, final_pool_function=T.max, input_size=128, output_size=188, p_dropout=0.5): ''' This uses PL 2D layer instead 2D layer, comparing to fcn_pnn_multisource early_conv_dict_list: list each element in the list is a dictionary containing the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' pool_filter_list: list each element is an integer pool_stride_list: list each element is int or None pl_dict: dict it contains the following keys: 'num_lambda', 'num_points', 'value_range', 'seg_size', 'seg_stride' late_conv_dict: dict it contains the following keys: 'conv_filter_list', 'pool_filter_list', 'pool_stride_list' dense_filter_size: int the filter size of the final dense-like conv layer ''' # early conv layers input_network = lasagne.layers.InputLayer(shape=(None, 1, None, input_size), input_var=input_var) total_stride = 1 network, total_stride = conv_layers(input_network, early_conv_dict, total_stride, init_input_size=input_size, p_dropout=0, base_name='early') # Persistence landscape num_lambda = pl_dict['num_lambda'] num_points = pl_dict['num_points'] value_range = pl_dict['value_range'] seg_size = pl_dict['seg_size'] seg_step = pl_dict['seg_step'] patch_size = (seg_size, 1) patch_step = (seg_step, 1) network = cl.PersistenceFlatten2DLayer(network, num_lambda, num_points, value_range, patch_size, patch_step) # late conv layers (dense layers) network, total_stride = conv_layers(network, late_conv_dict, total_stride, init_input_size=1, p_dropout=p_dropout, base_name='late') # frame output layer. every frame has a value network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout), num_filters=output_size, filter_size=(dense_filter_size, 1), nonlinearity=lasagne.nonlinearities.sigmoid, W=lasagne.init.GlorotUniform()) # pool network = layers.GlobalPoolLayer(network, pool_function=final_pool_function) network = layers.ReshapeLayer(network, ([0], -1)) return network
def __init__( self, image_shape, filter_shape, num_class, conv_type, kernel_size, kernel_pool_size, dropout_rate, ): """ """ self.filter_shape = filter_shape self.n_visible = numpy.prod(image_shape) self.n_layers = len(filter_shape) self.rng = RandomStreams(123) self.x = T.matrix() self.y = T.ivector() self.conv_layers = [] NoiseLayer = layers.DropoutLayer dropout_rate = float(dropout_rate) self.l_input = layers.InputLayer((None, self.n_visible), self.x) this_layer = layers.ReshapeLayer(self.l_input, ([0], ) + image_shape) for l in range(self.n_layers): activation = lasagne.nonlinearities.rectify if len(filter_shape[l]) == 3: if conv_type == 'double' and filter_shape[l][1] > kernel_size: this_layer = DoubleConvLayer( this_layer, filter_shape[l][0], filter_shape[l][1:], pad='same', nonlinearity=activation, kernel_size=kernel_size, kernel_pool_size=kernel_pool_size) this_layer = layers.batch_norm(this_layer) elif conv_type == 'maxout': this_layer = layers.Conv2DLayer(this_layer, filter_shape[l][0], filter_shape[l][1:], b=None, pad='same', nonlinearity=None) this_layer = layers.FeaturePoolLayer( this_layer, pool_size=kernel_pool_size**2) this_layer = layers.BatchNormLayer(this_layer) this_layer = layers.NonlinearityLayer( this_layer, activation) elif conv_type == 'cyclic': this_layers = [] this_layers.append( layers.Conv2DLayer(this_layer, filter_shape[l][0], filter_shape[l][1:], b=None, pad='same', nonlinearity=None)) for _ in range(3): W = this_layers[-1].W.dimshuffle(0, 1, 3, 2)[:, :, :, ::-1] this_layers.append( layers.Conv2DLayer(this_layer, filter_shape[l][0], filter_shape[l][1:], W=W, b=None, pad='same', nonlinearity=None)) this_layer = layers.ElemwiseMergeLayer( this_layers, T.maximum) this_layer = layers.BatchNormLayer(this_layer) this_layer = layers.NonlinearityLayer( this_layer, activation) elif conv_type == 'standard' \ or (conv_type == 'double' and filter_shape[l][1] <= kernel_size): this_layer = layers.Conv2DLayer(this_layer, filter_shape[l][0], filter_shape[l][1:], pad='same', nonlinearity=activation) this_layer = layers.batch_norm(this_layer) else: raise NotImplementedError self.conv_layers.append(this_layer) elif len(filter_shape[l]) == 2: this_layer = layers.MaxPool2DLayer(this_layer, filter_shape[l]) this_layer = NoiseLayer(this_layer, dropout_rate) elif len(filter_shape[l]) == 1: raise NotImplementedError self.top_conv_layer = this_layer this_layer = layers.GlobalPoolLayer(this_layer, T.mean) self.clf_layer = layers.DenseLayer(this_layer, num_class, W=lasagne.init.Constant(0.), nonlinearity=T.nnet.softmax) self.params = layers.get_all_params(self.clf_layer, trainable=True) self.params_all = layers.get_all_params(self.clf_layer)
def deep_cnn_2d_mtl_at_2(params): """""" assert 'targets' in params nonlin = nl.elu layers = L.InputLayer((None, 1, params['dur'], 128)) print layers.output_shape sclr = joblib.load(params['scaler']) layers = L.standardize(layers, sclr.mean_.astype(np.float32), sclr.scale_.astype(np.float32), shared_axes=(0, 1, 2)) print layers.output_shape n_filter = [32, 64, 64, 128, 256, 256] # l if 'kernel_multiplier' in params: n_filter = map(lambda x: x * params['kernel_multiplier'], n_filter) filter_sz = [(3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (1, 1)] # m if params['dur'] > 50: conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] # c pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), None, None] # n else: conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] # c pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), None, None] # n pool_strd = [None, None, None, None, None, None] # s batch_norm = [True, False, True, False, False, False] # b dropout = [True, False, True, False, False, False] # d # added conv_spec = zip(n_filter, filter_sz, conv_strd, pool_sz, pool_strd, batch_norm, dropout) # Shared first layer if 'kernel_multiplier' in params: n_1st_ker = 16 * params['kernel_multiplier'] else: n_1st_ker = 16 layers = L.Conv2DLayer(layers, n_1st_ker, (5, 5), stride=(1, 1), pad='same', nonlinearity=nonlin) layers = L.MaxPool2DLayer(layers, pool_size=(1, 2)) layers = L.dropout(layers, p=0.1) layer_heads = OrderedDict() for target in params['targets']: first_trgt_spec_layer = True # n_layer checker for l, m, c, n, s, b, d in conv_spec: if first_trgt_spec_layer: layer_heads[target['name']] = layers first_trgt_spec_layer = False if b: layer_heads[target['name']] = L.batch_norm( L.Conv2DLayer(layer_heads[target['name']], l, m, stride=c, pad='same', nonlinearity=nonlin), ) else: layer_heads[target['name']] = L.Conv2DLayer( layer_heads[target['name']], l, m, stride=c, pad='same', nonlinearity=nonlin) if n is not None: layer_heads[target['name']] = L.MaxPool2DLayer( layer_heads[target['name']], pool_size=n, stride=s) if d: layer_heads[target['name']] = L.dropout( layer_heads[target['name']], p=0.1) print layer_heads[target['name']].output_shape layer_heads[target['name']] = L.batch_norm( L.GlobalPoolLayer(layer_heads[target['name']])) layer_heads[target['name']] = L.dropout( layer_heads[target['name']]) # added print layer_heads[target['name']].output_shape if 'kernel_multiplier' in params: n_hid = 256 * params['kernel_multiplier'] else: n_hid = 256 layer_heads[target['name']] = L.batch_norm( L.DenseLayer(layer_heads[target['name']], n_hid, nonlinearity=nonlin)) layer_heads[target['name']] = L.dropout(layer_heads[target['name']]) print layer_heads[target['name']].output_shape layer_heads[target['name']] = L.DenseLayer(layer_heads[target['name']], target['n_out'], nonlinearity=nl.softmax) print target['name'], layer_heads[target['name']].output_shape return layer_heads