Beispiel #1
0
def pons_cnn(params):
    """"""
    layers = L.InputLayer((None, 1, params['dur'], 128))
    print layers.output_shape

    sclr = joblib.load(params['scaler'])
    layers = L.standardize(layers,
                           sclr.mean_.astype(np.float32),
                           sclr.scale_.astype(np.float32),
                           shared_axes=(0, 1, 2))
    print layers.output_shape

    layers_timbre = L.GlobalPoolLayer(
        L.batch_norm(L.Conv2DLayer(layers, 64, (1, 96))))

    layers_rhythm = L.GlobalPoolLayer(
        L.batch_norm(L.Conv2DLayer(layers, 64, (params['dur'] - 10, 1))))

    layers = L.ConcatLayer([layers_rhythm, layers_timbre], axis=-1)

    layers = L.DenseLayer(layers, 64, nonlinearity=nl.rectify)
    print layers.output_shape

    layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax)
    print layers.output_shape

    return layers
Beispiel #2
0
def get_feature_extractor_trans(model_D, model_bin, x_temp=T.tensor4()):
    features = ll.get_output(ll.GlobalPoolLayer(model_D),
                             x_temp,
                             deterministic=True)
    features = features / (T.abs_(features) + 0.001)
    features = ll.get_output(model_bin, features, deterministic=False)
    return th.function(inputs=[x_temp], outputs=features)
def output_block(net, config, non_lin, verbose=True):
    """
    """
    # output setting
    out_acts = []
    for out_act in config.hyper_parameters.out_act:
        exec('from lasagne.nonlinearities import {}'.format(out_act))
        out_acts.append(eval(out_act))
    n_outs = config.hyper_parameters.n_out

    # Global Average Pooling
    last_conv_block_name = next(reversed(net))
    net['gap'] = L.GlobalPoolLayer(net[last_conv_block_name], name='gap')
    net['gap.bn'] = L.BatchNormLayer(net['gap'], name='gap.bn')
    n_features = net['gap.bn'].output_shape[-1]

    # feature Layer
    net['fc'] = L.dropout(L.batch_norm(
        L.DenseLayer(net['gap.bn'],
                     num_units=n_features,
                     nonlinearity=non_lin,
                     name='fc')),
                          name='fc.bn.do')

    # output (prediction)
    # check whether the model if for MTL or STL
    # target is passed as list, regardless whether
    # it's MTL or STL (configuration checker checks it)
    targets = config.target
    out_layer_names = []
    for target, n_out, out_act in zip(targets, n_outs, out_acts):

        out_layer_names.append('out.{}'.format(target))

        if target == 'self':
            net[out_layer_names[-1]], inputs = build_siamese(net['fc'])
        else:
            net[out_layer_names[-1]] = L.DenseLayer(net['fc'],
                                                    num_units=n_out,
                                                    nonlinearity=out_act,
                                                    name=out_layer_names[-1])
            inputs = [net['input'].input_var]

    # make a concatation layer just for save/load purpose
    net['IO'] = L.ConcatLayer([
        L.FlattenLayer(net[target_layer_name])
        if target == 'self' else net[target_layer_name]
        for target_layer_name in out_layer_names
    ],
                              name='IO')

    if verbose:
        print(net['gap.bn'].output_shape)
        print(net['fc'].output_shape)
        for target in targets:
            print(net['out.{}'.format(target)].output_shape)

    return net, inputs
Beispiel #4
0
    def build_network(self, ra_input_var, mc_input_var):
        print('Building raw network with parameters:')
        pp = pprint.PrettyPrinter(indent=4)
        pp.pprint(self.net_opts)

        ra_network_1 = layers.InputLayer((None, 1, None), ra_input_var)
        ra_network_1 = self.set_conv_layer(ra_network_1,
                                           'ra_conv_1',
                                           dropout=False,
                                           pad='same')
        ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_1')
        ra_network_1 = self.set_conv_layer(ra_network_1,
                                           'ra_conv_2',
                                           pad='same')
        ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_2')
        ra_network_1 = self.set_conv_layer(ra_network_1,
                                           'ra_conv_3',
                                           pad='same')
        ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_3')
        ra_network_1 = self.set_conv_layer(ra_network_1,
                                           'ra_conv_4',
                                           pad='same')
        ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_4')
        concat_list = [ra_network_1]
        mc_input = layers.InputLayer((None, 2, None), mc_input_var)
        concat_list.append(mc_input)
        network = layers.ConcatLayer(concat_list,
                                     axis=1,
                                     cropping=[None, None, 'center'])
        network = self.set_conv_layer(network, 'conv_1')
        network = self.set_pool_layer(network, 'pool_1')
        network = self.set_conv_layer(network, 'conv_2')
        network = self.set_pool_layer(network, 'pool_2')
        network = self.set_conv_layer(network, 'conv_3')
        network = layers.GlobalPoolLayer(
            network, getattr(T, self.net_opts['global_pool_func']))
        # print(layers.get_output_shape(network))
        # network = layers.DenseLayer(layers.dropout(network, p=self.net_opts['dropout_p']),
        #                           self.net_opts['dens_1'],
        #                           nonlinearity=lasagne.nonlinearities.rectify)
        network = layers.DenseLayer(
            layers.dropout(network, p=self.net_opts['dropout_p']),
            self.net_opts['dens_2'],
            nonlinearity=lasagne.nonlinearities.rectify)
        network = layers.DenseLayer(
            layers.dropout(network, p=self.net_opts['dropout_p']),
            self.net_opts['num_class'],
            nonlinearity=lasagne.nonlinearities.softmax)
        # print(layers.get_output_shape(network))
        self.network = network
        return self.network
Beispiel #5
0
    def get_discriminator(self):
        ''' specify discriminator D0 '''
        """
        disc0_layers = [LL.InputLayer(shape=(self.args.batch_size, 3, 32, 32))]
        disc0_layers.append(LL.GaussianNoiseLayer(disc0_layers[-1], sigma=0.05))
        disc0_layers.append(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu))
        disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 16x16
        disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1))
        disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu)))
        disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 8x8
        disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1))
        disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=0, W=Normal(0.02), nonlinearity=nn.lrelu))) # 6x6
        disc0_layer_shared = LL.NINLayer(disc0_layers[-1], num_units=192, W=Normal(0.02), nonlinearity=nn.lrelu) # 6x6
        disc0_layers.append(disc0_layer_shared)

        disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared, num_units=50, W=Normal(0.02), nonlinearity=None)
        disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x

        disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared))
        disc0_layer_adv = LL.DenseLayer(disc0_layers[-1], num_units=10, W=Normal(0.02), nonlinearity=None)
        disc0_layers.append(disc0_layer_adv)

        return disc0_layers, disc0_layer_adv, disc0_layer_z_recon
        """
        disc_x_layers = [LL.InputLayer(shape=(None, 3, 32, 32))]
        disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2))
        disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))
        disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
        disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
        disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)))
        disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
        disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
        disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu)))
        disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu)
        disc_x_layers.append(disc_x_layers_shared)

        disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=self.args.z0dim, nonlinearity=None)
        disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x

        # disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100))
        disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared))
        disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None)
        disc_x_layers.append(disc_x_layer_adv)

        #output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False)
        #output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False)

        # temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True)
        # temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True)
        # init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])]
        return disc_x_layers, disc_x_layer_adv, disc_x_layer_z_recon
def build_dist_feat_fnc(net,
                        target,
                        conv_feat_locs=[5, 10, 12, 17, 19],
                        fc_feat_locs=[24, 28]):
    """"""
    layers = L.get_all_layers(net[target])
    assert len(layers) == 30  # only works for standard deep conv2d
    feat = [L.GlobalPoolLayer(layers[l]) for l in conv_feat_locs]
    feat += [layers[l] for l in fc_feat_locs]
    feat = L.ConcatLayer(feat, axis=1)
    f = L.get_output(feat, deterministic=True)

    f_feat = {target: {}}
    f_feat[target]['transform'] = theano.function([layers[0].input_var],
                                                  f,
                                                  allow_input_downcast=True)
    return f_feat
Beispiel #7
0
                           pad=0,
                           W=Normal(0.02),
                           nonlinearity=nn.lrelu)))  # 6x6
disc0_layer_shared = LL.NINLayer(disc0_layers[-1],
                                 num_units=192,
                                 W=Normal(0.02),
                                 nonlinearity=nn.lrelu)  # 6x6
disc0_layers.append(disc0_layer_shared)

disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared,
                                    num_units=16,
                                    W=Normal(0.02),
                                    nonlinearity=None)
disc0_layers.append(disc0_layer_z_recon)  # also need to recover z from x

disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared))
disc0_layer_adv = LL.DenseLayer(disc0_layers[-1],
                                num_units=10,
                                W=Normal(0.02),
                                nonlinearity=None)
disc0_layers.append(disc0_layer_adv)
''' forward pass '''

output_before_softmax_real0 = LL.get_output(disc0_layer_adv,
                                            x,
                                            deterministic=False)
output_before_softmax_gen0, recon_z0 = LL.get_output(
    [disc0_layer_adv, disc0_layer_z_recon], gen_x, deterministic=False
)  # discriminator's predicted probability that gen_x is real
''' loss for discriminator and Q '''
Beispiel #8
0
def pc_fcn(input_var,
           early_conv_dict,
           middle_conv_dict,
           pl_dict,
           late_conv_dict,
           dense_filter_size,
           final_pool_function=T.max,
           input_size=128,
           output_size=188,
           p_dropout=0.5):
    '''
    early_conv_dict_list: list
        each element in the list is a dictionary containing
        the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

    pl_dict: dict
        it contains the following keys:
        'num_lambda', 'num_points', 'value_range', 'seg_size', 'seg_stride'

    late_conv_dict: dict
        it contains the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

    dense_filter_size: int
        the filter size of the final dense-like conv layer

    pool_filter_list: list
        each element is an integer

    pool_stride_list: list
        each element is int or None

    '''
    # early conv layers
    input_network = lasagne.layers.InputLayer(shape=(None, 1, None,
                                                     input_size),
                                              input_var=input_var)

    total_stride = 1
    network, total_stride = conv_layers(input_network,
                                        early_conv_dict,
                                        total_stride,
                                        init_input_size=input_size,
                                        p_dropout=0,
                                        base_name='early')

    # middle conv layers (dense layers)
    network_c, _ = conv_layers(network,
                               middle_conv_dict,
                               total_stride,
                               init_input_size=1,
                               p_dropout=0,
                               base_name='middle_conv')

    # Persistence landscape
    network_p = network
    try:
        num_lambda = pl_dict['num_lambda']
    except:
        num_lambda = pl_dict['n_f_db']

    try:
        num_points = pl_dict['num_points']
    except:
        num_points = pl_dict['n_points']

    value_range = pl_dict['value_range']
    seg_size = pl_dict['seg_size']
    seg_step = pl_dict['seg_step']

    patch_size = (seg_size, 1)
    patch_step = (seg_step, 1)

    network_p = cl.PersistenceFlatten2DLayer(network_p, num_lambda, num_points,
                                             value_range, patch_size,
                                             patch_step)

    # Convolution+Persistence
    network = layers.ConcatLayer([network_c, network_p],
                                 axis=1,
                                 cropping=[None, None, 'lower', None],
                                 name='Convolution+Persistence')

    # late conv layers (dense layers)
    network, total_stride = conv_layers(network,
                                        late_conv_dict,
                                        total_stride,
                                        init_input_size=1,
                                        p_dropout=p_dropout,
                                        base_name='late')

    # frame output layer. every frame has a value
    network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout),
                              num_filters=output_size,
                              filter_size=(dense_filter_size, 1),
                              nonlinearity=lasagne.nonlinearities.sigmoid,
                              W=lasagne.init.GlorotUniform())

    # pool
    network = layers.GlobalPoolLayer(network,
                                     pool_function=final_pool_function)
    network = layers.ReshapeLayer(network, ([0], -1))

    return network
Beispiel #9
0
def deep_cnn_2d(params):
    """"""
    nonlin = nl.elu

    layers = L.InputLayer((None, 1, params['dur'], 128))
    print layers.output_shape

    sclr = joblib.load(params['scaler'])
    layers = L.standardize(layers,
                           sclr.mean_.astype(np.float32),
                           sclr.scale_.astype(np.float32),
                           shared_axes=(0, 1, 2))
    print layers.output_shape

    n_filter = [16, 32, 64, 64, 128, 256, 256]  # l
    filter_sz = [(5, 5), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (1, 1)]  # m
    if params['dur'] > 50:
        conv_strd = [(2, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1),
                     (1, 1)]  # c
        pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), (2, 2), None, None]  # n
    else:
        conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1),
                     (1, 1)]  # c
        pool_sz = [(1, 2), (2, 2), (2, 2), (2, 2), (2, 2), None, None]  # n
    pool_strd = [None, None, None, None, None, None, None]  # s
    batch_norm = [False, True, False, True, False, False, False]  # b
    dropout = [True, True, False, True, False, False, False]  # d # added
    conv_spec = zip(n_filter, filter_sz, conv_strd, pool_sz, pool_strd,
                    batch_norm, dropout)

    for l, m, c, n, s, b, d in conv_spec:
        if b:
            layers = L.batch_norm(
                L.Conv2DLayer(layers,
                              l,
                              m,
                              stride=c,
                              pad='same',
                              nonlinearity=nonlin), )
        else:
            layers = L.Conv2DLayer(layers,
                                   l,
                                   m,
                                   stride=c,
                                   pad='same',
                                   nonlinearity=nonlin)
        if n is not None:
            layers = L.MaxPool2DLayer(layers, pool_size=n, stride=s)

        if d:
            layers = L.dropout(layers, p=0.1)

        print layers.output_shape

    layers = L.batch_norm(L.GlobalPoolLayer(layers))
    layers = L.dropout(layers)  # added
    print layers.output_shape

    layers = L.batch_norm(L.DenseLayer(layers, 256, nonlinearity=nonlin))
    layers = L.dropout(layers)
    print layers.output_shape

    layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax)
    print layers.output_shape

    return layers
Beispiel #10
0
dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-00'))
dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-01'))
dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 32, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-02'), name='dis-03'))
dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-20'))
dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 32, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-21'), name='dis-22'))
dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-23'))
dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-30'))
dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 64, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-31'), name='dis-32'))
dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-40'))
dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 64, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-41'), name='dis-42'))
dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-43'))
dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-50'))
dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 128, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-51'), name='dis-52'))
dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-60'))
dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 128, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-61'), name='dis-62'))
dis_layers.append(ll.GlobalPoolLayer(dis_layers[-1], name='dis-63'))
dis_layers.append(MLPConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-70'))
dis_layers.append(nn.weight_norm(ll.DenseLayer(dis_layers[-1], num_units=1, W=Normal(0.05), nonlinearity=ln.sigmoid, name='dis-71'), train_g=True, init_stdv=0.1, name='dis-72'))

# inference module
inf_in_x = ll.InputLayer(shape=(None, in_channels) + dim_input)
inf_layers = [inf_in_x]
inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 64, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-02'), name='inf-03'))
inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 128, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-11'), name='inf-12'))
inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 256, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-21'), name='inf-22'))
inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 512, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-31'), name='inf-32'))
inf_layers.append(ll.DenseLayer(inf_layers[-1], num_units=n_z, W=Normal(0.05), nonlinearity=None, name='inf-4'))

# discriminator xz
disxz_in_x = ll.InputLayer(shape=(None, in_channels) + dim_input)
disxz_in_z = ll.InputLayer(shape=(None, n_z))
Beispiel #11
0
# specify discriminative model
disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))]
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(nn.weight_norm(ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(nn.weight_norm(ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu)))
disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
disc_layers.append(nn.weight_norm(ll.DenseLayer(disc_layers[-1], num_units=16, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1))
disc_params = ll.get_all_params(disc_layers, trainable=True)

x_temp = T.tensor4()

temp = ll.get_output(gen_layers[-1], deterministic=False, init=True)
temp = ll.get_output(disc_layers[-1], x_temp, deterministic=False, init=True)
init_updates = [u for l in gen_layers+disc_layers for u in getattr(l,'init_updates',[])]

init_param = th.function(inputs=[x_temp], outputs=None, updates=init_updates)

# costs
labels = T.ivector()
x_lab = T.tensor4()
x_unl = T.tensor4()
Beispiel #12
0
def buildNet():

    log.p('BUILDING BirdNET MODEL...', new_line=False)

    # Input layer for images
    net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0]))    

    # Pre-processing stage
    #log.p(("\tPRE-PROCESSING STAGE:"))
    net = l.batch_norm(l.Conv2DLayer(net,
                    num_filters=int(FILTERS[0] * RESNET_K),
                    filter_size=(5, 5),
                    pad='same',
                    nonlinearity=nl.rectify))
    
    #log.p(("\t\tFIRST  CONV OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))

    # Max pooling
    net = l.MaxPool2DLayer(net, pool_size=(1, 2))
    #log.p(("\t\tPRE-MAXPOOL OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))
    
    # Residual Stacks
    for i in range(1, len(FILTERS)):
        #log.p(("\tRES STACK", i, ':'))
        net = resblock(net,
                       filters=int(FILTERS[i] * RESNET_K),
                       kernel_size=KERNEL_SIZES[i],
                       stride=2,
                       preactivated=True,
                       block_id=i,
                       name='BLOCK ' + str(i) + '-1')
        
        for j in range(1, RESNET_N):
            net = resblock(net,
                           filters=int(FILTERS[i] * RESNET_K),
                           kernel_size=KERNEL_SIZES[i],
                           preactivated=False,
                           block_id=i+j,
                           name='BLOCK ' + str(i) + '-' + str(j + 1))
        
    # Post Activation
    net = l.batch_norm(net)
    net = l.NonlinearityLayer(net, nonlinearity=nl.rectify)
    
    # Classification branch
    #log.p(("\tCLASS BRANCH:"))
    net = classificationBranch(net,  (4, 10)) 
    #log.p(("\t\tBRANCH OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))

    # Pooling
    net = l.GlobalPoolLayer(net, pool_function=logmeanexp)
    #log.p(("\tGLOBAL POOLING SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))

    # Sigmoid output
    net = l.NonlinearityLayer(net, nonlinearity=nl.sigmoid)

    #log.p(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net))))
    log.p("DONE!")

    # Model stats
    #log.p(("MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"))
    #log.p(("MODEL HAS", l.count_params(net), "PARAMS"))

    return net
def build_baseline_model():

    log.i('BUILDING BASELINE MODEL...')

    # Random Seed
    lasagne_random.set_rng(cfg.getRandomState())

    # Input layer for images
    net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0]))

    # Stride size (as an alternative to max pooling)
    if cfg.MAX_POOLING:
        s = 1
    else:
        s = 2

    # Convolutinal layer groups
    for i in range(len(cfg.FILTERS)):

        # 3x3 Convolution + Stride
        net = batch_norm(
            l.Conv2DLayer(net,
                          num_filters=cfg.FILTERS[i],
                          filter_size=cfg.KERNEL_SIZES[i],
                          num_groups=cfg.NUM_OF_GROUPS[i],
                          pad='same',
                          stride=s,
                          W=initialization(cfg.NONLINEARITY),
                          nonlinearity=nonlinearity(cfg.NONLINEARITY)))

        # Pooling layer
        if cfg.MAX_POOLING:
            net = l.MaxPool2DLayer(net, pool_size=2)

        # Dropout Layer (we support different types of dropout)
        if cfg.DROPOUT_TYPE == 'channels' and cfg.DROPOUT > 0.0:
            net = l.dropout_channels(net, p=cfg.DROPOUT)
        elif cfg.DROPOUT_TYPE == 'location' and cfg.DROPOUT > 0.0:
            net = l.dropout_location(net, p=cfg.DROPOUT)
        elif cfg.DROPOUT > 0.0:
            net = l.DropoutLayer(net, p=cfg.DROPOUT)

        log.i(('\tGROUP', i + 1, 'OUT SHAPE:', l.get_output_shape(net)))

    # Final 1x1 Convolution
    net = batch_norm(
        l.Conv2DLayer(net,
                      num_filters=cfg.FILTERS[i] * 2,
                      filter_size=1,
                      W=initialization('identity'),
                      nonlinearity=nonlinearity('identity')))

    log.i(('\tFINAL CONV OUT SHAPE:', l.get_output_shape(net)))

    # Global Pooling layer (default mode = average)
    net = l.GlobalPoolLayer(net)
    log.i(("\tFINAL POOLING SHAPE:", l.get_output_shape(net)))

    # Classification Layer (Softmax)
    net = l.DenseLayer(net,
                       len(cfg.CLASSES),
                       nonlinearity=nonlinearity('softmax'),
                       W=initialization('softmax'))

    log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net)))
    log.i("...DONE!")

    # Model stats
    log.i(("MODEL HAS",
           (sum(hasattr(layer, 'W')
                for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"))
    log.i(("MODEL HAS", l.count_params(net), "PARAMS"))

    return net
Beispiel #14
0
def dmr_regularizer(f_1, f2):
    x_bin = f_1 / (T.abs_(f_1) + args.gamma)
    x_bin_2 = T.sgn(f2)
    M_reg = (T.dot(x_bin_2, x_bin_2.T) - T.dot(x_bin_2, x_bin_2.T) *
             (T.eye(x_bin_2.shape[0]))) / x_bin_2.shape[1]
    M_true = (T.dot(x_bin, x_bin.T) - T.dot(x_bin, x_bin.T) *
              (T.eye(x_bin.shape[0]))) / x_bin.shape[1]
    l_reg = T.sum(T.abs_(M_reg - M_true)) / (x_bin.shape[0] *
                                             (x_bin.shape[0] - 1))

    return l_reg


if args.dataset_type == 'brown':
    global_pool = ll.GlobalPoolLayer(disc_layers[f_low_dim])
    global_pool_2 = ll.ReshapeLayer(disc_layers[f_high_dim], ([0], -1))
else:
    global_pool = disc_layers[f_low_dim]
    global_pool_2 = ll.GlobalPoolLayer(disc_layers[f_high_dim])

features_1 = ll.get_output(global_pool, x_unl_1, deterministic=False)
features_2 = ll.get_output(global_pool_2, x_unl_1, deterministic=False)
loss_unl = loss_unl + args.lambda_dmr * dmr_regularizer(features_1, features_2) \
           + args.lambda_bre * bre_regularizer(features_1, features_2)

# Theano functions for training the disc net
lr = T.scalar()
disc_params = ll.get_all_params(disc_layers, trainable=True)
disc_param_updates = nn.adam_updates(disc_params, loss_unl, lr=lr, mom1=0.5)
disc_param_avg = [
    def build(self):
        """"""
        # output setting
        out_acts = []
        for out_act in self.config.hyper_parameters.out_act:
            # exec('from lasagne.nonlinearities import {}'.format(out_act))
            out_acts.append(eval(out_act))
        n_outs = self.config.hyper_parameters.n_out
        targets = self.config.target

        # apply size multiplier
        self.n_filters = map(lambda x: x * self.m, self.n_filters)

        # network dict & set input block
        self.net, self.variables['sigma'] = input_block(self.net,
                                                        self.config,
                                                        melspec=True)

        if self.branch_at == 'fc':
            # shares all layers except output layer
            for i in range(len(self.n_convs)):
                name = 'conv{:d}'.format(i + 1)
                self.net = conv_block(self.net, self.n_convs[i],
                                      self.n_filters[i], self.filter_sizes[i],
                                      self.strides[i], self.pool_sizes[i],
                                      self.non_lin, self.batch_norm, name,
                                      self.net.keys()[-1], self.verbose)
            # GAP
            self.net['gap'] = L.batch_norm(
                L.GlobalPoolLayer(self.net[next(reversed(self.net))],
                                  name='gap'))
            self.net['fc'] = L.dropout(L.batch_norm(
                L.DenseLayer(self.net['gap'],
                             num_units=self.net['gap'].output_shape[-1],
                             nonlinearity=self.non_lin,
                             name='fc')),
                                       name='fc.bn.do')

            # Out block
            out_layer_names = []
            for target, n_out, out_act in zip(targets, n_outs, out_acts):
                out_layer_names.append('{}.out'.format(target))
                if target == 'self':
                    self.net[out_layer_names[-1]], inputs = \
                            build_siamese(self.net['fc'])
                else:
                    self.net[out_layer_names[-1]] = L.DenseLayer(
                        self.net['fc'],
                        num_units=n_out,
                        nonlinearity=out_act,
                        name=out_layer_names[-1])
                    inputs = [self.net['input'].input_var]
                self.variables['{}.inputs'.format(target)] = inputs
        else:
            # shares lower part of the network and branch out

            # shared conv blocks
            for i in range(self.branch_at - 1):
                name = 'conv{:d}'.format(i + 1)
                self.net = conv_block(self.net, self.n_convs[i],
                                      self.n_filters[i], self.filter_sizes[i],
                                      self.strides[i], self.pool_sizes[i],
                                      self.non_lin, self.batch_norm, name,
                                      self.net.keys()[-1], self.verbose)
            branch_point = self.net.keys()[-1]

            # branch out to each targets
            out_layer_names = []
            for target, n_out, out_act in zip(targets, n_outs, out_acts):
                # first conv_block for each branch
                j = self.branch_at - 1  # branch_point_ix
                name = '{}.conv{:d}'.format(target, j + 1)
                self.net = conv_block(self.net, self.n_convs[j],
                                      self.n_filters[j], self.filter_sizes[j],
                                      self.strides[j], self.pool_sizes[j],
                                      self.non_lin, self.batch_norm, name,
                                      branch_point, self.verbose)

                for i in range(self.branch_at, len(self.n_convs)):
                    name = '{}.conv{:d}'.format(target, i + 1)
                    self.net = conv_block(self.net, self.n_convs[i],
                                          self.n_filters[i],
                                          self.filter_sizes[i],
                                          self.strides[i], self.pool_sizes[i],
                                          self.non_lin, self.batch_norm, name,
                                          self.net.keys()[-1], self.verbose)

                # GAP
                gap_name = '{}.gap'.format(target)
                self.net[gap_name] = L.batch_norm(
                    L.GlobalPoolLayer(self.net[next(reversed(self.net))],
                                      name=gap_name))

                # FC
                fc_name = '{}.fc'.format(target)
                self.net[fc_name] = L.dropout(L.batch_norm(
                    L.DenseLayer(self.net[gap_name],
                                 num_units=self.net[gap_name].output_shape[-1],
                                 nonlinearity=self.non_lin,
                                 name=fc_name)),
                                              name=fc_name)

                # OUT
                out_layer_names.append('{}.out'.format(target))
                if target == 'self':
                    self.net[out_layer_names[-1]], inputs = \
                            build_siamese(self.net[fc_name])
                else:
                    self.net[out_layer_names[-1]] = L.DenseLayer(
                        self.net[fc_name],
                        num_units=n_out,
                        nonlinearity=out_act,
                        name=out_layer_names[-1])
                    inputs = [self.net['input'].input_var]

                self.variables['{}.inputs'.format(target)] = inputs

        # make a concatation layer just for save/load purpose
        self.net['IO'] = L.ConcatLayer([
            L.FlattenLayer(self.net[target_layer_name])
            if target == 'self' else self.net[target_layer_name]
            for target_layer_name in out_layer_names
        ],
                                       name='IO')

        if self.verbose:
            for target in targets:
                print(self.net['{}.out'.format(target)].output_shape)

        return self.net, self.variables
Beispiel #16
0
def fcn_multiscale(
        input_var_list,
        early_conv_dict_list,
        late_conv_dict,
        dense_filter_size,
        final_pool_function=T.max,
        input_size_list=[128], output_size=188,
        p_dropout=0.5
        ):
    '''
    early_conv_dict_list: list
        each element in the list is a dictionary containing the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

    late_conv_dict: dict
        it contains the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

    dense_filter_size: int
        the filter size of the final dense-like conv layer

    pool_filter_list: list
        each element is an integer

    pool_stride_list: list
        each element is int or None

    '''
    assert(len(early_conv_dict_list) == len(input_var_list) ==
           len(input_size_list))

    # early conv layers
    conv_network_list = list()
    total_stride_list = list()
    for jj, [early_conv_dict, input_var, input_size] in enumerate(zip(
            early_conv_dict_list, input_var_list, input_size_list)):
        input_network = lasagne.layers.InputLayer(
            shape=(None, 1, None, input_size), input_var=input_var)

        total_stride = 1
        network, total_stride = conv_layers(input_network, early_conv_dict,
                                            total_stride,
                                            init_input_size=input_size,
                                            p_dropout=0,
                                            base_name='early{}'.format(jj))
        total_stride_list.append(total_stride)
        conv_network_list.append(network)

    # Concatenate
    network = layers.ConcatLayer(conv_network_list,
                                 axis=1,
                                 cropping=[None, None, 'lower', None],
                                 name='MultisourceConcatenate')

    # late conv layers (dense layers)
    network, total_stride = conv_layers(network, late_conv_dict,
                                        total_stride,
                                        init_input_size=1,
                                        p_dropout=p_dropout,
                                        base_name='late')

    # frame output layer. every frame has a value
    network = cl.Conv2DXLayer(
        lasagne.layers.dropout(network, p=p_dropout),
        num_filters=output_size, filter_size=(dense_filter_size, 1),
        nonlinearity=lasagne.nonlinearities.sigmoid,
        W=lasagne.init.GlorotUniform()
    )

    # pool
    network = layers.GlobalPoolLayer(network,
                                     pool_function=final_pool_function)
    network = layers.ReshapeLayer(network, ([0], -1))

    return network
Beispiel #17
0
    def build_critic(self, version=1, extra_middle=False, only_middle=False):
        assert self.generator != None

        if version == 1:
            from lasagne.nonlinearities import sigmoid
            disc_layers = [
                ll.InputLayer(shape=(self.batch_size, 3, 64, 64),
                              input_var=self.input_c)
            ]
            # b_s x 3 x 64 x 64 --> b_s x 32 x 32 x 32
            disc_layers.append(
                nn.batch_norm(
                    ll.Conv2DLayer(disc_layers[-1],
                                   128, (3, 3),
                                   pad=1,
                                   stride=2,
                                   W=Normal(0.03),
                                   nonlinearity=nn.lrelu)))  #nn.weight_norm

            # disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
            # b_s x 32 x 32 x 32 --> b_s x 64 x 16 x 16
            disc_layers.append(
                nn.batch_norm(
                    ll.Conv2DLayer(disc_layers[-1],
                                   256, (3, 3),
                                   pad=1,
                                   stride=2,
                                   W=Normal(0.03),
                                   nonlinearity=nn.lrelu)))  #nn.weight_norm
            # b_s x 64 x 16 x 16 --> b_s x 128 x 8 x 8
            # note : this layer will be used to compare statistics (output of disc_layer[3])
            disc_layers.append(
                nn.batch_norm(
                    ll.Conv2DLayer(disc_layers[-1],
                                   512, (3, 3),
                                   pad=1,
                                   stride=2,
                                   W=Normal(0.03),
                                   nonlinearity=nn.lrelu)))  #nn.weight_norm
            # b_s x 128 x 8 x 8 --> b_s x 256 x 4 x 4
            disc_layers.append(
                nn.batch_norm(
                    ll.Conv2DLayer(disc_layers[-1],
                                   1024, (3, 3),
                                   pad=1,
                                   stride=2,
                                   W=Normal(0.03),
                                   nonlinearity=nn.lrelu)))  #nn.weight_norm

            disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
            disc_layers.append(
                nn.MinibatchLayer(disc_layers[-1], num_kernels=100))

            if extra_middle or only_middle:
                # goal : add mechanism that checks for mode collapse, but only for the middle part

                layer = nn.ExtractMiddleLayer(disc_layers[0], extra=2)
                # two convolutions
                print layer.output_shape
                layer = nn.batch_norm(
                    ll.Conv2DLayer(layer,
                                   128,
                                   5,
                                   stride=2,
                                   pad='same',
                                   nonlinearity=nn.lrelu))

                print layer.output_shape
                layer = nn.batch_norm(
                    ll.Conv2DLayer(layer,
                                   256,
                                   5,
                                   stride=2,
                                   pad='same',
                                   nonlinearity=nn.lrelu))

                print layer.output_shape
                layer = nn.batch_norm(
                    ll.Conv2DLayer(layer,
                                   512,
                                   5,
                                   stride=2,
                                   pad='same',
                                   nonlinearity=nn.lrelu))

                #layer = nn.batch_norm(ll.Conv2DLayer(layer, 1024, 5, stride=2, pad='same',
                #			   nonlinearity=nn.lrelu))

                #disc_layers.append(layer)

                #print layer.output_shape
                layer = ll.GlobalPoolLayer(layer)

                #print layer.output_shape
                layer = nn.MinibatchLayer(layer, num_kernels=400)

                if only_middle:
                    disc_layers = []
                    disc_layers.append((ll.DenseLayer(layer,
                                                      num_units=1,
                                                      W=Normal(0.03),
                                                      nonlinearity=None)))
                    return disc_layers

                disc_layers.append(ll.ConcatLayer([layer, disc_layers[-1]]))

            disc_layers.append((ll.DenseLayer(disc_layers[-1],
                                              num_units=1,
                                              W=Normal(0.03),
                                              nonlinearity=None)))

            # seeing how there is strong intra-batch normalization, I think it would be good to do minibatch discrimination solely on the center of the images.
            # I think that the statistics of the border are throwing

            for layer in disc_layers:
                print layer.output_shape
            print ''

            return disc_layers
Beispiel #18
0
def build_fcn_fixedgaussian_multiscale(num_scales):

    input_var_list = list()
    earlyconv_list = list()

    # A stack of convolution layers for each scale
    for ii in range(num_scales):
        # input tensor
        input_var = T.tensor4('input.{}'.format(ii))

        # input layer
        network = lasagne.layers.InputLayer(shape=(None, 1, None, 128),
                                            input_var=input_var)

        # early conv layers
        network = cl.Conv2DXLayer(network,
                                  num_filters=32,
                                  filter_size=(8, 128),
                                  stride=(1, 1),
                                  nonlinearity=lasagne.nonlinearities.rectify,
                                  pad='strictsamex',
                                  name='early_conv.{}_1'.format(ii))
        network = layers.MaxPool2DLayer(network,
                                        pool_size=(4, 1),
                                        stride=(4, 1),
                                        ignore_border=False,
                                        name='early_maxpool.{}_1'.format(ii))

        network = cl.Conv2DXLayer(network,
                                  num_filters=32,
                                  filter_size=(8, 1),
                                  stride=(1, 1),
                                  nonlinearity=lasagne.nonlinearities.rectify,
                                  pad='strictsamex',
                                  name='early_conv.{}_2'.format(ii))
        network = layers.MaxPool2DLayer(network,
                                        pool_size=(4, 1),
                                        stride=(4, 1),
                                        ignore_border=False,
                                        name='early_maxpool.{}_2'.format(ii))

        input_var_list.append(input_var)
        earlyconv_list.append(network)

    network = layers.ConcatLayer(earlyconv_list,
                                 axis=1,
                                 cropping=[None, None, 'lower', None],
                                 name='multiscale_concat')

    # late conv layers
    network = layers.Conv2DLayer(layers.dropout(network, p=0.5),
                                 512, (1, 1), (1, 1),
                                 name='late_conv.{}_1'.format(ii))

    network = layers.Conv2DLayer(layers.dropout(network, p=0.5),
                                 512, (1, 1), (1, 1),
                                 name='late_conv.{}_2'.format(ii))

    network = layers.Conv2DLayer(layers.dropout(network, p=0.5),
                                 188, (1, 1), (1, 1),
                                 nonlinearity=lasagne.nonlinearities.sigmoid,
                                 name='late_conv.{}_output'.format(ii))

    # Gaussian scanning
    network = layers.ReshapeLayer(network, ([0], [1], [2]))
    nogaussian_layer = network

    network = cl.FixedGaussianScan1DLayer(network,
                                          filter_size=256,
                                          init_std=256 / 4**2,
                                          stride=1,
                                          pad='strictsame',
                                          name='fixed_gaussian_filter')
    gaussian_layer = network

    # pool
    network = layers.GlobalPoolLayer(network, pool_function=T.mean)

    return network, input_var_list, nogaussian_layer, gaussian_layer
Beispiel #19
0
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0.5, ps=2, n_kerns=32, d_kerns=(5, 5),
                            pad='valid', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
                            name='cla-1'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=64, d_kerns=(3, 3),
                            pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
                            name='cla-2'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0.5, ps=2, n_kerns=64, d_kerns=(3, 3),
                            pad='valid', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
                            name='cla-3'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=128, d_kerns=(3, 3),
                            pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
                            name='cla-4'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=128, d_kerns=(3, 3),
                            pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
                            name='cla-5'))
cla_layers.append(ll.GlobalPoolLayer(cla_layers[-1]))
cla_layers.append(ll.DenseLayer(cla_layers[-1], num_units=num_classes, W=lasagne.init.Normal(1e-2, 0),
                                nonlinearity=ln.softmax, name='cla-6'))

classifier = cla_layers[-1]  #


gen_in_z = ll.InputLayer(shape=(None, n_z))
gen_in_y = ll.InputLayer(shape=(None,))
gen_layers = [gen_in_z]
gen_layers.append(MLPConcatLayer([gen_layers[-1], gen_in_y], num_classes, name='gen-1'))

gen_layers.append(ll.batch_norm(ll.DenseLayer(gen_layers[-1], num_units=500,
                                              nonlinearity=ln.softplus, name='gen-2'), name='gen-3'))
gen_layers.append(MLPConcatLayer([gen_layers[-1], gen_in_y], num_classes, name='gen-4'))
Beispiel #20
0
def get_discriminator_brown(num_feature=256):

    disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))]
    disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               96, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               96, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               96, (3, 3),
                               pad=1,
                               stride=2,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               128, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               128, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               128, (3, 3),
                               pad=1,
                               stride=2,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               128, (3, 3),
                               pad=0,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            ll.NINLayer(disc_layers[-1],
                        num_units=num_feature,
                        W=Normal(0.05),
                        nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            ll.NINLayer(disc_layers[-1],
                        num_units=128,
                        W=Normal(0.05),
                        nonlinearity=nn.lrelu)))
    disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
    disc_layers.append(
        nn.weight_norm(ll.DenseLayer(disc_layers[-1],
                                     num_units=2,
                                     W=Normal(0.05),
                                     nonlinearity=None),
                       train_g=True,
                       init_stdv=0.1))
    #disc_layers.append(ll.ReshapeLayer(disc_layers[-4], ([0], -1)))
    #disc_layers.append(ll.GlobalPoolLayer(disc_layers[-4]))
    disc_layer_features_low_dim = -4
    disc_layer_features_high_dim = -5

    return disc_layers, disc_layer_features_low_dim, disc_layer_features_high_dim
def build_resnet_model():

    log.i('BUILDING RESNET MODEL...')

    # Random Seed
    lasagne_random.set_rng(cfg.getRandomState())

    # Input layer for images
    net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0]))

    # First Convolution
    net = l.Conv2DLayer(net,
                        num_filters=cfg.FILTERS[0],
                        filter_size=cfg.KERNEL_SIZES[0],
                        pad='same',
                        W=initialization(cfg.NONLINEARITY),
                        nonlinearity=None)

    log.i(("\tFIRST CONV OUT SHAPE:", l.get_output_shape(net), "LAYER:",
           len(l.get_all_layers(net)) - 1))

    # Residual Stacks
    for i in range(0, len(cfg.FILTERS)):
        net = resblock(net,
                       filters=cfg.FILTERS[i] * cfg.RESNET_K,
                       kernel_size=cfg.KERNEL_SIZES[i],
                       stride=2,
                       num_groups=cfg.NUM_OF_GROUPS[i])
        for _ in range(1, cfg.RESNET_N):
            net = resblock(net,
                           filters=cfg.FILTERS[i] * cfg.RESNET_K,
                           kernel_size=cfg.KERNEL_SIZES[i],
                           num_groups=cfg.NUM_OF_GROUPS[i],
                           preactivated=False)
        log.i(("\tRES STACK", i + 1, "OUT SHAPE:", l.get_output_shape(net),
               "LAYER:", len(l.get_all_layers(net)) - 1))

    # Post Activation
    net = batch_norm(net)
    net = l.NonlinearityLayer(net, nonlinearity=nonlinearity(cfg.NONLINEARITY))

    # Pooling
    net = l.GlobalPoolLayer(net)
    log.i(("\tFINAL POOLING SHAPE:", l.get_output_shape(net), "LAYER:",
           len(l.get_all_layers(net)) - 1))

    # Classification Layer
    net = l.DenseLayer(net,
                       len(cfg.CLASSES),
                       nonlinearity=nonlinearity('identity'),
                       W=initialization('identity'))
    net = l.NonlinearityLayer(net, nonlinearity=nonlinearity('softmax'))

    log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net), "LAYER:",
           len(l.get_all_layers(net))))
    log.i("...DONE!")

    # Model stats
    log.i(("MODEL HAS",
           (sum(hasattr(layer, 'W')
                for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"))
    log.i(("MODEL HAS", l.count_params(net), "PARAMS"))

    return net
disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2))
disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu)
disc_x_layers.append(disc_x_layers_shared)

disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=50, nonlinearity=None)
disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x

# disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100))
disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared))
disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None)
disc_x_layers.append(disc_x_layer_adv)

#output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False)
#output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False)

# temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True)
# temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True)
# init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])]

output_before_softmax_real = LL.get_output(disc_x_layer_adv, x, deterministic=False) 
output_before_softmax_gen, recon_z = LL.get_output([disc_x_layer_adv, disc_x_layer_z_recon], gen_x, deterministic=False) # discriminator's predicted probability that gen_x is real

l_lab = output_before_softmax_real[T.arange(args.batch_size),y]
l_unl = nn.log_sum_exp(output_before_softmax_real)
    nn.batch_norm(dnn.Conv2DDNNLayer(genz_layers[-1],
                                     512, (3, 3),
                                     pad=1,
                                     stride=2,
                                     W=Normal(0.05),
                                     nonlinearity=nn.lrelu,
                                     name='gz2'),
                  g=None))
genz_layers.append(
    nn.batch_norm(ll.NINLayer(genz_layers[-1],
                              num_units=512,
                              W=Normal(0.05),
                              nonlinearity=nn.lrelu,
                              name='gz5'),
                  g=None))
genz_layers.append(ll.GlobalPoolLayer(genz_layers[-1], name='gz6'))
genz_layers.append(
    ll.DenseLayer(genz_layers[-1],
                  num_units=100,
                  W=Normal(0.05),
                  nonlinearity=lasagne.nonlinearities.sigmoid,
                  name='gz7'))

# specify discriminative model

# for z
discz_layers = [z_input]
discz_layers.append(ll.DropoutLayer(discz_layers[-1], p=0.2))
discz_layers.append(
    ll.DenseLayer(discz_layers[-1],
                  500,
Beispiel #24
0
                             stretch=True)
img = plotting.plot_img(img_tile, title='')
plotting.plt.savefig(args.data_name + "_patches_gen.png")

img_bhwc = np.transpose(trainx[:100, ], (0, 2, 3, 1))
img_tile = plotting.img_tile(img_bhwc,
                             aspect_ratio=1.0,
                             border_color=1.0,
                             stretch=True)
img = plotting.plot_img(img_tile, title='')
plotting.plt.savefig(args.data_name + "_patches_train.png")

evens = [x for x in range(int(test_matched.shape[0] / 2)) if x % 2 == 0]
odds = [x for x in range(int(test_matched.shape[0] / 2)) if x % 2 == 1]

disc_layers.append(ll.GlobalPoolLayer(disc_layers[f_low_dim]))
batch_size_test = 100
print('Extracting features from matched data...')
features_matched = extract_features(disc_layers, test_matched, batch_size_test)

print('Extracting features from unmatched data...')
features_unmatched = extract_features(disc_layers, test_unmatched,
                                      batch_size_test)

print('Number of features considered in the experiment: ' +
      str(features_matched[evens].shape[1]))

features_matched[features_matched >= 0.0] = 1
features_matched[features_matched < 0.0] = 0

features_unmatched[features_unmatched >= 0.0] = 1
def cls_net(_incoming,
            noise_sigma=0.05,
            drop_ratio_conv=0.3,
            drop_ratio_fc=0.5):
    _noise = L.GaussianNoiseLayer(_incoming, sigma=noise_sigma)

    _conv1 = conv(_noise,
                  num_filters=128,
                  filter_size=4,
                  stride=2,
                  pad=1,
                  W=I.Normal(0.02),
                  b=I.Constant(0.),
                  nonlinearity=NL.rectify)
    _lrn1 = L.LocalResponseNormalization2DLayer(_conv1,
                                                alpha=0.0001,
                                                k=2,
                                                beta=0.75,
                                                n=5)

    _drop2 = L.DropoutLayer(_lrn1, p=drop_ratio_conv, rescale=True)
    _conv2 = batch_norm_n(
        conv(_drop2,
             num_filters=256,
             filter_size=4,
             stride=2,
             pad=1,
             W=I.Normal(0.02),
             b=None,
             nonlinearity=NL.rectify))

    _drop3 = L.DropoutLayer(_conv2, p=drop_ratio_conv, rescale=True)
    _conv3 = batch_norm_n(
        conv(_drop3,
             num_filters=384,
             filter_size=3,
             stride=1,
             pad=0,
             W=I.Normal(0.02),
             b=None,
             nonlinearity=NL.rectify))

    _drop4 = L.DropoutLayer(_conv3, p=drop_ratio_conv, rescale=True)
    _conv4 = batch_norm_n(
        conv(_drop4,
             num_filters=512,
             filter_size=3,
             stride=1,
             pad=0,
             W=I.Normal(0.02),
             b=None,
             nonlinearity=NL.rectify))

    _drop5 = L.DropoutLayer(_conv4, p=drop_ratio_fc, rescale=True)
    _conv5 = batch_norm_n(
        conv(_drop5,
             num_filters=1024,
             filter_size=1,
             stride=1,
             pad=0,
             W=I.Normal(0.02),
             b=None,
             nonlinearity=NL.rectify))

    _pool6 = L.GlobalPoolLayer(_conv5, pool_function=theano.tensor.max)  #mean
    _drop6 = L.DropoutLayer(_pool6, p=drop_ratio_fc, rescale=True)
    _fc6 = L.DenseLayer(_drop6,
                        ny,
                        W=I.Normal(0.02),
                        b=I.Constant(0),
                        nonlinearity=NL.softmax)

    _aux = [
        _conv1, _conv2, _conv3, _conv4, _conv5,
        L.DimshuffleLayer(_fc6, (0, 1, 'x', 'x'))
    ]  ### used to have a tanh() around everything except last
    return _aux, _fc6
Beispiel #26
0
def get_discriminator_binary():

    disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))]
    disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               96, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               96, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               96, (3, 3),
                               pad=1,
                               stride=2,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               192, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               192, (3, 3),
                               pad=1,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               192, (3, 3),
                               pad=1,
                               stride=2,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
    disc_layers.append(
        nn.weight_norm(
            dnn.Conv2DDNNLayer(disc_layers[-1],
                               192, (3, 3),
                               pad=0,
                               W=Normal(0.05),
                               nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            ll.NINLayer(disc_layers[-1],
                        num_units=192,
                        W=Normal(0.05),
                        nonlinearity=nn.lrelu)))
    disc_layers.append(
        nn.weight_norm(
            ll.NINLayer(disc_layers[-1],
                        num_units=192,
                        W=Normal(0.05),
                        nonlinearity=nn.lrelu)))
    disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
    disc_layers.append(
        nn.weight_norm(ll.DenseLayer(disc_layers[-1],
                                     num_units=2,
                                     W=Normal(0.05),
                                     nonlinearity=None),
                       train_g=True,
                       init_stdv=0.1))
    disc_layers.append(
        ll.DenseLayer(disc_layers[-2],
                      num_units=128,
                      W=Normal(0.05),
                      nonlinearity=T.nnet.sigmoid))

    return disc_layers
Beispiel #27
0
def fcn(input_var,
        early_conv_dict,
        late_conv_dict,
        dense_filter_size,
        final_pool_function=T.max,
        input_size=128,
        output_size=188,
        p_dropout=0.5):
    '''
    early_conv_dict: dict
        it contains the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

        pool_filter_list: list
            each element is an integer

        pool_stride_list: list
            each element is int or None

    late_conv_dict: dict
        it contains the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

    dense_filter_size: int
        the filter size of the final dense-like conv layer

    '''
    # early conv layers
    input_network = lasagne.layers.InputLayer(shape=(None, 1, None,
                                                     input_size),
                                              input_var=input_var)

    total_stride = 1
    network, total_stride = conv_layers(input_network,
                                        early_conv_dict,
                                        total_stride,
                                        init_input_size=input_size,
                                        p_dropout=0,
                                        base_name='early')

    # late conv layers (dense layers)
    network, total_stride = conv_layers(network,
                                        late_conv_dict,
                                        total_stride,
                                        init_input_size=1,
                                        p_dropout=p_dropout,
                                        base_name='late')

    # frame output layer. every frame has a value
    network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout),
                              num_filters=output_size,
                              filter_size=(dense_filter_size, 1),
                              nonlinearity=lasagne.nonlinearities.sigmoid,
                              W=lasagne.init.GlorotUniform())

    # pool
    network = layers.GlobalPoolLayer(network,
                                     pool_function=final_pool_function)
    network = layers.ReshapeLayer(network, ([0], -1))

    return network
Beispiel #28
0
def p_fcn(input_var,
          early_conv_dict,
          pl_dict,
          late_conv_dict,
          dense_filter_size,
          final_pool_function=T.max,
          input_size=128,
          output_size=188,
          p_dropout=0.5):
    '''
    This uses PL 2D layer instead 2D layer, comparing to fcn_pnn_multisource

    early_conv_dict_list: list
        each element in the list is a dictionary containing
        the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

        pool_filter_list: list
            each element is an integer

        pool_stride_list: list
            each element is int or None

    pl_dict: dict
        it contains the following keys:
        'num_lambda', 'num_points', 'value_range', 'seg_size', 'seg_stride'

    late_conv_dict: dict
        it contains the following keys:
        'conv_filter_list', 'pool_filter_list', 'pool_stride_list'

    dense_filter_size: int
        the filter size of the final dense-like conv layer


    '''
    # early conv layers
    input_network = lasagne.layers.InputLayer(shape=(None, 1, None,
                                                     input_size),
                                              input_var=input_var)

    total_stride = 1
    network, total_stride = conv_layers(input_network,
                                        early_conv_dict,
                                        total_stride,
                                        init_input_size=input_size,
                                        p_dropout=0,
                                        base_name='early')

    # Persistence landscape
    num_lambda = pl_dict['num_lambda']
    num_points = pl_dict['num_points']
    value_range = pl_dict['value_range']
    seg_size = pl_dict['seg_size']
    seg_step = pl_dict['seg_step']

    patch_size = (seg_size, 1)
    patch_step = (seg_step, 1)

    network = cl.PersistenceFlatten2DLayer(network, num_lambda, num_points,
                                           value_range, patch_size, patch_step)

    # late conv layers (dense layers)
    network, total_stride = conv_layers(network,
                                        late_conv_dict,
                                        total_stride,
                                        init_input_size=1,
                                        p_dropout=p_dropout,
                                        base_name='late')

    # frame output layer. every frame has a value
    network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout),
                              num_filters=output_size,
                              filter_size=(dense_filter_size, 1),
                              nonlinearity=lasagne.nonlinearities.sigmoid,
                              W=lasagne.init.GlorotUniform())

    # pool
    network = layers.GlobalPoolLayer(network,
                                     pool_function=final_pool_function)
    network = layers.ReshapeLayer(network, ([0], -1))

    return network
Beispiel #29
0
    def __init__(
        self,
        image_shape,
        filter_shape,
        num_class,
        conv_type,
        kernel_size,
        kernel_pool_size,
        dropout_rate,
    ):
        """

        """
        self.filter_shape = filter_shape
        self.n_visible = numpy.prod(image_shape)
        self.n_layers = len(filter_shape)
        self.rng = RandomStreams(123)
        self.x = T.matrix()
        self.y = T.ivector()

        self.conv_layers = []

        NoiseLayer = layers.DropoutLayer

        dropout_rate = float(dropout_rate)

        self.l_input = layers.InputLayer((None, self.n_visible), self.x)
        this_layer = layers.ReshapeLayer(self.l_input, ([0], ) + image_shape)

        for l in range(self.n_layers):
            activation = lasagne.nonlinearities.rectify
            if len(filter_shape[l]) == 3:
                if conv_type == 'double' and filter_shape[l][1] > kernel_size:
                    this_layer = DoubleConvLayer(
                        this_layer,
                        filter_shape[l][0],
                        filter_shape[l][1:],
                        pad='same',
                        nonlinearity=activation,
                        kernel_size=kernel_size,
                        kernel_pool_size=kernel_pool_size)
                    this_layer = layers.batch_norm(this_layer)
                elif conv_type == 'maxout':
                    this_layer = layers.Conv2DLayer(this_layer,
                                                    filter_shape[l][0],
                                                    filter_shape[l][1:],
                                                    b=None,
                                                    pad='same',
                                                    nonlinearity=None)
                    this_layer = layers.FeaturePoolLayer(
                        this_layer, pool_size=kernel_pool_size**2)
                    this_layer = layers.BatchNormLayer(this_layer)
                    this_layer = layers.NonlinearityLayer(
                        this_layer, activation)

                elif conv_type == 'cyclic':
                    this_layers = []
                    this_layers.append(
                        layers.Conv2DLayer(this_layer,
                                           filter_shape[l][0],
                                           filter_shape[l][1:],
                                           b=None,
                                           pad='same',
                                           nonlinearity=None))
                    for _ in range(3):
                        W = this_layers[-1].W.dimshuffle(0, 1, 3,
                                                         2)[:, :, :, ::-1]
                        this_layers.append(
                            layers.Conv2DLayer(this_layer,
                                               filter_shape[l][0],
                                               filter_shape[l][1:],
                                               W=W,
                                               b=None,
                                               pad='same',
                                               nonlinearity=None))
                    this_layer = layers.ElemwiseMergeLayer(
                        this_layers, T.maximum)
                    this_layer = layers.BatchNormLayer(this_layer)
                    this_layer = layers.NonlinearityLayer(
                        this_layer, activation)

                elif conv_type == 'standard' \
                     or (conv_type == 'double' and filter_shape[l][1] <= kernel_size):
                    this_layer = layers.Conv2DLayer(this_layer,
                                                    filter_shape[l][0],
                                                    filter_shape[l][1:],
                                                    pad='same',
                                                    nonlinearity=activation)
                    this_layer = layers.batch_norm(this_layer)
                else:
                    raise NotImplementedError

                self.conv_layers.append(this_layer)

            elif len(filter_shape[l]) == 2:
                this_layer = layers.MaxPool2DLayer(this_layer, filter_shape[l])
                this_layer = NoiseLayer(this_layer, dropout_rate)
            elif len(filter_shape[l]) == 1:
                raise NotImplementedError

        self.top_conv_layer = this_layer
        this_layer = layers.GlobalPoolLayer(this_layer, T.mean)
        self.clf_layer = layers.DenseLayer(this_layer,
                                           num_class,
                                           W=lasagne.init.Constant(0.),
                                           nonlinearity=T.nnet.softmax)

        self.params = layers.get_all_params(self.clf_layer, trainable=True)

        self.params_all = layers.get_all_params(self.clf_layer)
Beispiel #30
0
def deep_cnn_2d_mtl_at_2(params):
    """"""
    assert 'targets' in params
    nonlin = nl.elu

    layers = L.InputLayer((None, 1, params['dur'], 128))
    print layers.output_shape

    sclr = joblib.load(params['scaler'])
    layers = L.standardize(layers,
                           sclr.mean_.astype(np.float32),
                           sclr.scale_.astype(np.float32),
                           shared_axes=(0, 1, 2))
    print layers.output_shape

    n_filter = [32, 64, 64, 128, 256, 256]  # l
    if 'kernel_multiplier' in params:
        n_filter = map(lambda x: x * params['kernel_multiplier'], n_filter)
    filter_sz = [(3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (1, 1)]  # m
    if params['dur'] > 50:
        conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)]  # c
        pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), None, None]  # n
    else:
        conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)]  # c
        pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), None, None]  # n
    pool_strd = [None, None, None, None, None, None]  # s
    batch_norm = [True, False, True, False, False, False]  # b
    dropout = [True, False, True, False, False, False]  # d # added
    conv_spec = zip(n_filter, filter_sz, conv_strd, pool_sz, pool_strd,
                    batch_norm, dropout)

    # Shared first layer
    if 'kernel_multiplier' in params:
        n_1st_ker = 16 * params['kernel_multiplier']
    else:
        n_1st_ker = 16
    layers = L.Conv2DLayer(layers,
                           n_1st_ker, (5, 5),
                           stride=(1, 1),
                           pad='same',
                           nonlinearity=nonlin)
    layers = L.MaxPool2DLayer(layers, pool_size=(1, 2))
    layers = L.dropout(layers, p=0.1)

    layer_heads = OrderedDict()
    for target in params['targets']:
        first_trgt_spec_layer = True  # n_layer checker
        for l, m, c, n, s, b, d in conv_spec:
            if first_trgt_spec_layer:
                layer_heads[target['name']] = layers
                first_trgt_spec_layer = False
            if b:
                layer_heads[target['name']] = L.batch_norm(
                    L.Conv2DLayer(layer_heads[target['name']],
                                  l,
                                  m,
                                  stride=c,
                                  pad='same',
                                  nonlinearity=nonlin), )
            else:
                layer_heads[target['name']] = L.Conv2DLayer(
                    layer_heads[target['name']],
                    l,
                    m,
                    stride=c,
                    pad='same',
                    nonlinearity=nonlin)
            if n is not None:
                layer_heads[target['name']] = L.MaxPool2DLayer(
                    layer_heads[target['name']], pool_size=n, stride=s)

            if d:
                layer_heads[target['name']] = L.dropout(
                    layer_heads[target['name']], p=0.1)
            print layer_heads[target['name']].output_shape

        layer_heads[target['name']] = L.batch_norm(
            L.GlobalPoolLayer(layer_heads[target['name']]))
        layer_heads[target['name']] = L.dropout(
            layer_heads[target['name']])  # added
        print layer_heads[target['name']].output_shape

        if 'kernel_multiplier' in params:
            n_hid = 256 * params['kernel_multiplier']
        else:
            n_hid = 256
        layer_heads[target['name']] = L.batch_norm(
            L.DenseLayer(layer_heads[target['name']],
                         n_hid,
                         nonlinearity=nonlin))
        layer_heads[target['name']] = L.dropout(layer_heads[target['name']])
        print layer_heads[target['name']].output_shape

        layer_heads[target['name']] = L.DenseLayer(layer_heads[target['name']],
                                                   target['n_out'],
                                                   nonlinearity=nl.softmax)
        print target['name'], layer_heads[target['name']].output_shape

    return layer_heads