def pons_cnn(params):
""""""
layers = L.InputLayer((None, 1, params['dur'], 128))
print layers.output_shape
sclr = joblib.load(params['scaler'])
layers = L.standardize(layers,
sclr.mean_.astype(np.float32),
sclr.scale_.astype(np.float32),
shared_axes=(0, 1, 2))
print layers.output_shape
layers_timbre = L.GlobalPoolLayer(
L.batch_norm(L.Conv2DLayer(layers, 64, (1, 96))))
layers_rhythm = L.GlobalPoolLayer(
L.batch_norm(L.Conv2DLayer(layers, 64, (params['dur'] - 10, 1))))
layers = L.ConcatLayer([layers_rhythm, layers_timbre], axis=-1)
layers = L.DenseLayer(layers, 64, nonlinearity=nl.rectify)
print layers.output_shape
layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax)
print layers.output_shape
return layers
def get_feature_extractor_trans(model_D, model_bin, x_temp=T.tensor4()):
features = ll.get_output(ll.GlobalPoolLayer(model_D),
x_temp,
deterministic=True)
features = features / (T.abs_(features) + 0.001)
features = ll.get_output(model_bin, features, deterministic=False)
return th.function(inputs=[x_temp], outputs=features)
def output_block(net, config, non_lin, verbose=True):
"""
"""
# output setting
out_acts = []
for out_act in config.hyper_parameters.out_act:
exec('from lasagne.nonlinearities import {}'.format(out_act))
out_acts.append(eval(out_act))
n_outs = config.hyper_parameters.n_out
# Global Average Pooling
last_conv_block_name = next(reversed(net))
net['gap'] = L.GlobalPoolLayer(net[last_conv_block_name], name='gap')
net['gap.bn'] = L.BatchNormLayer(net['gap'], name='gap.bn')
n_features = net['gap.bn'].output_shape[-1]
# feature Layer
net['fc'] = L.dropout(L.batch_norm(
L.DenseLayer(net['gap.bn'],
num_units=n_features,
nonlinearity=non_lin,
name='fc')),
name='fc.bn.do')
# output (prediction)
# check whether the model if for MTL or STL
# target is passed as list, regardless whether
# it's MTL or STL (configuration checker checks it)
targets = config.target
out_layer_names = []
for target, n_out, out_act in zip(targets, n_outs, out_acts):
out_layer_names.append('out.{}'.format(target))
if target == 'self':
net[out_layer_names[-1]], inputs = build_siamese(net['fc'])
else:
net[out_layer_names[-1]] = L.DenseLayer(net['fc'],
num_units=n_out,
nonlinearity=out_act,
name=out_layer_names[-1])
inputs = [net['input'].input_var]
# make a concatation layer just for save/load purpose
net['IO'] = L.ConcatLayer([
L.FlattenLayer(net[target_layer_name])
if target == 'self' else net[target_layer_name]
for target_layer_name in out_layer_names
],
name='IO')
if verbose:
print(net['gap.bn'].output_shape)
print(net['fc'].output_shape)
for target in targets:
print(net['out.{}'.format(target)].output_shape)
return net, inputs
def build_network(self, ra_input_var, mc_input_var):
print('Building raw network with parameters:')
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(self.net_opts)
ra_network_1 = layers.InputLayer((None, 1, None), ra_input_var)
ra_network_1 = self.set_conv_layer(ra_network_1,
'ra_conv_1',
dropout=False,
pad='same')
ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_1')
ra_network_1 = self.set_conv_layer(ra_network_1,
'ra_conv_2',
pad='same')
ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_2')
ra_network_1 = self.set_conv_layer(ra_network_1,
'ra_conv_3',
pad='same')
ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_3')
ra_network_1 = self.set_conv_layer(ra_network_1,
'ra_conv_4',
pad='same')
ra_network_1 = self.set_pool_layer(ra_network_1, 'ra_pool_4')
concat_list = [ra_network_1]
mc_input = layers.InputLayer((None, 2, None), mc_input_var)
concat_list.append(mc_input)
network = layers.ConcatLayer(concat_list,
axis=1,
cropping=[None, None, 'center'])
network = self.set_conv_layer(network, 'conv_1')
network = self.set_pool_layer(network, 'pool_1')
network = self.set_conv_layer(network, 'conv_2')
network = self.set_pool_layer(network, 'pool_2')
network = self.set_conv_layer(network, 'conv_3')
network = layers.GlobalPoolLayer(
network, getattr(T, self.net_opts['global_pool_func']))
# print(layers.get_output_shape(network))
# network = layers.DenseLayer(layers.dropout(network, p=self.net_opts['dropout_p']),
# self.net_opts['dens_1'],
# nonlinearity=lasagne.nonlinearities.rectify)
network = layers.DenseLayer(
layers.dropout(network, p=self.net_opts['dropout_p']),
self.net_opts['dens_2'],
nonlinearity=lasagne.nonlinearities.rectify)
network = layers.DenseLayer(
layers.dropout(network, p=self.net_opts['dropout_p']),
self.net_opts['num_class'],
nonlinearity=lasagne.nonlinearities.softmax)
# print(layers.get_output_shape(network))
self.network = network
return self.network
def get_discriminator(self):
''' specify discriminator D0 '''
"""
disc0_layers = [LL.InputLayer(shape=(self.args.batch_size, 3, 32, 32))]
disc0_layers.append(LL.GaussianNoiseLayer(disc0_layers[-1], sigma=0.05))
disc0_layers.append(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 16x16
disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, W=Normal(0.02), nonlinearity=nn.lrelu)))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.02), nonlinearity=nn.lrelu))) # 8x8
disc0_layers.append(LL.DropoutLayer(disc0_layers[-1], p=0.1))
disc0_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc0_layers[-1], 192, (3,3), pad=0, W=Normal(0.02), nonlinearity=nn.lrelu))) # 6x6
disc0_layer_shared = LL.NINLayer(disc0_layers[-1], num_units=192, W=Normal(0.02), nonlinearity=nn.lrelu) # 6x6
disc0_layers.append(disc0_layer_shared)
disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared, num_units=50, W=Normal(0.02), nonlinearity=None)
disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x
disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared))
disc0_layer_adv = LL.DenseLayer(disc0_layers[-1], num_units=10, W=Normal(0.02), nonlinearity=None)
disc0_layers.append(disc0_layer_adv)
return disc0_layers, disc0_layer_adv, disc0_layer_z_recon
"""
disc_x_layers = [LL.InputLayer(shape=(None, 3, 32, 32))]
disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2))
disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5))
disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu)))
disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu)
disc_x_layers.append(disc_x_layers_shared)
disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=self.args.z0dim, nonlinearity=None)
disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x
# disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100))
disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared))
disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None)
disc_x_layers.append(disc_x_layer_adv)
#output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False)
#output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False)
# temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True)
# temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True)
# init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])]
return disc_x_layers, disc_x_layer_adv, disc_x_layer_z_recon
def build_dist_feat_fnc(net,
target,
conv_feat_locs=[5, 10, 12, 17, 19],
fc_feat_locs=[24, 28]):
""""""
layers = L.get_all_layers(net[target])
assert len(layers) == 30 # only works for standard deep conv2d
feat = [L.GlobalPoolLayer(layers[l]) for l in conv_feat_locs]
feat += [layers[l] for l in fc_feat_locs]
feat = L.ConcatLayer(feat, axis=1)
f = L.get_output(feat, deterministic=True)
f_feat = {target: {}}
f_feat[target]['transform'] = theano.function([layers[0].input_var],
f,
allow_input_downcast=True)
return f_feat
pad=0,
W=Normal(0.02),
nonlinearity=nn.lrelu))) # 6x6
disc0_layer_shared = LL.NINLayer(disc0_layers[-1],
num_units=192,
W=Normal(0.02),
nonlinearity=nn.lrelu) # 6x6
disc0_layers.append(disc0_layer_shared)
disc0_layer_z_recon = LL.DenseLayer(disc0_layer_shared,
num_units=16,
W=Normal(0.02),
nonlinearity=None)
disc0_layers.append(disc0_layer_z_recon) # also need to recover z from x
disc0_layers.append(LL.GlobalPoolLayer(disc0_layer_shared))
disc0_layer_adv = LL.DenseLayer(disc0_layers[-1],
num_units=10,
W=Normal(0.02),
nonlinearity=None)
disc0_layers.append(disc0_layer_adv)
''' forward pass '''
output_before_softmax_real0 = LL.get_output(disc0_layer_adv,
x,
deterministic=False)
output_before_softmax_gen0, recon_z0 = LL.get_output(
[disc0_layer_adv, disc0_layer_z_recon], gen_x, deterministic=False
) # discriminator's predicted probability that gen_x is real
''' loss for discriminator and Q '''
def pc_fcn(input_var,
early_conv_dict,
middle_conv_dict,
pl_dict,
late_conv_dict,
dense_filter_size,
final_pool_function=T.max,
input_size=128,
output_size=188,
p_dropout=0.5):
'''
early_conv_dict_list: list
each element in the list is a dictionary containing
the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
pl_dict: dict
it contains the following keys:
'num_lambda', 'num_points', 'value_range', 'seg_size', 'seg_stride'
late_conv_dict: dict
it contains the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
dense_filter_size: int
the filter size of the final dense-like conv layer
pool_filter_list: list
each element is an integer
pool_stride_list: list
each element is int or None
'''
# early conv layers
input_network = lasagne.layers.InputLayer(shape=(None, 1, None,
input_size),
input_var=input_var)
total_stride = 1
network, total_stride = conv_layers(input_network,
early_conv_dict,
total_stride,
init_input_size=input_size,
p_dropout=0,
base_name='early')
# middle conv layers (dense layers)
network_c, _ = conv_layers(network,
middle_conv_dict,
total_stride,
init_input_size=1,
p_dropout=0,
base_name='middle_conv')
# Persistence landscape
network_p = network
try:
num_lambda = pl_dict['num_lambda']
except:
num_lambda = pl_dict['n_f_db']
try:
num_points = pl_dict['num_points']
except:
num_points = pl_dict['n_points']
value_range = pl_dict['value_range']
seg_size = pl_dict['seg_size']
seg_step = pl_dict['seg_step']
patch_size = (seg_size, 1)
patch_step = (seg_step, 1)
network_p = cl.PersistenceFlatten2DLayer(network_p, num_lambda, num_points,
value_range, patch_size,
patch_step)
# Convolution+Persistence
network = layers.ConcatLayer([network_c, network_p],
axis=1,
cropping=[None, None, 'lower', None],
name='Convolution+Persistence')
# late conv layers (dense layers)
network, total_stride = conv_layers(network,
late_conv_dict,
total_stride,
init_input_size=1,
p_dropout=p_dropout,
base_name='late')
# frame output layer. every frame has a value
network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout),
num_filters=output_size,
filter_size=(dense_filter_size, 1),
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.GlorotUniform())
# pool
network = layers.GlobalPoolLayer(network,
pool_function=final_pool_function)
network = layers.ReshapeLayer(network, ([0], -1))
return network
def deep_cnn_2d(params):
""""""
nonlin = nl.elu
layers = L.InputLayer((None, 1, params['dur'], 128))
print layers.output_shape
sclr = joblib.load(params['scaler'])
layers = L.standardize(layers,
sclr.mean_.astype(np.float32),
sclr.scale_.astype(np.float32),
shared_axes=(0, 1, 2))
print layers.output_shape
n_filter = [16, 32, 64, 64, 128, 256, 256] # l
filter_sz = [(5, 5), (3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (1, 1)] # m
if params['dur'] > 50:
conv_strd = [(2, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1),
(1, 1)] # c
pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), (2, 2), None, None] # n
else:
conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1),
(1, 1)] # c
pool_sz = [(1, 2), (2, 2), (2, 2), (2, 2), (2, 2), None, None] # n
pool_strd = [None, None, None, None, None, None, None] # s
batch_norm = [False, True, False, True, False, False, False] # b
dropout = [True, True, False, True, False, False, False] # d # added
conv_spec = zip(n_filter, filter_sz, conv_strd, pool_sz, pool_strd,
batch_norm, dropout)
for l, m, c, n, s, b, d in conv_spec:
if b:
layers = L.batch_norm(
L.Conv2DLayer(layers,
l,
m,
stride=c,
pad='same',
nonlinearity=nonlin), )
else:
layers = L.Conv2DLayer(layers,
l,
m,
stride=c,
pad='same',
nonlinearity=nonlin)
if n is not None:
layers = L.MaxPool2DLayer(layers, pool_size=n, stride=s)
if d:
layers = L.dropout(layers, p=0.1)
print layers.output_shape
layers = L.batch_norm(L.GlobalPoolLayer(layers))
layers = L.dropout(layers) # added
print layers.output_shape
layers = L.batch_norm(L.DenseLayer(layers, 256, nonlinearity=nonlin))
layers = L.dropout(layers)
print layers.output_shape
layers = L.DenseLayer(layers, 16, nonlinearity=nl.softmax)
print layers.output_shape
return layers
dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-00')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-01')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 32, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-02'), name='dis-03')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-20')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 32, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-21'), name='dis-22')) dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-23')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-30')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 64, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-31'), name='dis-32')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-40')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 64, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-41'), name='dis-42')) dis_layers.append(ll.DropoutLayer(dis_layers[-1], p=0.2, name='dis-43')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-50')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 128, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-51'), name='dis-52')) dis_layers.append(ConvConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-60')) dis_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(dis_layers[-1], 128, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu, name='dis-61'), name='dis-62')) dis_layers.append(ll.GlobalPoolLayer(dis_layers[-1], name='dis-63')) dis_layers.append(MLPConcatLayer([dis_layers[-1], dis_in_y], num_classes, name='dis-70')) dis_layers.append(nn.weight_norm(ll.DenseLayer(dis_layers[-1], num_units=1, W=Normal(0.05), nonlinearity=ln.sigmoid, name='dis-71'), train_g=True, init_stdv=0.1, name='dis-72')) # inference module inf_in_x = ll.InputLayer(shape=(None, in_channels) + dim_input) inf_layers = [inf_in_x] inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 64, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-02'), name='inf-03')) inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 128, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-11'), name='inf-12')) inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 256, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-21'), name='inf-22')) inf_layers.append(ll.batch_norm(dnn.Conv2DDNNLayer(inf_layers[-1], 512, (4,4), stride=2, pad=1, W=Normal(0.05), nonlinearity=nn.lrelu, name='inf-31'), name='inf-32')) inf_layers.append(ll.DenseLayer(inf_layers[-1], num_units=n_z, W=Normal(0.05), nonlinearity=None, name='inf-4')) # discriminator xz disxz_in_x = ll.InputLayer(shape=(None, in_channels) + dim_input) disxz_in_z = ll.InputLayer(shape=(None, n_z))
# specify discriminative model disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))] disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5)) disc_layers.append(nn.weight_norm(dnn.Conv2DDNNLayer(disc_layers[-1], 192, (3,3), pad=0, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(nn.weight_norm(ll.NINLayer(disc_layers[-1], num_units=192, W=Normal(0.05), nonlinearity=nn.lrelu))) disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1])) disc_layers.append(nn.weight_norm(ll.DenseLayer(disc_layers[-1], num_units=16, W=Normal(0.05), nonlinearity=None), train_g=True, init_stdv=0.1)) disc_params = ll.get_all_params(disc_layers, trainable=True) x_temp = T.tensor4() temp = ll.get_output(gen_layers[-1], deterministic=False, init=True) temp = ll.get_output(disc_layers[-1], x_temp, deterministic=False, init=True) init_updates = [u for l in gen_layers+disc_layers for u in getattr(l,'init_updates',[])] init_param = th.function(inputs=[x_temp], outputs=None, updates=init_updates) # costs labels = T.ivector() x_lab = T.tensor4() x_unl = T.tensor4()
def buildNet():
log.p('BUILDING BirdNET MODEL...', new_line=False)
# Input layer for images
net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0]))
# Pre-processing stage
#log.p(("\tPRE-PROCESSING STAGE:"))
net = l.batch_norm(l.Conv2DLayer(net,
num_filters=int(FILTERS[0] * RESNET_K),
filter_size=(5, 5),
pad='same',
nonlinearity=nl.rectify))
#log.p(("\t\tFIRST CONV OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))
# Max pooling
net = l.MaxPool2DLayer(net, pool_size=(1, 2))
#log.p(("\t\tPRE-MAXPOOL OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))
# Residual Stacks
for i in range(1, len(FILTERS)):
#log.p(("\tRES STACK", i, ':'))
net = resblock(net,
filters=int(FILTERS[i] * RESNET_K),
kernel_size=KERNEL_SIZES[i],
stride=2,
preactivated=True,
block_id=i,
name='BLOCK ' + str(i) + '-1')
for j in range(1, RESNET_N):
net = resblock(net,
filters=int(FILTERS[i] * RESNET_K),
kernel_size=KERNEL_SIZES[i],
preactivated=False,
block_id=i+j,
name='BLOCK ' + str(i) + '-' + str(j + 1))
# Post Activation
net = l.batch_norm(net)
net = l.NonlinearityLayer(net, nonlinearity=nl.rectify)
# Classification branch
#log.p(("\tCLASS BRANCH:"))
net = classificationBranch(net, (4, 10))
#log.p(("\t\tBRANCH OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))
# Pooling
net = l.GlobalPoolLayer(net, pool_function=logmeanexp)
#log.p(("\tGLOBAL POOLING SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net)) - 1))
# Sigmoid output
net = l.NonlinearityLayer(net, nonlinearity=nl.sigmoid)
#log.p(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net), "LAYER:", len(l.get_all_layers(net))))
log.p("DONE!")
# Model stats
#log.p(("MODEL HAS", (sum(hasattr(layer, 'W') for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"))
#log.p(("MODEL HAS", l.count_params(net), "PARAMS"))
return net
def build_baseline_model():
log.i('BUILDING BASELINE MODEL...')
# Random Seed
lasagne_random.set_rng(cfg.getRandomState())
# Input layer for images
net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0]))
# Stride size (as an alternative to max pooling)
if cfg.MAX_POOLING:
s = 1
else:
s = 2
# Convolutinal layer groups
for i in range(len(cfg.FILTERS)):
# 3x3 Convolution + Stride
net = batch_norm(
l.Conv2DLayer(net,
num_filters=cfg.FILTERS[i],
filter_size=cfg.KERNEL_SIZES[i],
num_groups=cfg.NUM_OF_GROUPS[i],
pad='same',
stride=s,
W=initialization(cfg.NONLINEARITY),
nonlinearity=nonlinearity(cfg.NONLINEARITY)))
# Pooling layer
if cfg.MAX_POOLING:
net = l.MaxPool2DLayer(net, pool_size=2)
# Dropout Layer (we support different types of dropout)
if cfg.DROPOUT_TYPE == 'channels' and cfg.DROPOUT > 0.0:
net = l.dropout_channels(net, p=cfg.DROPOUT)
elif cfg.DROPOUT_TYPE == 'location' and cfg.DROPOUT > 0.0:
net = l.dropout_location(net, p=cfg.DROPOUT)
elif cfg.DROPOUT > 0.0:
net = l.DropoutLayer(net, p=cfg.DROPOUT)
log.i(('\tGROUP', i + 1, 'OUT SHAPE:', l.get_output_shape(net)))
# Final 1x1 Convolution
net = batch_norm(
l.Conv2DLayer(net,
num_filters=cfg.FILTERS[i] * 2,
filter_size=1,
W=initialization('identity'),
nonlinearity=nonlinearity('identity')))
log.i(('\tFINAL CONV OUT SHAPE:', l.get_output_shape(net)))
# Global Pooling layer (default mode = average)
net = l.GlobalPoolLayer(net)
log.i(("\tFINAL POOLING SHAPE:", l.get_output_shape(net)))
# Classification Layer (Softmax)
net = l.DenseLayer(net,
len(cfg.CLASSES),
nonlinearity=nonlinearity('softmax'),
W=initialization('softmax'))
log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net)))
log.i("...DONE!")
# Model stats
log.i(("MODEL HAS",
(sum(hasattr(layer, 'W')
for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"))
log.i(("MODEL HAS", l.count_params(net), "PARAMS"))
return net
def dmr_regularizer(f_1, f2):
x_bin = f_1 / (T.abs_(f_1) + args.gamma)
x_bin_2 = T.sgn(f2)
M_reg = (T.dot(x_bin_2, x_bin_2.T) - T.dot(x_bin_2, x_bin_2.T) *
(T.eye(x_bin_2.shape[0]))) / x_bin_2.shape[1]
M_true = (T.dot(x_bin, x_bin.T) - T.dot(x_bin, x_bin.T) *
(T.eye(x_bin.shape[0]))) / x_bin.shape[1]
l_reg = T.sum(T.abs_(M_reg - M_true)) / (x_bin.shape[0] *
(x_bin.shape[0] - 1))
return l_reg
if args.dataset_type == 'brown':
global_pool = ll.GlobalPoolLayer(disc_layers[f_low_dim])
global_pool_2 = ll.ReshapeLayer(disc_layers[f_high_dim], ([0], -1))
else:
global_pool = disc_layers[f_low_dim]
global_pool_2 = ll.GlobalPoolLayer(disc_layers[f_high_dim])
features_1 = ll.get_output(global_pool, x_unl_1, deterministic=False)
features_2 = ll.get_output(global_pool_2, x_unl_1, deterministic=False)
loss_unl = loss_unl + args.lambda_dmr * dmr_regularizer(features_1, features_2) \
+ args.lambda_bre * bre_regularizer(features_1, features_2)
# Theano functions for training the disc net
lr = T.scalar()
disc_params = ll.get_all_params(disc_layers, trainable=True)
disc_param_updates = nn.adam_updates(disc_params, loss_unl, lr=lr, mom1=0.5)
disc_param_avg = [
def build(self):
""""""
# output setting
out_acts = []
for out_act in self.config.hyper_parameters.out_act:
# exec('from lasagne.nonlinearities import {}'.format(out_act))
out_acts.append(eval(out_act))
n_outs = self.config.hyper_parameters.n_out
targets = self.config.target
# apply size multiplier
self.n_filters = map(lambda x: x * self.m, self.n_filters)
# network dict & set input block
self.net, self.variables['sigma'] = input_block(self.net,
self.config,
melspec=True)
if self.branch_at == 'fc':
# shares all layers except output layer
for i in range(len(self.n_convs)):
name = 'conv{:d}'.format(i + 1)
self.net = conv_block(self.net, self.n_convs[i],
self.n_filters[i], self.filter_sizes[i],
self.strides[i], self.pool_sizes[i],
self.non_lin, self.batch_norm, name,
self.net.keys()[-1], self.verbose)
# GAP
self.net['gap'] = L.batch_norm(
L.GlobalPoolLayer(self.net[next(reversed(self.net))],
name='gap'))
self.net['fc'] = L.dropout(L.batch_norm(
L.DenseLayer(self.net['gap'],
num_units=self.net['gap'].output_shape[-1],
nonlinearity=self.non_lin,
name='fc')),
name='fc.bn.do')
# Out block
out_layer_names = []
for target, n_out, out_act in zip(targets, n_outs, out_acts):
out_layer_names.append('{}.out'.format(target))
if target == 'self':
self.net[out_layer_names[-1]], inputs = \
build_siamese(self.net['fc'])
else:
self.net[out_layer_names[-1]] = L.DenseLayer(
self.net['fc'],
num_units=n_out,
nonlinearity=out_act,
name=out_layer_names[-1])
inputs = [self.net['input'].input_var]
self.variables['{}.inputs'.format(target)] = inputs
else:
# shares lower part of the network and branch out
# shared conv blocks
for i in range(self.branch_at - 1):
name = 'conv{:d}'.format(i + 1)
self.net = conv_block(self.net, self.n_convs[i],
self.n_filters[i], self.filter_sizes[i],
self.strides[i], self.pool_sizes[i],
self.non_lin, self.batch_norm, name,
self.net.keys()[-1], self.verbose)
branch_point = self.net.keys()[-1]
# branch out to each targets
out_layer_names = []
for target, n_out, out_act in zip(targets, n_outs, out_acts):
# first conv_block for each branch
j = self.branch_at - 1 # branch_point_ix
name = '{}.conv{:d}'.format(target, j + 1)
self.net = conv_block(self.net, self.n_convs[j],
self.n_filters[j], self.filter_sizes[j],
self.strides[j], self.pool_sizes[j],
self.non_lin, self.batch_norm, name,
branch_point, self.verbose)
for i in range(self.branch_at, len(self.n_convs)):
name = '{}.conv{:d}'.format(target, i + 1)
self.net = conv_block(self.net, self.n_convs[i],
self.n_filters[i],
self.filter_sizes[i],
self.strides[i], self.pool_sizes[i],
self.non_lin, self.batch_norm, name,
self.net.keys()[-1], self.verbose)
# GAP
gap_name = '{}.gap'.format(target)
self.net[gap_name] = L.batch_norm(
L.GlobalPoolLayer(self.net[next(reversed(self.net))],
name=gap_name))
# FC
fc_name = '{}.fc'.format(target)
self.net[fc_name] = L.dropout(L.batch_norm(
L.DenseLayer(self.net[gap_name],
num_units=self.net[gap_name].output_shape[-1],
nonlinearity=self.non_lin,
name=fc_name)),
name=fc_name)
# OUT
out_layer_names.append('{}.out'.format(target))
if target == 'self':
self.net[out_layer_names[-1]], inputs = \
build_siamese(self.net[fc_name])
else:
self.net[out_layer_names[-1]] = L.DenseLayer(
self.net[fc_name],
num_units=n_out,
nonlinearity=out_act,
name=out_layer_names[-1])
inputs = [self.net['input'].input_var]
self.variables['{}.inputs'.format(target)] = inputs
# make a concatation layer just for save/load purpose
self.net['IO'] = L.ConcatLayer([
L.FlattenLayer(self.net[target_layer_name])
if target == 'self' else self.net[target_layer_name]
for target_layer_name in out_layer_names
],
name='IO')
if self.verbose:
for target in targets:
print(self.net['{}.out'.format(target)].output_shape)
return self.net, self.variables
def fcn_multiscale(
input_var_list,
early_conv_dict_list,
late_conv_dict,
dense_filter_size,
final_pool_function=T.max,
input_size_list=[128], output_size=188,
p_dropout=0.5
):
'''
early_conv_dict_list: list
each element in the list is a dictionary containing the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
late_conv_dict: dict
it contains the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
dense_filter_size: int
the filter size of the final dense-like conv layer
pool_filter_list: list
each element is an integer
pool_stride_list: list
each element is int or None
'''
assert(len(early_conv_dict_list) == len(input_var_list) ==
len(input_size_list))
# early conv layers
conv_network_list = list()
total_stride_list = list()
for jj, [early_conv_dict, input_var, input_size] in enumerate(zip(
early_conv_dict_list, input_var_list, input_size_list)):
input_network = lasagne.layers.InputLayer(
shape=(None, 1, None, input_size), input_var=input_var)
total_stride = 1
network, total_stride = conv_layers(input_network, early_conv_dict,
total_stride,
init_input_size=input_size,
p_dropout=0,
base_name='early{}'.format(jj))
total_stride_list.append(total_stride)
conv_network_list.append(network)
# Concatenate
network = layers.ConcatLayer(conv_network_list,
axis=1,
cropping=[None, None, 'lower', None],
name='MultisourceConcatenate')
# late conv layers (dense layers)
network, total_stride = conv_layers(network, late_conv_dict,
total_stride,
init_input_size=1,
p_dropout=p_dropout,
base_name='late')
# frame output layer. every frame has a value
network = cl.Conv2DXLayer(
lasagne.layers.dropout(network, p=p_dropout),
num_filters=output_size, filter_size=(dense_filter_size, 1),
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.GlorotUniform()
)
# pool
network = layers.GlobalPoolLayer(network,
pool_function=final_pool_function)
network = layers.ReshapeLayer(network, ([0], -1))
return network
def build_critic(self, version=1, extra_middle=False, only_middle=False):
assert self.generator != None
if version == 1:
from lasagne.nonlinearities import sigmoid
disc_layers = [
ll.InputLayer(shape=(self.batch_size, 3, 64, 64),
input_var=self.input_c)
]
# b_s x 3 x 64 x 64 --> b_s x 32 x 32 x 32
disc_layers.append(
nn.batch_norm(
ll.Conv2DLayer(disc_layers[-1],
128, (3, 3),
pad=1,
stride=2,
W=Normal(0.03),
nonlinearity=nn.lrelu))) #nn.weight_norm
# disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
# b_s x 32 x 32 x 32 --> b_s x 64 x 16 x 16
disc_layers.append(
nn.batch_norm(
ll.Conv2DLayer(disc_layers[-1],
256, (3, 3),
pad=1,
stride=2,
W=Normal(0.03),
nonlinearity=nn.lrelu))) #nn.weight_norm
# b_s x 64 x 16 x 16 --> b_s x 128 x 8 x 8
# note : this layer will be used to compare statistics (output of disc_layer[3])
disc_layers.append(
nn.batch_norm(
ll.Conv2DLayer(disc_layers[-1],
512, (3, 3),
pad=1,
stride=2,
W=Normal(0.03),
nonlinearity=nn.lrelu))) #nn.weight_norm
# b_s x 128 x 8 x 8 --> b_s x 256 x 4 x 4
disc_layers.append(
nn.batch_norm(
ll.Conv2DLayer(disc_layers[-1],
1024, (3, 3),
pad=1,
stride=2,
W=Normal(0.03),
nonlinearity=nn.lrelu))) #nn.weight_norm
disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
disc_layers.append(
nn.MinibatchLayer(disc_layers[-1], num_kernels=100))
if extra_middle or only_middle:
# goal : add mechanism that checks for mode collapse, but only for the middle part
layer = nn.ExtractMiddleLayer(disc_layers[0], extra=2)
# two convolutions
print layer.output_shape
layer = nn.batch_norm(
ll.Conv2DLayer(layer,
128,
5,
stride=2,
pad='same',
nonlinearity=nn.lrelu))
print layer.output_shape
layer = nn.batch_norm(
ll.Conv2DLayer(layer,
256,
5,
stride=2,
pad='same',
nonlinearity=nn.lrelu))
print layer.output_shape
layer = nn.batch_norm(
ll.Conv2DLayer(layer,
512,
5,
stride=2,
pad='same',
nonlinearity=nn.lrelu))
#layer = nn.batch_norm(ll.Conv2DLayer(layer, 1024, 5, stride=2, pad='same',
# nonlinearity=nn.lrelu))
#disc_layers.append(layer)
#print layer.output_shape
layer = ll.GlobalPoolLayer(layer)
#print layer.output_shape
layer = nn.MinibatchLayer(layer, num_kernels=400)
if only_middle:
disc_layers = []
disc_layers.append((ll.DenseLayer(layer,
num_units=1,
W=Normal(0.03),
nonlinearity=None)))
return disc_layers
disc_layers.append(ll.ConcatLayer([layer, disc_layers[-1]]))
disc_layers.append((ll.DenseLayer(disc_layers[-1],
num_units=1,
W=Normal(0.03),
nonlinearity=None)))
# seeing how there is strong intra-batch normalization, I think it would be good to do minibatch discrimination solely on the center of the images.
# I think that the statistics of the border are throwing
for layer in disc_layers:
print layer.output_shape
print ''
return disc_layers
def build_fcn_fixedgaussian_multiscale(num_scales):
input_var_list = list()
earlyconv_list = list()
# A stack of convolution layers for each scale
for ii in range(num_scales):
# input tensor
input_var = T.tensor4('input.{}'.format(ii))
# input layer
network = lasagne.layers.InputLayer(shape=(None, 1, None, 128),
input_var=input_var)
# early conv layers
network = cl.Conv2DXLayer(network,
num_filters=32,
filter_size=(8, 128),
stride=(1, 1),
nonlinearity=lasagne.nonlinearities.rectify,
pad='strictsamex',
name='early_conv.{}_1'.format(ii))
network = layers.MaxPool2DLayer(network,
pool_size=(4, 1),
stride=(4, 1),
ignore_border=False,
name='early_maxpool.{}_1'.format(ii))
network = cl.Conv2DXLayer(network,
num_filters=32,
filter_size=(8, 1),
stride=(1, 1),
nonlinearity=lasagne.nonlinearities.rectify,
pad='strictsamex',
name='early_conv.{}_2'.format(ii))
network = layers.MaxPool2DLayer(network,
pool_size=(4, 1),
stride=(4, 1),
ignore_border=False,
name='early_maxpool.{}_2'.format(ii))
input_var_list.append(input_var)
earlyconv_list.append(network)
network = layers.ConcatLayer(earlyconv_list,
axis=1,
cropping=[None, None, 'lower', None],
name='multiscale_concat')
# late conv layers
network = layers.Conv2DLayer(layers.dropout(network, p=0.5),
512, (1, 1), (1, 1),
name='late_conv.{}_1'.format(ii))
network = layers.Conv2DLayer(layers.dropout(network, p=0.5),
512, (1, 1), (1, 1),
name='late_conv.{}_2'.format(ii))
network = layers.Conv2DLayer(layers.dropout(network, p=0.5),
188, (1, 1), (1, 1),
nonlinearity=lasagne.nonlinearities.sigmoid,
name='late_conv.{}_output'.format(ii))
# Gaussian scanning
network = layers.ReshapeLayer(network, ([0], [1], [2]))
nogaussian_layer = network
network = cl.FixedGaussianScan1DLayer(network,
filter_size=256,
init_std=256 / 4**2,
stride=1,
pad='strictsame',
name='fixed_gaussian_filter')
gaussian_layer = network
# pool
network = layers.GlobalPoolLayer(network, pool_function=T.mean)
return network, input_var_list, nogaussian_layer, gaussian_layer
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0.5, ps=2, n_kerns=32, d_kerns=(5, 5),
pad='valid', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
name='cla-1'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=64, d_kerns=(3, 3),
pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
name='cla-2'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0.5, ps=2, n_kerns=64, d_kerns=(3, 3),
pad='valid', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
name='cla-3'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=128, d_kerns=(3, 3),
pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
name='cla-4'))
cla_layers.append(convlayer(l=cla_layers[-1], bn=True, dr=0, ps=1, n_kerns=128, d_kerns=(3, 3),
pad='same', stride=1, W=Normal(0.05), nonlinearity=ln.rectify,
name='cla-5'))
cla_layers.append(ll.GlobalPoolLayer(cla_layers[-1]))
cla_layers.append(ll.DenseLayer(cla_layers[-1], num_units=num_classes, W=lasagne.init.Normal(1e-2, 0),
nonlinearity=ln.softmax, name='cla-6'))
classifier = cla_layers[-1] #
gen_in_z = ll.InputLayer(shape=(None, n_z))
gen_in_y = ll.InputLayer(shape=(None,))
gen_layers = [gen_in_z]
gen_layers.append(MLPConcatLayer([gen_layers[-1], gen_in_y], num_classes, name='gen-1'))
gen_layers.append(ll.batch_norm(ll.DenseLayer(gen_layers[-1], num_units=500,
nonlinearity=ln.softplus, name='gen-2'), name='gen-3'))
gen_layers.append(MLPConcatLayer([gen_layers[-1], gen_in_y], num_classes, name='gen-4'))
def get_discriminator_brown(num_feature=256):
disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))]
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
96, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
96, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
96, (3, 3),
pad=1,
stride=2,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
128, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
128, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
128, (3, 3),
pad=1,
stride=2,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
128, (3, 3),
pad=0,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
ll.NINLayer(disc_layers[-1],
num_units=num_feature,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
ll.NINLayer(disc_layers[-1],
num_units=128,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
disc_layers.append(
nn.weight_norm(ll.DenseLayer(disc_layers[-1],
num_units=2,
W=Normal(0.05),
nonlinearity=None),
train_g=True,
init_stdv=0.1))
#disc_layers.append(ll.ReshapeLayer(disc_layers[-4], ([0], -1)))
#disc_layers.append(ll.GlobalPoolLayer(disc_layers[-4]))
disc_layer_features_low_dim = -4
disc_layer_features_high_dim = -5
return disc_layers, disc_layer_features_low_dim, disc_layer_features_high_dim
def build_resnet_model():
log.i('BUILDING RESNET MODEL...')
# Random Seed
lasagne_random.set_rng(cfg.getRandomState())
# Input layer for images
net = l.InputLayer((None, cfg.IM_DIM, cfg.IM_SIZE[1], cfg.IM_SIZE[0]))
# First Convolution
net = l.Conv2DLayer(net,
num_filters=cfg.FILTERS[0],
filter_size=cfg.KERNEL_SIZES[0],
pad='same',
W=initialization(cfg.NONLINEARITY),
nonlinearity=None)
log.i(("\tFIRST CONV OUT SHAPE:", l.get_output_shape(net), "LAYER:",
len(l.get_all_layers(net)) - 1))
# Residual Stacks
for i in range(0, len(cfg.FILTERS)):
net = resblock(net,
filters=cfg.FILTERS[i] * cfg.RESNET_K,
kernel_size=cfg.KERNEL_SIZES[i],
stride=2,
num_groups=cfg.NUM_OF_GROUPS[i])
for _ in range(1, cfg.RESNET_N):
net = resblock(net,
filters=cfg.FILTERS[i] * cfg.RESNET_K,
kernel_size=cfg.KERNEL_SIZES[i],
num_groups=cfg.NUM_OF_GROUPS[i],
preactivated=False)
log.i(("\tRES STACK", i + 1, "OUT SHAPE:", l.get_output_shape(net),
"LAYER:", len(l.get_all_layers(net)) - 1))
# Post Activation
net = batch_norm(net)
net = l.NonlinearityLayer(net, nonlinearity=nonlinearity(cfg.NONLINEARITY))
# Pooling
net = l.GlobalPoolLayer(net)
log.i(("\tFINAL POOLING SHAPE:", l.get_output_shape(net), "LAYER:",
len(l.get_all_layers(net)) - 1))
# Classification Layer
net = l.DenseLayer(net,
len(cfg.CLASSES),
nonlinearity=nonlinearity('identity'),
W=initialization('identity'))
net = l.NonlinearityLayer(net, nonlinearity=nonlinearity('softmax'))
log.i(("\tFINAL NET OUT SHAPE:", l.get_output_shape(net), "LAYER:",
len(l.get_all_layers(net))))
log.i("...DONE!")
# Model stats
log.i(("MODEL HAS",
(sum(hasattr(layer, 'W')
for layer in l.get_all_layers(net))), "WEIGHTED LAYERS"))
log.i(("MODEL HAS", l.count_params(net), "PARAMS"))
return net
disc_x_layers.append(LL.GaussianNoiseLayer(disc_x_layers[-1], sigma=0.2)) disc_x_layers.append(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 96, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=1, stride=2, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers.append(LL.DropoutLayer(disc_x_layers[-1], p=0.5)) disc_x_layers.append(nn.batch_norm(dnn.Conv2DDNNLayer(disc_x_layers[-1], 192, (3,3), pad=0, W=Normal(0.01), nonlinearity=nn.lrelu))) disc_x_layers_shared = LL.NINLayer(disc_x_layers[-1], num_units=192, W=Normal(0.01), nonlinearity=nn.lrelu) disc_x_layers.append(disc_x_layers_shared) disc_x_layer_z_recon = LL.DenseLayer(disc_x_layers_shared, num_units=50, nonlinearity=None) disc_x_layers.append(disc_x_layer_z_recon) # also need to recover z from x # disc_x_layers.append(nn.MinibatchLayer(disc_x_layers_shared, num_kernels=100)) disc_x_layers.append(LL.GlobalPoolLayer(disc_x_layers_shared)) disc_x_layer_adv = LL.DenseLayer(disc_x_layers[-1], num_units=10, W=Normal(0.01), nonlinearity=None) disc_x_layers.append(disc_x_layer_adv) #output_before_softmax_x = LL.get_output(disc_x_layer_adv, x, deterministic=False) #output_before_softmax_gen = LL.get_output(disc_x_layer_adv, gen_x, deterministic=False) # temp = LL.get_output(gen_x_layers[-1], deterministic=False, init=True) # temp = LL.get_output(disc_x_layers[-1], x, deterministic=False, init=True) # init_updates = [u for l in LL.get_all_layers(gen_x_layers)+LL.get_all_layers(disc_x_layers) for u in getattr(l,'init_updates',[])] output_before_softmax_real = LL.get_output(disc_x_layer_adv, x, deterministic=False) output_before_softmax_gen, recon_z = LL.get_output([disc_x_layer_adv, disc_x_layer_z_recon], gen_x, deterministic=False) # discriminator's predicted probability that gen_x is real l_lab = output_before_softmax_real[T.arange(args.batch_size),y] l_unl = nn.log_sum_exp(output_before_softmax_real)
nn.batch_norm(dnn.Conv2DDNNLayer(genz_layers[-1],
512, (3, 3),
pad=1,
stride=2,
W=Normal(0.05),
nonlinearity=nn.lrelu,
name='gz2'),
g=None))
genz_layers.append(
nn.batch_norm(ll.NINLayer(genz_layers[-1],
num_units=512,
W=Normal(0.05),
nonlinearity=nn.lrelu,
name='gz5'),
g=None))
genz_layers.append(ll.GlobalPoolLayer(genz_layers[-1], name='gz6'))
genz_layers.append(
ll.DenseLayer(genz_layers[-1],
num_units=100,
W=Normal(0.05),
nonlinearity=lasagne.nonlinearities.sigmoid,
name='gz7'))
# specify discriminative model
# for z
discz_layers = [z_input]
discz_layers.append(ll.DropoutLayer(discz_layers[-1], p=0.2))
discz_layers.append(
ll.DenseLayer(discz_layers[-1],
500,
stretch=True)
img = plotting.plot_img(img_tile, title='')
plotting.plt.savefig(args.data_name + "_patches_gen.png")
img_bhwc = np.transpose(trainx[:100, ], (0, 2, 3, 1))
img_tile = plotting.img_tile(img_bhwc,
aspect_ratio=1.0,
border_color=1.0,
stretch=True)
img = plotting.plot_img(img_tile, title='')
plotting.plt.savefig(args.data_name + "_patches_train.png")
evens = [x for x in range(int(test_matched.shape[0] / 2)) if x % 2 == 0]
odds = [x for x in range(int(test_matched.shape[0] / 2)) if x % 2 == 1]
disc_layers.append(ll.GlobalPoolLayer(disc_layers[f_low_dim]))
batch_size_test = 100
print('Extracting features from matched data...')
features_matched = extract_features(disc_layers, test_matched, batch_size_test)
print('Extracting features from unmatched data...')
features_unmatched = extract_features(disc_layers, test_unmatched,
batch_size_test)
print('Number of features considered in the experiment: ' +
str(features_matched[evens].shape[1]))
features_matched[features_matched >= 0.0] = 1
features_matched[features_matched < 0.0] = 0
features_unmatched[features_unmatched >= 0.0] = 1
def cls_net(_incoming,
noise_sigma=0.05,
drop_ratio_conv=0.3,
drop_ratio_fc=0.5):
_noise = L.GaussianNoiseLayer(_incoming, sigma=noise_sigma)
_conv1 = conv(_noise,
num_filters=128,
filter_size=4,
stride=2,
pad=1,
W=I.Normal(0.02),
b=I.Constant(0.),
nonlinearity=NL.rectify)
_lrn1 = L.LocalResponseNormalization2DLayer(_conv1,
alpha=0.0001,
k=2,
beta=0.75,
n=5)
_drop2 = L.DropoutLayer(_lrn1, p=drop_ratio_conv, rescale=True)
_conv2 = batch_norm_n(
conv(_drop2,
num_filters=256,
filter_size=4,
stride=2,
pad=1,
W=I.Normal(0.02),
b=None,
nonlinearity=NL.rectify))
_drop3 = L.DropoutLayer(_conv2, p=drop_ratio_conv, rescale=True)
_conv3 = batch_norm_n(
conv(_drop3,
num_filters=384,
filter_size=3,
stride=1,
pad=0,
W=I.Normal(0.02),
b=None,
nonlinearity=NL.rectify))
_drop4 = L.DropoutLayer(_conv3, p=drop_ratio_conv, rescale=True)
_conv4 = batch_norm_n(
conv(_drop4,
num_filters=512,
filter_size=3,
stride=1,
pad=0,
W=I.Normal(0.02),
b=None,
nonlinearity=NL.rectify))
_drop5 = L.DropoutLayer(_conv4, p=drop_ratio_fc, rescale=True)
_conv5 = batch_norm_n(
conv(_drop5,
num_filters=1024,
filter_size=1,
stride=1,
pad=0,
W=I.Normal(0.02),
b=None,
nonlinearity=NL.rectify))
_pool6 = L.GlobalPoolLayer(_conv5, pool_function=theano.tensor.max) #mean
_drop6 = L.DropoutLayer(_pool6, p=drop_ratio_fc, rescale=True)
_fc6 = L.DenseLayer(_drop6,
ny,
W=I.Normal(0.02),
b=I.Constant(0),
nonlinearity=NL.softmax)
_aux = [
_conv1, _conv2, _conv3, _conv4, _conv5,
L.DimshuffleLayer(_fc6, (0, 1, 'x', 'x'))
] ### used to have a tanh() around everything except last
return _aux, _fc6
def get_discriminator_binary():
disc_layers = [ll.InputLayer(shape=(None, 3, 32, 32))]
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.2))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
96, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
96, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
96, (3, 3),
pad=1,
stride=2,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
192, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
192, (3, 3),
pad=1,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
192, (3, 3),
pad=1,
stride=2,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(ll.DropoutLayer(disc_layers[-1], p=0.5))
disc_layers.append(
nn.weight_norm(
dnn.Conv2DDNNLayer(disc_layers[-1],
192, (3, 3),
pad=0,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
ll.NINLayer(disc_layers[-1],
num_units=192,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(
nn.weight_norm(
ll.NINLayer(disc_layers[-1],
num_units=192,
W=Normal(0.05),
nonlinearity=nn.lrelu)))
disc_layers.append(ll.GlobalPoolLayer(disc_layers[-1]))
disc_layers.append(
nn.weight_norm(ll.DenseLayer(disc_layers[-1],
num_units=2,
W=Normal(0.05),
nonlinearity=None),
train_g=True,
init_stdv=0.1))
disc_layers.append(
ll.DenseLayer(disc_layers[-2],
num_units=128,
W=Normal(0.05),
nonlinearity=T.nnet.sigmoid))
return disc_layers
def fcn(input_var,
early_conv_dict,
late_conv_dict,
dense_filter_size,
final_pool_function=T.max,
input_size=128,
output_size=188,
p_dropout=0.5):
'''
early_conv_dict: dict
it contains the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
pool_filter_list: list
each element is an integer
pool_stride_list: list
each element is int or None
late_conv_dict: dict
it contains the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
dense_filter_size: int
the filter size of the final dense-like conv layer
'''
# early conv layers
input_network = lasagne.layers.InputLayer(shape=(None, 1, None,
input_size),
input_var=input_var)
total_stride = 1
network, total_stride = conv_layers(input_network,
early_conv_dict,
total_stride,
init_input_size=input_size,
p_dropout=0,
base_name='early')
# late conv layers (dense layers)
network, total_stride = conv_layers(network,
late_conv_dict,
total_stride,
init_input_size=1,
p_dropout=p_dropout,
base_name='late')
# frame output layer. every frame has a value
network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout),
num_filters=output_size,
filter_size=(dense_filter_size, 1),
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.GlorotUniform())
# pool
network = layers.GlobalPoolLayer(network,
pool_function=final_pool_function)
network = layers.ReshapeLayer(network, ([0], -1))
return network
def p_fcn(input_var,
early_conv_dict,
pl_dict,
late_conv_dict,
dense_filter_size,
final_pool_function=T.max,
input_size=128,
output_size=188,
p_dropout=0.5):
'''
This uses PL 2D layer instead 2D layer, comparing to fcn_pnn_multisource
early_conv_dict_list: list
each element in the list is a dictionary containing
the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
pool_filter_list: list
each element is an integer
pool_stride_list: list
each element is int or None
pl_dict: dict
it contains the following keys:
'num_lambda', 'num_points', 'value_range', 'seg_size', 'seg_stride'
late_conv_dict: dict
it contains the following keys:
'conv_filter_list', 'pool_filter_list', 'pool_stride_list'
dense_filter_size: int
the filter size of the final dense-like conv layer
'''
# early conv layers
input_network = lasagne.layers.InputLayer(shape=(None, 1, None,
input_size),
input_var=input_var)
total_stride = 1
network, total_stride = conv_layers(input_network,
early_conv_dict,
total_stride,
init_input_size=input_size,
p_dropout=0,
base_name='early')
# Persistence landscape
num_lambda = pl_dict['num_lambda']
num_points = pl_dict['num_points']
value_range = pl_dict['value_range']
seg_size = pl_dict['seg_size']
seg_step = pl_dict['seg_step']
patch_size = (seg_size, 1)
patch_step = (seg_step, 1)
network = cl.PersistenceFlatten2DLayer(network, num_lambda, num_points,
value_range, patch_size, patch_step)
# late conv layers (dense layers)
network, total_stride = conv_layers(network,
late_conv_dict,
total_stride,
init_input_size=1,
p_dropout=p_dropout,
base_name='late')
# frame output layer. every frame has a value
network = cl.Conv2DXLayer(lasagne.layers.dropout(network, p=p_dropout),
num_filters=output_size,
filter_size=(dense_filter_size, 1),
nonlinearity=lasagne.nonlinearities.sigmoid,
W=lasagne.init.GlorotUniform())
# pool
network = layers.GlobalPoolLayer(network,
pool_function=final_pool_function)
network = layers.ReshapeLayer(network, ([0], -1))
return network
def __init__(
self,
image_shape,
filter_shape,
num_class,
conv_type,
kernel_size,
kernel_pool_size,
dropout_rate,
):
"""
"""
self.filter_shape = filter_shape
self.n_visible = numpy.prod(image_shape)
self.n_layers = len(filter_shape)
self.rng = RandomStreams(123)
self.x = T.matrix()
self.y = T.ivector()
self.conv_layers = []
NoiseLayer = layers.DropoutLayer
dropout_rate = float(dropout_rate)
self.l_input = layers.InputLayer((None, self.n_visible), self.x)
this_layer = layers.ReshapeLayer(self.l_input, ([0], ) + image_shape)
for l in range(self.n_layers):
activation = lasagne.nonlinearities.rectify
if len(filter_shape[l]) == 3:
if conv_type == 'double' and filter_shape[l][1] > kernel_size:
this_layer = DoubleConvLayer(
this_layer,
filter_shape[l][0],
filter_shape[l][1:],
pad='same',
nonlinearity=activation,
kernel_size=kernel_size,
kernel_pool_size=kernel_pool_size)
this_layer = layers.batch_norm(this_layer)
elif conv_type == 'maxout':
this_layer = layers.Conv2DLayer(this_layer,
filter_shape[l][0],
filter_shape[l][1:],
b=None,
pad='same',
nonlinearity=None)
this_layer = layers.FeaturePoolLayer(
this_layer, pool_size=kernel_pool_size**2)
this_layer = layers.BatchNormLayer(this_layer)
this_layer = layers.NonlinearityLayer(
this_layer, activation)
elif conv_type == 'cyclic':
this_layers = []
this_layers.append(
layers.Conv2DLayer(this_layer,
filter_shape[l][0],
filter_shape[l][1:],
b=None,
pad='same',
nonlinearity=None))
for _ in range(3):
W = this_layers[-1].W.dimshuffle(0, 1, 3,
2)[:, :, :, ::-1]
this_layers.append(
layers.Conv2DLayer(this_layer,
filter_shape[l][0],
filter_shape[l][1:],
W=W,
b=None,
pad='same',
nonlinearity=None))
this_layer = layers.ElemwiseMergeLayer(
this_layers, T.maximum)
this_layer = layers.BatchNormLayer(this_layer)
this_layer = layers.NonlinearityLayer(
this_layer, activation)
elif conv_type == 'standard' \
or (conv_type == 'double' and filter_shape[l][1] <= kernel_size):
this_layer = layers.Conv2DLayer(this_layer,
filter_shape[l][0],
filter_shape[l][1:],
pad='same',
nonlinearity=activation)
this_layer = layers.batch_norm(this_layer)
else:
raise NotImplementedError
self.conv_layers.append(this_layer)
elif len(filter_shape[l]) == 2:
this_layer = layers.MaxPool2DLayer(this_layer, filter_shape[l])
this_layer = NoiseLayer(this_layer, dropout_rate)
elif len(filter_shape[l]) == 1:
raise NotImplementedError
self.top_conv_layer = this_layer
this_layer = layers.GlobalPoolLayer(this_layer, T.mean)
self.clf_layer = layers.DenseLayer(this_layer,
num_class,
W=lasagne.init.Constant(0.),
nonlinearity=T.nnet.softmax)
self.params = layers.get_all_params(self.clf_layer, trainable=True)
self.params_all = layers.get_all_params(self.clf_layer)
def deep_cnn_2d_mtl_at_2(params):
""""""
assert 'targets' in params
nonlin = nl.elu
layers = L.InputLayer((None, 1, params['dur'], 128))
print layers.output_shape
sclr = joblib.load(params['scaler'])
layers = L.standardize(layers,
sclr.mean_.astype(np.float32),
sclr.scale_.astype(np.float32),
shared_axes=(0, 1, 2))
print layers.output_shape
n_filter = [32, 64, 64, 128, 256, 256] # l
if 'kernel_multiplier' in params:
n_filter = map(lambda x: x * params['kernel_multiplier'], n_filter)
filter_sz = [(3, 3), (3, 3), (3, 3), (3, 3), (3, 3), (1, 1)] # m
if params['dur'] > 50:
conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] # c
pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), None, None] # n
else:
conv_strd = [(1, 1), (1, 1), (1, 1), (1, 1), (1, 1), (1, 1)] # c
pool_sz = [(2, 2), (2, 2), (2, 2), (2, 2), None, None] # n
pool_strd = [None, None, None, None, None, None] # s
batch_norm = [True, False, True, False, False, False] # b
dropout = [True, False, True, False, False, False] # d # added
conv_spec = zip(n_filter, filter_sz, conv_strd, pool_sz, pool_strd,
batch_norm, dropout)
# Shared first layer
if 'kernel_multiplier' in params:
n_1st_ker = 16 * params['kernel_multiplier']
else:
n_1st_ker = 16
layers = L.Conv2DLayer(layers,
n_1st_ker, (5, 5),
stride=(1, 1),
pad='same',
nonlinearity=nonlin)
layers = L.MaxPool2DLayer(layers, pool_size=(1, 2))
layers = L.dropout(layers, p=0.1)
layer_heads = OrderedDict()
for target in params['targets']:
first_trgt_spec_layer = True # n_layer checker
for l, m, c, n, s, b, d in conv_spec:
if first_trgt_spec_layer:
layer_heads[target['name']] = layers
first_trgt_spec_layer = False
if b:
layer_heads[target['name']] = L.batch_norm(
L.Conv2DLayer(layer_heads[target['name']],
l,
m,
stride=c,
pad='same',
nonlinearity=nonlin), )
else:
layer_heads[target['name']] = L.Conv2DLayer(
layer_heads[target['name']],
l,
m,
stride=c,
pad='same',
nonlinearity=nonlin)
if n is not None:
layer_heads[target['name']] = L.MaxPool2DLayer(
layer_heads[target['name']], pool_size=n, stride=s)
if d:
layer_heads[target['name']] = L.dropout(
layer_heads[target['name']], p=0.1)
print layer_heads[target['name']].output_shape
layer_heads[target['name']] = L.batch_norm(
L.GlobalPoolLayer(layer_heads[target['name']]))
layer_heads[target['name']] = L.dropout(
layer_heads[target['name']]) # added
print layer_heads[target['name']].output_shape
if 'kernel_multiplier' in params:
n_hid = 256 * params['kernel_multiplier']
else:
n_hid = 256
layer_heads[target['name']] = L.batch_norm(
L.DenseLayer(layer_heads[target['name']],
n_hid,
nonlinearity=nonlin))
layer_heads[target['name']] = L.dropout(layer_heads[target['name']])
print layer_heads[target['name']].output_shape
layer_heads[target['name']] = L.DenseLayer(layer_heads[target['name']],
target['n_out'],
nonlinearity=nl.softmax)
print target['name'], layer_heads[target['name']].output_shape
return layer_heads
