def build_model(in_shape=INPUT_SHAPE):
""" Compile net architecture """
nonlin = elu
net1 = lasagne.layers.InputLayer(shape=(None, in_shape[0], in_shape[1],
in_shape[2]),
name='Input')
# number of filters in first layer
# decreased by factor 2 in each block
nf0 = 8
# --- encoder ---
net1 = conv_bn(net1,
num_filters=nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
p1 = net1
net1 = MaxPool2DLayer(net1, pool_size=2, stride=2)
net1 = conv_bn(net1,
num_filters=2 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=2 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
p2 = net1
net1 = MaxPool2DLayer(net1, pool_size=2, stride=2)
net1 = conv_bn(net1,
num_filters=4 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=4 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
p3 = net1
net1 = MaxPool2DLayer(net1, pool_size=2, stride=2)
net1 = conv_bn(net1,
num_filters=8 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=8 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
# --- decoder ---
net1 = TransposedConv2DLayer(net1,
num_filters=4 * nf0,
filter_size=2,
stride=2)
net1 = batch_norm(net1)
net1 = ElemwiseSumLayer((p3, net1))
net1 = batch_norm(net1)
net1 = conv_bn(net1,
num_filters=4 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=4 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = DropoutLayer(net1, p=0.2)
net1 = TransposedConv2DLayer(net1,
num_filters=2 * nf0,
filter_size=2,
stride=2)
net1 = batch_norm(net1)
net1 = ElemwiseSumLayer((p2, net1))
net1 = batch_norm(net1)
net1 = conv_bn(net1,
num_filters=2 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=2 * nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = DropoutLayer(net1, p=0.1)
net1 = TransposedConv2DLayer(net1,
num_filters=nf0,
filter_size=2,
stride=2)
net1 = batch_norm(net1)
net1 = ElemwiseSumLayer((p1, net1))
net1 = batch_norm(net1)
net1 = conv_bn(net1,
num_filters=nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = conv_bn(net1,
num_filters=nf0,
filter_size=3,
nonlinearity=nonlin,
pad='same')
net1 = Conv2DLayer(net1,
num_filters=1,
filter_size=1,
nonlinearity=sigmoid,
pad='same')
return net1
def build_model(weights_path, options):
"""
Build the CNN model. Create the Neural Net object and return it back.
Inputs:
- subject name: used to save the net weights accordingly.
- options: several hyper-parameters used to configure the net.
Output:
- net: a NeuralNet object
"""
net_model_name = options['experiment']
try:
os.mkdir(os.path.join(weights_path, net_model_name))
except:
pass
net_weights = os.path.join(weights_path, net_model_name,
net_model_name + '.pkl')
net_history = os.path.join(weights_path, net_model_name,
net_model_name + '_history.pkl')
# select hyper-parameters
t_verbose = options['net_verbose']
train_split_perc = options['train_split']
num_epochs = options['max_epochs']
max_epochs_patience = options['patience']
early_stopping = EarlyStopping(patience=max_epochs_patience)
save_weights = SaveWeights(net_weights, only_best=True, pickle=False)
save_training_history = SaveTrainingHistory(net_history)
# build the architecture
ps = options['patch_size'][0]
num_channels = 1
fc_conv = 180
fc_fc = 180
dropout_conv = 0.5
dropout_fc = 0.5
# --------------------------------------------------
# channel_1: axial
# --------------------------------------------------
axial_ch = InputLayer(name='in1', shape=(None, num_channels, ps, ps))
axial_ch = prelu(batch_norm(
Conv2DLayer(axial_ch,
name='axial_ch_conv1',
num_filters=20,
filter_size=3)),
name='axial_ch_prelu1')
axial_ch = prelu(batch_norm(
Conv2DLayer(axial_ch,
name='axial_ch_conv2',
num_filters=20,
filter_size=3)),
name='axial_ch_prelu2')
axial_ch = MaxPool2DLayer(axial_ch, name='axial_max_pool_1', pool_size=2)
axial_ch = prelu(batch_norm(
Conv2DLayer(axial_ch,
name='axial_ch_conv3',
num_filters=40,
filter_size=3)),
name='axial_ch_prelu3')
axial_ch = prelu(batch_norm(
Conv2DLayer(axial_ch,
name='axial_ch_conv4',
num_filters=40,
filter_size=3)),
name='axial_ch_prelu4')
axial_ch = MaxPool2DLayer(axial_ch, name='axial_max_pool_2', pool_size=2)
axial_ch = prelu(batch_norm(
Conv2DLayer(axial_ch,
name='axial_ch_conv5',
num_filters=60,
filter_size=3)),
name='axial_ch_prelu5')
axial_ch = DropoutLayer(axial_ch, name='axial_l1drop', p=dropout_conv)
axial_ch = DenseLayer(axial_ch, name='axial_d1', num_units=fc_conv)
axial_ch = prelu(axial_ch, name='axial_prelu_d1')
# --------------------------------------------------
# channel_1: coronal
# --------------------------------------------------
coronal_ch = InputLayer(name='in2', shape=(None, num_channels, ps, ps))
coronal_ch = prelu(batch_norm(
Conv2DLayer(coronal_ch,
name='coronal_ch_conv1',
num_filters=20,
filter_size=3)),
name='coronal_ch_prelu1')
coronal_ch = prelu(batch_norm(
Conv2DLayer(coronal_ch,
name='coronal_ch_conv2',
num_filters=20,
filter_size=3)),
name='coronal_ch_prelu2')
coronal_ch = MaxPool2DLayer(coronal_ch,
name='coronal_max_pool_1',
pool_size=2)
coronal_ch = prelu(batch_norm(
Conv2DLayer(coronal_ch,
name='coronal_ch_conv3',
num_filters=40,
filter_size=3)),
name='coronal_ch_prelu3')
coronal_ch = prelu(batch_norm(
Conv2DLayer(coronal_ch,
name='coronal_ch_conv4',
num_filters=40,
filter_size=3)),
name='coronal_ch_prelu4')
coronal_ch = MaxPool2DLayer(coronal_ch,
name='coronal_max_pool_2',
pool_size=2)
coronal_ch = prelu(batch_norm(
Conv2DLayer(coronal_ch,
name='coronal_ch_conv5',
num_filters=60,
filter_size=3)),
name='coronal_ch_prelu5')
coronal_ch = DropoutLayer(coronal_ch,
name='coronal_l1drop',
p=dropout_conv)
coronal_ch = DenseLayer(coronal_ch, name='coronal_d1', num_units=fc_conv)
coronal_ch = prelu(coronal_ch, name='coronal_prelu_d1')
# --------------------------------------------------
# channel_1: saggital
# --------------------------------------------------
saggital_ch = InputLayer(name='in3', shape=(None, num_channels, ps, ps))
saggital_ch = prelu(batch_norm(
Conv2DLayer(saggital_ch,
name='saggital_ch_conv1',
num_filters=20,
filter_size=3)),
name='saggital_ch_prelu1')
saggital_ch = prelu(batch_norm(
Conv2DLayer(saggital_ch,
name='saggital_ch_conv2',
num_filters=20,
filter_size=3)),
name='saggital_ch_prelu2')
saggital_ch = MaxPool2DLayer(saggital_ch,
name='saggital_max_pool_1',
pool_size=2)
saggital_ch = prelu(batch_norm(
Conv2DLayer(saggital_ch,
name='saggital_ch_conv3',
num_filters=40,
filter_size=3)),
name='saggital_ch_prelu3')
saggital_ch = prelu(batch_norm(
Conv2DLayer(saggital_ch,
name='saggital_ch_conv4',
num_filters=40,
filter_size=3)),
name='saggital_ch_prelu4')
saggital_ch = MaxPool2DLayer(saggital_ch,
name='saggital_max_pool_2',
pool_size=2)
saggital_ch = prelu(batch_norm(
Conv2DLayer(saggital_ch,
name='saggital_ch_conv5',
num_filters=60,
filter_size=3)),
name='saggital_ch_prelu5')
saggital_ch = DropoutLayer(saggital_ch,
name='saggital_l1drop',
p=dropout_conv)
saggital_ch = DenseLayer(saggital_ch,
name='saggital_d1',
num_units=fc_conv)
saggital_ch = prelu(saggital_ch, name='saggital_prelu_d1')
# FC layer 540
layer = ConcatLayer(name='elem_channels',
incomings=[axial_ch, coronal_ch, saggital_ch])
layer = DropoutLayer(layer, name='f1_drop', p=dropout_fc)
layer = DenseLayer(layer, name='FC1', num_units=540)
layer = prelu(layer, name='prelu_f1')
# concatenate channels 540 + 15
layer = DropoutLayer(layer, name='f2_drop', p=dropout_fc)
atlas_layer = DropoutLayer(InputLayer(name='in4', shape=(None, 15)),
name='Dropout_atlas',
p=.2)
atlas_layer = InputLayer(name='in4', shape=(None, 15))
layer = ConcatLayer(name='elem_channels2', incomings=[layer, atlas_layer])
# FC layer 270
layer = DenseLayer(layer, name='fc_2', num_units=270)
layer = prelu(layer, name='prelu_f2')
# FC output 15 (softmax)
net_layer = DenseLayer(layer,
name='out_layer',
num_units=15,
nonlinearity=softmax)
net = NeuralNet(
layers=net_layer,
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
update_learning_rate=0.001,
on_epoch_finished=[
save_weights,
save_training_history,
early_stopping,
],
verbose=t_verbose,
max_epochs=num_epochs,
train_split=TrainSplit(eval_size=train_split_perc),
)
if options['load_weights'] == 'True':
try:
print " --> loading weights from ", net_weights
net.load_params_from(net_weights)
except:
pass
return net
def build_network_cifar10_v2(input, nbClasses):
net = {}
net['input'] = InputLayer((None, 1, 120, 120), input_var=input)
net['conv1'] = ConvLayer(net['input'],
num_filters=192,
filter_size=5,
pad=2,
flip_filters=False)
net['cccp1'] = ConvLayer(net['conv1'],
num_filters=160,
filter_size=1,
flip_filters=False)
net['cccp2'] = ConvLayer(net['cccp1'],
num_filters=96,
filter_size=1,
flip_filters=False)
net['pool1'] = PoolLayer(net['cccp2'],
pool_size=3,
stride=2,
mode='max',
ignore_border=False)
net['drop3'] = DropoutLayer(net['pool1'], p=0.5)
net['conv2'] = ConvLayer(net['drop3'],
num_filters=192,
filter_size=5,
pad=2,
flip_filters=False)
net['cccp3'] = ConvLayer(net['conv2'],
num_filters=192,
filter_size=1,
flip_filters=False)
net['cccp4'] = ConvLayer(net['cccp3'],
num_filters=192,
filter_size=1,
flip_filters=False)
net['pool2'] = PoolLayer(net['cccp4'],
pool_size=3,
stride=2,
mode='average_exc_pad',
ignore_border=False)
net['drop6'] = DropoutLayer(net['pool2'], p=0.5)
net['conv3'] = ConvLayer(net['drop6'],
num_filters=192,
filter_size=3,
pad=1,
flip_filters=False)
net['cccp5'] = ConvLayer(net['conv3'],
num_filters=192,
filter_size=1,
flip_filters=False)
net['cccp6'] = ConvLayer(net['cccp5'],
num_filters=10,
filter_size=1,
flip_filters=False)
net['pool3'] = PoolLayer(net['cccp6'],
pool_size=8,
mode='average_exc_pad',
ignore_border=False)
# net['output'] = FlattenLayer(net['pool3'])
net['dense1'] = lasagne.layers.DenseLayer(
net['pool3'],
nonlinearity=lasagne.nonlinearities.identity,
num_units=1024)
net['output'] = lasagne.layers.DenseLayer(
net['dense1'],
nonlinearity=lasagne.nonlinearities.softmax,
num_units=nbClasses)
return net, net['output']
class BatchNormLayer(Layer):
def __init__(self,
incoming,
axes='auto',
droprate=0.2,
epsilon=1e-4,
alpha=0.1,
sparsity=1.0,
beta=init.Constant(0),
gamma=init.Constant(1),
mean=init.Constant(0),
inv_std=init.Constant(1),
**kwargs):
super(BatchNormLayer, self).__init__(incoming, **kwargs)
if axes == 'auto':
# default: normalize over all but the second axis
axes = (0, ) + tuple(range(2, len(self.input_shape)))
elif isinstance(axes, int):
axes = (axes, )
self.axes = axes
if len(axes) == 1:
self.mean_axes = self.axes
else:
self.mean_axes = (axes[1], )
self.epsilon = epsilon
self.alpha = alpha
# create parameters, ignoring all dimensions in axes
shape = [
size for axis, size in enumerate(self.input_shape)
if axis not in self.axes
]
meanshape = [
size for axis, size in enumerate(self.input_shape)
if axis not in self.mean_axes
]
if any(size is None for size in shape):
raise ValueError("BatchNormLayer needs specified input sizes for "
"all axes not normalized over.")
if beta is None:
self.beta = None
else:
self.beta = self.add_param(beta,
shape,
'beta',
trainable=True,
regularizable=False)
if gamma is None:
self.gamma = None
else:
self.gamma = self.add_param(gamma,
shape,
'gamma',
trainable=True,
regularizable=True)
self.mean = self.add_param(mean,
meanshape,
'mean',
trainable=False,
regularizable=False)
self.inv_std = self.add_param(inv_std,
meanshape,
'inv_std',
trainable=False,
regularizable=False)
#print('here',len(self.input_shape))
self.sparsity = sparsity
if len(self.input_shape) == 3:
self.dropout = DropoutLayer(
(self.input_shape[0], self.input_shape[1],
self.input_shape[2]),
p=droprate,
shared_axes=(0, 1),
**kwargs)
else:
self.dropout = DropoutLayer(
(self.input_shape[0], self.input_shape[1]),
p=droprate,
shared_axes=(0, ),
**kwargs)
def get_output_for(self,
input,
deterministic=False,
batch_norm_use_averages=None,
batch_norm_update_averages=None,
**kwargs):
if self.sparsity == 1:
input_mean = input.mean(self.mean_axes)
input_inv_std = T.inv(
T.sqrt(input.var(self.mean_axes) + self.epsilon))
else:
input_mean = input.mean(self.mean_axes) * (1.0 / self.sparsity)
input_inv_std = T.inv(
T.sqrt(
input.var(self.mean_axes) * (1.0 / self.sparsity) -
(1 - self.sparsity) * T.sqr(input_mean) + self.epsilon))
# Decide whether to use the stored averages or mini-batch statistics
if batch_norm_use_averages is None:
batch_norm_use_averages = deterministic
use_averages = batch_norm_use_averages
if use_averages:
mean = self.mean
inv_std = self.inv_std
else:
mean = input_mean
inv_std = input_inv_std
# Decide whether to update the stored averages
if batch_norm_update_averages is None:
batch_norm_update_averages = not deterministic
update_averages = batch_norm_update_averages
if update_averages:
# Trick: To update the stored statistics, we create memory-aliased
# clones of the stored statistics:
running_mean = theano.clone(self.mean, share_inputs=False)
running_inv_std = theano.clone(self.inv_std, share_inputs=False)
# set a default update for them:
running_mean.default_update = ((1 - self.alpha) * running_mean +
self.alpha * input_mean)
running_inv_std.default_update = (
(1 - self.alpha) * running_inv_std +
self.alpha * input_inv_std)
# and make sure they end up in the graph without participating in
# the computation (this way their default_update will be collected
# and applied, but the computation will be optimized away):
mean += 0 * running_mean
inv_std += 0 * running_inv_std
# prepare dimshuffle pattern inserting broadcastable axes as needed
param_axes = iter(range(input.ndim - len(self.axes)))
pattern = [
'x' if input_axis in self.axes else next(param_axes)
for input_axis in range(input.ndim)
]
# apply dimshuffle pattern to all parameters
beta = 0 if self.beta is None else self.beta.dimshuffle(pattern)
gamma = 1 if self.gamma is None else self.gamma.dimshuffle(pattern)
mean_param_axes = iter(range(input.ndim - len(self.mean_axes)))
mean_pattern = [
'x' if input_axis in self.mean_axes else next(mean_param_axes)
for input_axis in range(input.ndim)
]
mean = mean.dimshuffle(mean_pattern)
inv_std = inv_std.dimshuffle(mean_pattern)
input = self.dropout.get_output_for(input, deterministic=deterministic)
# normalize
normalized = (input - mean) * (gamma * inv_std) + beta
return normalized
def build(myNet, idxSiam, verbose=True):
INITIALIZATION_GAIN = 1.0
# -----------------------------------------------------------------------------
# input layer (2d croped patch)
# myNet.layers[idxSiam]['ori-input']
# -----------------------------------------------------------------------------
# 3x Convolution and Max Pooling layers
# --------------
# Conv 0
if idxSiam == 0:
W_init = HeNormal(gain=INITIALIZATION_GAIN)
# W_init = Constant(0.0)
b_init = Constant(0.0)
else:
W_init = myNet.layers[0]['ori-c0'].W
b_init = myNet.layers[0]['ori-c0'].b
myNet.layers[idxSiam]['ori-c0'] = Conv2DLayer(
myNet.layers[idxSiam]['ori-input'],
num_filters=10,
filter_size=5,
W=W_init,
b=b_init,
nonlinearity=None,
flip_filters=False,
name='ori-c0',
)
# Activation 0
myNet.layers[idxSiam]['ori-c0a'] = NonlinearityLayer(
myNet.layers[idxSiam]['ori-c0'],
nonlinearity=relu,
name='ori-c0a',
)
# Pool 0
myNet.layers[idxSiam]['ori-c0p'] = MaxPool2DLayer(
myNet.layers[idxSiam]['ori-c0a'],
pool_size=2,
name='ori-c0p',
)
# --------------
# Conv 1
if idxSiam == 0:
W_init = HeNormal(gain=INITIALIZATION_GAIN)
# W_init = Constant(0.0)
b_init = Constant(0.0)
else:
W_init = myNet.layers[0]['ori-c1'].W
b_init = myNet.layers[0]['ori-c1'].b
myNet.layers[idxSiam]['ori-c1'] = Conv2DLayer(
myNet.layers[idxSiam]['ori-c0p'],
num_filters=20,
filter_size=5,
W=W_init,
b=b_init,
nonlinearity=None,
flip_filters=False,
name='ori-c1',
)
# Activation 1
myNet.layers[idxSiam]['ori-c1a'] = NonlinearityLayer(
myNet.layers[idxSiam]['ori-c1'],
nonlinearity=relu,
name='ori-c1a',
)
# Pool 1
myNet.layers[idxSiam]['ori-c1p'] = MaxPool2DLayer(
myNet.layers[idxSiam]['ori-c1a'],
pool_size=2,
name='ori-c1p',
)
# --------------
# Conv 2
if idxSiam == 0:
W_init = HeNormal(gain=INITIALIZATION_GAIN)
# W_init = Constant(0.0)
b_init = Constant(0.0)
else:
W_init = myNet.layers[0]['ori-c2'].W
b_init = myNet.layers[0]['ori-c2'].b
myNet.layers[idxSiam]['ori-c2'] = Conv2DLayer(
myNet.layers[idxSiam]['ori-c1p'],
num_filters=50,
filter_size=3,
W=W_init,
b=b_init,
nonlinearity=None,
flip_filters=False,
name='ori-c2',
)
# Activation 2
myNet.layers[idxSiam]['ori-c2a'] = NonlinearityLayer(
myNet.layers[idxSiam]['ori-c2'],
nonlinearity=relu,
name='ori-c2a',
)
# Pool 2
myNet.layers[idxSiam]['ori-c2p'] = MaxPool2DLayer(
myNet.layers[idxSiam]['ori-c2a'],
pool_size=2,
name='ori-c2p',
)
# -----------------------------------------------------------------------------
# Fully Connected Layers
# --------------
# FC 3
nu = 100
ns = 4
nm = 4
if idxSiam == 0:
W_init = HeNormal(gain=INITIALIZATION_GAIN)
# W_init = Constant(0.0)
b_init = Constant(0.0)
else:
W_init = myNet.layers[0]['ori-f3'].W
b_init = myNet.layers[0]['ori-f3'].b
myNet.layers[idxSiam]['ori-f3'] = DenseLayer(
myNet.layers[idxSiam]['ori-c2a'],
num_units=nu * ns * nm,
W=W_init,
b=b_init,
nonlinearity=None,
name='ori-f3',
)
# Activation 3
myNet.layers[idxSiam]['ori-f3a'] = GHHFeaturePoolLayer(
myNet.layers[idxSiam]['ori-f3'],
num_in_sum=ns,
num_in_max=nm,
max_strength=myNet.config.max_strength,
name='ori-f3a',
)
# Dropout 3
myNet.layers[idxSiam]['ori-f3d'] = DropoutLayer(
myNet.layers[idxSiam]['ori-f3a'],
p=0.3,
name='ori-f3d',
)
# --------------
# FC 4
nu = 2
ns = 4
nm = 4
if idxSiam == 0:
W_init = HeNormal(gain=INITIALIZATION_GAIN)
# W_init = Constant(0.0)
b_init = Constant(0.0)
else:
W_init = myNet.layers[0]['ori-f4'].W
b_init = myNet.layers[0]['ori-f4'].b
myNet.layers[idxSiam]['ori-f4'] = DenseLayer(
myNet.layers[idxSiam]['ori-f3d'],
num_units=nu * ns * nm,
W=W_init,
b=b_init,
nonlinearity=None,
name='ori-f4',
)
# Activation 4
myNet.layers[idxSiam]['ori-f4a'] = GHHFeaturePoolLayer(
myNet.layers[idxSiam]['ori-f4'],
num_in_sum=ns,
num_in_max=nm,
max_strength=myNet.config.max_strength,
name='ori-f4a',
)
# -----------------------------------------------------------------------------
# Arctan2 Layer
myNet.layers[idxSiam]['ori-output'] = ExpressionLayer(
myNet.layers[idxSiam]['ori-f4a'],
lambda x: CT.custom_arctan2(x[:, 0], x[:, 1]).flatten().dimshuffle(
0, 'x'),
output_shape=(myNet.config.batch_size, 1),
name='ori-output',
)
def residual_block(l, increase_dim=False, first=False, filters=16):
if increase_dim:
first_stride = (2, 2)
else:
first_stride = (1, 1)
if first:
# hacky solution to keep layers correct
bn_pre_relu = l
else:
# contains the BN -> ReLU portion, steps 1 to 2
bn_pre_conv = BatchNormLayer(l)
bn_pre_relu = NonlinearityLayer(bn_pre_conv, rectify)
# contains the weight -> BN -> ReLU portion, steps 3 to 5
conv_1 = batch_norm(
ConvLayer(bn_pre_relu,
num_filters=filters,
filter_size=(3, 3),
stride=first_stride,
nonlinearity=rectify,
pad='same',
W=HeNormal(gain='relu')))
if dropoutrate > 0: # with dropout
dropout = DropoutLayer(conv_1, p=dropoutrate)
# contains the last weight portion, step 6
conv_2 = ConvLayer(dropout,
num_filters=filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=None,
pad='same',
W=HeNormal(gain='relu'))
else: # without dropout
conv_2 = ConvLayer(conv_1,
num_filters=filters,
filter_size=(3, 3),
stride=(1, 1),
nonlinearity=None,
pad='same',
W=HeNormal(gain='relu'))
# add shortcut connections
if increase_dim:
# projection shortcut, as option B in paper
projection = ConvLayer(l,
num_filters=filters,
filter_size=(1, 1),
stride=(2, 2),
nonlinearity=None,
pad='same',
b=None)
block = ElemwiseSumLayer([conv_2, projection])
elif first:
# projection shortcut, as option B in paper
projection = ConvLayer(l,
num_filters=filters,
filter_size=(1, 1),
stride=(1, 1),
nonlinearity=None,
pad='same',
b=None)
block = ElemwiseSumLayer([conv_2, projection])
else:
block = ElemwiseSumLayer([conv_2, l])
return block
command1 = "mkdir -p" +" "+ wts_path
process1 = subprocess.check_call(command1.split())
command2 = "mkdir -p" +" "+ epoch_path
process2 = subprocess.check_call(command2.split())
network={}
X = T.tensor3(name='features',dtype='float32')
Masks = T.matrix(name='masks',dtype='float32')
labels = T.ivector(name='spk-phr-labels')
print("Building network ...")
network['input'] = InputLayer(shape=(None,382,40), input_var = X)
batchs,_,_ = network['input'].input_var.shape
#added for lstm-dropout
network['drop1'] = DropoutLayer(network['input'],p=0.3)
network['mask'] = InputLayer((None,None), input_var = Masks)
gate_parameters = lasagne.layers.recurrent.Gate(
W_in=lasagne.init.Orthogonal(), W_hid=lasagne.init.Orthogonal(),
b=lasagne.init.Constant(0.))
cell_parameters = lasagne.layers.recurrent.Gate(
W_in=lasagne.init.Orthogonal(), W_hid=lasagne.init.Orthogonal(),
W_cell=None, b=lasagne.init.Constant(0.),
nonlinearity=lasagne.nonlinearities.tanh)
network['lstm-forward'] = lasagne.layers.recurrent.LSTMLayer(network['drop1'],600, mask_input=network['mask'],
ingate=gate_parameters, forgetgate=gate_parameters, cell=cell_parameters, outgate=gate_parameters,
def _build_network(self):
net = OrderedDict()
net['input'] = InputLayer((None, self.n_input_channels,
self.input_dim[0], self.input_dim[1]))
net['contr_1_1'] = batch_norm(
ConvLayer(net['input'],
self.base_n_filters,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['contr_1_2'] = batch_norm(
ConvLayer(net['contr_1_1'],
self.base_n_filters,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['pool1'] = Pool2DLayer(net['contr_1_2'], 2)
net['contr_2_1'] = batch_norm(
ConvLayer(net['pool1'],
self.base_n_filters * 2,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['contr_2_2'] = batch_norm(
ConvLayer(net['contr_2_1'],
self.base_n_filters * 2,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['pool2'] = Pool2DLayer(net['contr_2_2'], 2)
net['contr_3_1'] = batch_norm(
ConvLayer(net['pool2'],
self.base_n_filters * 4,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['contr_3_2'] = batch_norm(
ConvLayer(net['contr_3_1'],
self.base_n_filters * 4,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['pool3'] = Pool2DLayer(net['contr_3_2'], 2)
net['contr_4_1'] = batch_norm(
ConvLayer(net['pool3'],
self.base_n_filters * 8,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['contr_4_2'] = batch_norm(
ConvLayer(net['contr_4_1'],
self.base_n_filters * 8,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2)
# the paper does not really describe where and how dropout is added. Feel free to try more options
if self.do_dropout:
l = DropoutLayer(l, p=0.4)
net['encode_1'] = batch_norm(
ConvLayer(l,
self.base_n_filters * 16,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['encode_2'] = batch_norm(
ConvLayer(net['encode_1'],
self.base_n_filters * 16,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['upscale1'] = batch_norm(
Deconv2DLayer(net['encode_2'],
self.base_n_filters * 16,
2,
2,
crop="valid",
nonlinearity=self.nonlinearity,
W=HeNormal(gain="relu")))
net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']],
cropping=(None, None, "center", "center"))
net['expand_1_1'] = batch_norm(
ConvLayer(net['concat1'],
self.base_n_filters * 8,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['expand_1_2'] = batch_norm(
ConvLayer(net['expand_1_1'],
self.base_n_filters * 8,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['upscale2'] = batch_norm(
Deconv2DLayer(net['expand_1_2'],
self.base_n_filters * 8,
2,
2,
crop="valid",
nonlinearity=self.nonlinearity,
W=HeNormal(gain="relu")))
net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']],
cropping=(None, None, "center", "center"))
net['expand_2_1'] = batch_norm(
ConvLayer(net['concat2'],
self.base_n_filters * 4,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['expand_2_2'] = batch_norm(
ConvLayer(net['expand_2_1'],
self.base_n_filters * 4,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['upscale3'] = batch_norm(
Deconv2DLayer(net['expand_2_2'],
self.base_n_filters * 4,
2,
2,
crop="valid",
nonlinearity=self.nonlinearity,
W=HeNormal(gain="relu")))
net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']],
cropping=(None, None, "center", "center"))
net['expand_3_1'] = batch_norm(
ConvLayer(net['concat3'],
self.base_n_filters * 2,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['expand_3_2'] = batch_norm(
ConvLayer(net['expand_3_1'],
self.base_n_filters * 2,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['upscale4'] = batch_norm(
Deconv2DLayer(net['expand_3_2'],
self.base_n_filters * 2,
2,
2,
crop="valid",
nonlinearity=self.nonlinearity,
W=HeNormal(gain="relu")))
net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']],
cropping=(None, None, "center", "center"))
net['expand_4_1'] = batch_norm(
ConvLayer(net['concat4'],
self.base_n_filters,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['expand_4_2'] = batch_norm(
ConvLayer(net['expand_4_1'],
self.base_n_filters,
3,
nonlinearity=self.nonlinearity,
pad=self.pad,
W=HeNormal(gain="relu")))
net['output'] = ConvLayer(net['expand_4_2'],
self.n_input_channels,
1,
nonlinearity=T.tanh)
return net
def execute(dataset,
n_hidden_u,
num_epochs=500,
learning_rate=.001,
learning_rate_annealing=1.0,
lmd=.0001,
embedding_input='raw',
which_fold=0,
save_path='/Tmp/$USER/DietNetworks/newmodel/',
save_copy='/Tmp/$USER/DietNetworks/newmodel/',
dataset_path='/Tmp/$USER/DietNetworks/newmodel/'):
# Load the dataset
print("Loading data")
x_unsup = mlh.load_data(dataset,
dataset_path,
None,
which_fold=which_fold,
keep_labels=1.0,
missing_labels_val=-1.0,
embedding_input=embedding_input,
transpose=True)
x_train = x_unsup[0][0]
x_valid = x_unsup[1][0]
# Extract required information from data
n_row, n_col = x_train.shape
print('Data size ' + str(n_row) + 'x' + str(n_col))
# Set some variables
batch_size = 256
# Define experiment name
exp_name = 'pretrain_' + mlh.define_exp_name(
1., 0, 0, 0, lmd, n_hidden_u, [], [], [], which_fold, embedding_input,
learning_rate, 0, 0, 'reconst_loss', learning_rate_annealing)
print('Experiment: ' + exp_name)
# Preparing folder to save stuff
save_path = os.path.join(save_path, dataset, exp_name)
save_copy = os.path.join(save_copy, dataset, exp_name)
if not os.path.exists(save_path):
os.makedirs(save_path)
# Prepare Theano variables for inputs and targets
input_var = T.matrix('input_unsup')
lr = theano.shared(np.float32(learning_rate), 'learning_rate')
# Build model
print("Building model")
# Some checkings
assert len(n_hidden_u) > 0
# Build unsupervised network
encoder_net = InputLayer((None, n_col), input_var)
for out in n_hidden_u:
encoder_net = DenseLayer(encoder_net, num_units=out, nonlinearity=tanh)
encoder_net = DropoutLayer(encoder_net)
decoder_net = encoder_net
for i in range(len(n_hidden_u) - 2, -1, -1):
decoder_net = DenseLayer(decoder_net,
num_units=n_hidden_u[i],
nonlinearity=linear)
decoder_net = DropoutLayer(decoder_net)
decoder_net = DenseLayer(decoder_net, num_units=n_col, nonlinearity=linear)
if embedding_input == 'raw' or embedding_input == 'w2v':
final_nonlin = linear
elif embedding_input == 'bin':
final_nonlin = sigmoid
elif 'histo' in embedding_input:
final_nonlin = softmax
if embedding_input == 'histo3x26':
laySize = lasagne.layers.get_output(decoder_net).shape
decoder_net = ReshapeLayer(decoder_net, (laySize[0] * 26, 3))
decoder_net = NonlinearityLayer(decoder_net, nonlinearity=final_nonlin)
if embedding_input == 'histo3x26':
decoder_net = ReshapeLayer(decoder_net, (laySize[0], laySize[1]))
print("Building and compiling training functions")
# Build and compile training functions
predictions, predictions_det = mh.define_predictions(
[encoder_net, decoder_net], start=0)
prediction_sup, prediction_sup_det = mh.define_predictions(
[encoder_net, decoder_net], start=0)
# Define losses
# reconstruction losses
loss, loss_det = mh.define_loss(predictions[1], predictions_det[1],
input_var, embedding_input)
# Define parameters
params = lasagne.layers.get_all_params(decoder_net, trainable=True)
l2_penalty = apply_penalty(params, l2)
loss = loss + lmd * l2_penalty
loss_det = loss_det + lmd * l2_penalty
# Compute network updates
updates = lasagne.updates.adam(loss, params, learning_rate=lr)
# updates = lasagne.updates.sgd(loss,
# params,
# learning_rate=lr)
# updates = lasagne.updates.momentum(loss, params,
# learning_rate=lr, momentum=0.0)
# Apply norm constraints on the weights
for k in updates.keys():
if updates[k].ndim == 2:
updates[k] = lasagne.updates.norm_constraint(updates[k], 1.0)
# Compile training function
train_fn = theano.function([input_var],
loss,
updates=updates,
on_unused_input='ignore')
# Expressions required for test
monitor_labels = ['loss']
val_outputs = [loss_det]
# Add some monitoring on the learned feature embedding
val_outputs += [
predictions[0].min(), predictions[0].mean(), predictions[0].max(),
predictions[0].var()
]
monitor_labels += [
"feat. emb. min", "feat. emb. mean", "feat. emb. max", "feat. emb. var"
]
# Compile validation function
val_fn = theano.function([input_var], val_outputs)
pred_feat_emb = theano.function([input_var], predictions_det[0])
# Finally, launch the training loop.
print("Starting training...")
# Some variables
max_patience = 100
patience = 0
train_monitored = []
valid_monitored = []
train_loss = []
nb_minibatches = n_row / batch_size
print("Nb of minibatches: " + str(nb_minibatches))
start_training = time.time()
for epoch in range(num_epochs):
start_time = time.time()
print("Epoch {} of {}".format(epoch + 1, num_epochs))
loss_epoch = 0
# Train pass
for batch in mlh.iterate_minibatches_unsup(x_train,
batch_size,
shuffle=True):
loss_epoch += train_fn(batch)
loss_epoch /= nb_minibatches
train_loss += [loss_epoch]
train_minibatches = mlh.iterate_minibatches_unsup(x_train,
batch_size,
shuffle=True)
train_err = mlh.monitoring(train_minibatches,
"train",
val_fn,
monitor_labels,
start=0)
train_monitored += [train_err]
# Validation pass
valid_minibatches = mlh.iterate_minibatches_unsup(x_valid,
batch_size,
shuffle=True)
valid_err = mlh.monitoring(valid_minibatches,
"valid",
val_fn,
monitor_labels,
start=0)
valid_monitored += [valid_err]
try:
early_stop_val = valid_err[monitor_labels.index('loss')]
except:
raise ValueError("There is no monitored value by the name of %s" %
early_stop_criterion)
# Eearly stopping
if epoch == 0:
best_valid = early_stop_val
elif early_stop_val < best_valid:
best_valid = early_stop_val
patience = 0
# Save stuff
np.savez(
os.path.join(save_path, 'model_enc_unsupervised_best.npz'),
*lasagne.layers.get_all_param_values(encoder_net))
np.savez(os.path.join(save_path, 'model_ae_unsupervised_best.npz'),
*lasagne.layers.get_all_param_values(encoder_net))
np.savez(os.path.join(save_path, "errors_unsupervised_best.npz"),
zip(*train_monitored), zip(*valid_monitored))
else:
patience += 1
# Save stuff
np.savez(
os.path.join(save_path, 'model_enc_unsupervised_last.npz'),
*lasagne.layers.get_all_param_values(encoder_net))
np.savez(os.path.join(save_path, 'model_ae_unsupervised_last.npz'),
*lasagne.layers.get_all_param_values(encoder_net))
np.savez(os.path.join(save_path, "errors_unsupervised_last.npz"),
zip(*train_monitored), zip(*valid_monitored))
# End training
if patience == max_patience or epoch == num_epochs - 1:
print(" Ending training")
# Load unsupervised best model
if not os.path.exists(save_path +
'/model_enc_unsupervised_best.npz'):
print("No saved model to be tested and/or generate"
" the embedding !")
else:
with np.load(save_path +
'/model_enc_unsupervised_best.npz', ) as f:
param_values = [
f['arr_%d' % i] for i in range(len(f.files))
]
lasagne.layers.set_all_param_values(
encoder_net, param_values)
# Save embedding
preds = []
for batch in mlh.iterate_minibatches_unsup(x_train,
1,
shuffle=False):
preds.append(pred_feat_emb(batch))
for batch in mlh.iterate_minibatches_unsup(x_valid,
1,
shuffle=False):
preds.append(pred_feat_emb(batch))
preds = np.vstack(preds)
np.savez(os.path.join(save_path, 'feature_embedding.npz'),
preds)
# Stop
print(" epoch time:\t\t\t{:.3f}s".format(time.time() - start_time))
break
print(" epoch time:\t\t\t{:.3f}s".format(time.time() - start_time))
# Anneal the learning rate
lr.set_value(float(lr.get_value() * learning_rate_annealing))
# Print all final errors for train, validation and test
print("Training time:\t\t\t{:.3f}s".format(time.time() - start_training))
# Copy files to loadpath
if save_path != save_copy:
print('Copying model and other training files to {}'.format(save_copy))
copy_tree(save_path, save_copy)
def create_model(incoming, options):
conv_num_filters1 = 100
conv_num_filters2 = 150
conv_num_filters3 = 200
filter_size1 = 5
filter_size2 = 5
filter_size3 = 3
pool_size = 2
encode_size = options['BOTTLENECK']
dense_mid_size = options['DENSE']
pad_in = 'valid'
pad_out = 'full'
scaled_tanh = create_scaled_tanh()
dropout0 = DropoutLayer(incoming, p=0.2, name='dropout0')
conv2d1 = Conv2DLayer(dropout0,
num_filters=conv_num_filters1,
filter_size=filter_size1,
pad=pad_in,
name='conv2d1',
nonlinearity=scaled_tanh)
bn1 = BatchNormLayer(conv2d1, name='batchnorm1')
maxpool2d2 = MaxPool2DLayer(bn1, pool_size=pool_size, name='maxpool2d2')
dropout1 = DropoutLayer(maxpool2d2, name='dropout1')
conv2d3 = Conv2DLayer(dropout1,
num_filters=conv_num_filters2,
filter_size=filter_size2,
pad=pad_in,
name='conv2d3',
nonlinearity=scaled_tanh)
bn2 = BatchNormLayer(conv2d3, name='batchnorm2')
maxpool2d4 = MaxPool2DLayer(bn2,
pool_size=pool_size,
name='maxpool2d4',
pad=(1, 0))
dropout2 = DropoutLayer(maxpool2d4, name='dropout2')
conv2d5 = Conv2DLayer(dropout2,
num_filters=conv_num_filters3,
filter_size=filter_size3,
pad=pad_in,
name='conv2d5',
nonlinearity=scaled_tanh)
bn3 = BatchNormLayer(conv2d5, name='batchnorm3')
reshape6 = ReshapeLayer(bn3, shape=([0], -1), name='reshape6') # 3000
reshape6_output = reshape6.output_shape[1]
dropout3 = DropoutLayer(reshape6, name='dropout3')
dense7 = DenseLayer(dropout3,
num_units=dense_mid_size,
name='dense7',
nonlinearity=scaled_tanh)
bn4 = BatchNormLayer(dense7, name='batchnorm4')
dropout4 = DropoutLayer(bn4, name='dropout4')
bottleneck = DenseLayer(dropout4,
num_units=encode_size,
name='bottleneck',
nonlinearity=linear)
# print_network(bottleneck)
dense8 = DenseLayer(bottleneck,
num_units=dense_mid_size,
W=bottleneck.W.T,
name='dense8',
nonlinearity=linear)
dense9 = DenseLayer(dense8,
num_units=reshape6_output,
W=dense7.W.T,
nonlinearity=scaled_tanh,
name='dense9')
reshape10 = ReshapeLayer(dense9,
shape=([0], conv_num_filters3, 3, 5),
name='reshape10') # 32 x 4 x 7
deconv2d11 = Deconv2DLayer(reshape10,
conv2d5.input_shape[1],
conv2d5.filter_size,
stride=conv2d5.stride,
W=conv2d5.W,
flip_filters=not conv2d5.flip_filters,
name='deconv2d11',
nonlinearity=scaled_tanh)
upscale2d12 = Upscale2DLayer(deconv2d11,
scale_factor=pool_size,
name='upscale2d12')
deconv2d13 = Deconv2DLayer(upscale2d12,
conv2d3.input_shape[1],
conv2d3.filter_size,
stride=conv2d3.stride,
W=conv2d3.W,
flip_filters=not conv2d3.flip_filters,
name='deconv2d13',
nonlinearity=scaled_tanh)
upscale2d14 = Upscale2DLayer(deconv2d13,
scale_factor=pool_size,
name='upscale2d14')
deconv2d15 = Deconv2DLayer(upscale2d14,
conv2d1.input_shape[1],
conv2d1.filter_size,
stride=conv2d1.stride,
crop=(1, 0),
W=conv2d1.W,
flip_filters=not conv2d1.flip_filters,
name='deconv2d14',
nonlinearity=scaled_tanh)
reshape16 = ReshapeLayer(deconv2d15, ([0], -1), name='reshape16')
return reshape16, bottleneck
def build_net(input_var=None,
input_shape=(128, 128, 128),
num_output_classes=4,
num_input_channels=4,
base_n_filter=8,
do_instance_norm=True,
batch_size=None,
dropout_p=0.3,
do_norm=True):
nonlin = lasagne.nonlinearities.leaky_rectify
if do_instance_norm:
axes = (2, 3, 4)
else:
axes = 'auto'
def conv_norm_lrelu(l_in, feat_out):
l = Conv3DLayer(l_in,
feat_out,
3,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
if do_norm:
l = BatchNormLayer(l, axes=axes)
return NonlinearityLayer(l, nonlin)
def norm_lrelu_conv(l_in, feat_out, stride=1, filter_size=3):
if do_norm:
l_in = BatchNormLayer(l_in, axes=axes)
l = NonlinearityLayer(l_in, nonlin)
return Conv3DLayer(l,
feat_out,
filter_size,
stride,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
def lrelu_conv(l_in, feat_out, stride=1, filter_size=3):
l = NonlinearityLayer(l_in, nonlin)
return Conv3DLayer(l,
feat_out,
filter_size,
stride,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
def norm_lrelu_upscale_conv_norm_lrelu(l_in, feat_out):
if do_norm:
l_in = BatchNormLayer(l_in, axes=axes)
l = NonlinearityLayer(l_in, nonlin)
l = Upscale3DLayer(l, 2)
l = Conv3DLayer(l,
feat_out,
3,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = NonlinearityLayer(l, nonlin)
return l
l_in = InputLayer(shape=(batch_size, num_input_channels, input_shape[0],
input_shape[1], input_shape[2]),
input_var=input_var)
l = r = Conv3DLayer(l_in,
num_filters=base_n_filter,
filter_size=3,
stride=1,
nonlinearity=linear,
pad='same',
W=HeNormal(gain='relu'))
l = NonlinearityLayer(l, nonlin)
l = Conv3DLayer(l,
num_filters=base_n_filter,
filter_size=3,
stride=1,
nonlinearity=linear,
pad='same',
W=HeNormal(gain='relu'))
l = DropoutLayer(l, dropout_p)
l = lrelu_conv(l, base_n_filter, 1, 3)
l = ElemwiseSumLayer((l, r))
skip1 = NonlinearityLayer(l, nonlin)
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = NonlinearityLayer(l, nonlin)
l = r = Conv3DLayer(l,
base_n_filter * 2,
3,
2,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_conv(l, base_n_filter * 2)
l = DropoutLayer(l, dropout_p)
l = norm_lrelu_conv(l, base_n_filter * 2)
l = ElemwiseSumLayer((l, r))
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = skip2 = NonlinearityLayer(l, nonlin)
l = r = Conv3DLayer(l,
base_n_filter * 4,
3,
2,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_conv(l, base_n_filter * 4)
l = DropoutLayer(l, dropout_p)
l = norm_lrelu_conv(l, base_n_filter * 4)
l = ElemwiseSumLayer((l, r))
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = skip3 = NonlinearityLayer(l, nonlin)
l = r = Conv3DLayer(l,
base_n_filter * 8,
3,
2,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_conv(l, base_n_filter * 8)
l = DropoutLayer(l, dropout_p)
l = norm_lrelu_conv(l, base_n_filter * 8)
l = ElemwiseSumLayer((l, r))
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = skip4 = NonlinearityLayer(l, nonlin)
l = r = Conv3DLayer(l,
base_n_filter * 16,
3,
2,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_conv(l, base_n_filter * 16)
l = DropoutLayer(l, dropout_p)
l = norm_lrelu_conv(l, base_n_filter * 16)
l = ElemwiseSumLayer((l, r))
l = norm_lrelu_upscale_conv_norm_lrelu(l, base_n_filter * 8)
l = Conv3DLayer(l,
base_n_filter * 8,
1,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
if do_norm:
l = BatchNormLayer(l, axes=axes)
l = NonlinearityLayer(l, nonlin)
l = ConcatLayer((skip4, l), cropping=[None, None, 'center', 'center'])
l = conv_norm_lrelu(l, base_n_filter * 16)
l = Conv3DLayer(l,
base_n_filter * 8,
1,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_upscale_conv_norm_lrelu(l, base_n_filter * 4)
l = ConcatLayer((skip3, l), cropping=[None, None, 'center', 'center'])
l = ds2 = conv_norm_lrelu(l, base_n_filter * 8)
l = Conv3DLayer(l,
base_n_filter * 4,
1,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_upscale_conv_norm_lrelu(l, base_n_filter * 2)
l = ConcatLayer((skip2, l), cropping=[None, None, 'center', 'center'])
l = ds3 = conv_norm_lrelu(l, base_n_filter * 4)
l = Conv3DLayer(l,
base_n_filter * 2,
1,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
l = norm_lrelu_upscale_conv_norm_lrelu(l, base_n_filter)
l = ConcatLayer((skip1, l), cropping=[None, None, 'center', 'center'])
l = conv_norm_lrelu(l, base_n_filter * 2)
l_pred = Conv3DLayer(l,
num_output_classes,
1,
pad='same',
nonlinearity=None)
ds2_1x1_conv = Conv3DLayer(ds2,
num_output_classes,
1,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
ds1_ds2_sum_upscale = Upscale3DLayer(ds2_1x1_conv, 2)
ds3_1x1_conv = Conv3DLayer(ds3,
num_output_classes,
1,
1,
'same',
nonlinearity=linear,
W=HeNormal(gain='relu'))
ds1_ds2_sum_upscale_ds3_sum = ElemwiseSumLayer(
(ds1_ds2_sum_upscale, ds3_1x1_conv))
ds1_ds2_sum_upscale_ds3_sum_upscale = Upscale3DLayer(
ds1_ds2_sum_upscale_ds3_sum, 2)
l = seg_layer = ElemwiseSumLayer(
(l_pred, ds1_ds2_sum_upscale_ds3_sum_upscale))
l = DimshuffleLayer(l, (0, 2, 3, 4, 1))
batch_size, n_rows, n_cols, n_z, _ = lasagne.layers.get_output(l).shape
l = ReshapeLayer(l,
(batch_size * n_rows * n_cols * n_z, num_output_classes))
l = NonlinearityLayer(l, nonlinearity=lasagne.nonlinearities.softmax)
return l, seg_layer
def build_model(input_var, nOutput):
net = {}
net['input'] = InputLayer((None, 3, 32, 32), input_var=input_var)
net['conv1'] = ConvLayer(net['input'],
num_filters=192,
filter_size=5,
pad=2,
flip_filters=False,
W=lasagne.init.GlorotUniform())
net['cccp1'] = ConvLayer(net['conv1'],
num_filters=160,
filter_size=1,
flip_filters=False)
net['cccp2'] = ConvLayer(net['cccp1'],
num_filters=96,
filter_size=1,
flip_filters=False)
net['pool1'] = PoolLayer(net['cccp2'],
pool_size=3,
stride=2,
mode='max',
ignore_border=False)
net['drop3'] = DropoutLayer(net['pool1'], p=0.5)
net['conv2'] = ConvLayer(net['drop3'],
num_filters=192,
filter_size=5,
pad=2,
flip_filters=False)
net['cccp3'] = ConvLayer(net['conv2'],
num_filters=192,
filter_size=1,
flip_filters=False)
net['cccp4'] = ConvLayer(net['cccp3'],
num_filters=192,
filter_size=1,
flip_filters=False)
net['pool2'] = PoolLayer(net['cccp4'],
pool_size=3,
stride=2,
mode='average_exc_pad',
ignore_border=False)
net['drop6'] = DropoutLayer(net['pool2'], p=0.5)
net['conv3'] = ConvLayer(net['drop6'],
num_filters=192,
filter_size=3,
pad=1,
flip_filters=False)
net['cccp5'] = ConvLayer(net['conv3'],
num_filters=192,
filter_size=1,
flip_filters=False)
net['cccp6'] = ConvLayer(net['cccp5'],
num_filters=nOutput,
filter_size=1,
flip_filters=False)
net['pool3'] = PoolLayer(net['cccp6'],
pool_size=8,
mode='average_exc_pad',
ignore_border=False)
net['output'] = FlattenLayer(net['pool3'])
return net
def build_network(self, input_var, target_var):
from lasagne.layers import InputLayer
from lasagne.layers import DenseLayer
from lasagne.layers import NonlinearityLayer
from lasagne.layers import DropoutLayer
from lasagne.layers import ReshapeLayer
from lasagne.layers import Pool2DLayer as PoolLayer
from lasagne.layers import TransposedConv2DLayer as Deconv2DLayer
from lasagne.nonlinearities import softmax, sigmoid, tanh
import cPickle as pickle
try:
from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer
except ImportError as e:
from lasagne.layers import Conv2DLayer as ConvLayer
print_warning("Cannot import 'lasagne.layers.dnn.Conv2DDNNLayer' as it requires GPU support and a functional cuDNN installation. Falling back on slower convolution function 'lasagne.layers.Conv2DLayer'.")
batch_size = settings.BATCH_SIZE
net = {}
net['input'] = InputLayer((batch_size, 3, 64, 64), input_var=input_var)
net['conv1'] = ConvLayer(net['input'], 64, 3, stride=1, pad='same') # 64x64
net['pool1'] = PoolLayer(net['conv1'], 2) # 32x32
net['conv2'] = ConvLayer(net['pool1'], 64, 3, stride=1, pad='same') # 32x32
net['dropout1'] = DropoutLayer(net['conv2'], p=0.5)
net['conv3'] = ConvLayer(net['dropout1'], 64, 3, stride=1, pad='same') # 32x32
net['dropout3'] = DropoutLayer(net['conv3'], p=0.5)
net['fc1'] = DenseLayer(net['dropout3'], 3*32*32)
net['output'] = ReshapeLayer(net['fc1'], (batch_size, 3, 32, 32))
# net['input'] = InputLayer((batch_size, 3, 64, 64), input_var=input_var)
# net['dropout1'] = DropoutLayer(net['input'], p=0.1)
# net['conv1'] = ConvLayer(net['dropout1'], 256, 5, stride=2, pad='same') # 32x32
# net['dropout2'] = DropoutLayer(net['conv1'], p=0.5)
# net['conv2'] = ConvLayer(net['dropout2'], 256, 7, stride=1, pad='same') # 32x32
# net['dropout3'] = DropoutLayer(net['conv2'], p=0.5)
# net['deconv1'] = Deconv2DLayer(net['dropout3'], 256, 7, stride=1, crop='same', output_size=32) # 32x32
# net['dropout4'] = DropoutLayer(net['deconv1'], p=0.5)
# net['deconv3'] = Deconv2DLayer(net['dropout4'], 256, 9, stride=1, crop='same', output_size=32) # 32x32
# net['dropout5'] = DropoutLayer(net['deconv3'], p=0.5)
# net['fc1'] = DenseLayer(net['dropout5'], 2048)
# net['dropout6'] = DropoutLayer(net['fc1'], p=0.5)
# net['fc2'] = DenseLayer(net['dropout6'], 2048)
# net['dropout7'] = DropoutLayer(net['fc2'], p=0.5)
# net['fc3'] = DenseLayer(net['dropout7'], 3*32*32)
# net['dropout8'] = DropoutLayer(net['fc3'], p=0.5)
# net['reshape'] = ReshapeLayer(net['dropout8'], ([0], 3, 32, 32))
# net['output'] = Deconv2DLayer(net['reshape'], 3, 9, stride=1, crop='same', output_size=32, nonlinearity=sigmoid)
self.network, self.network_out = net, net['output']
print ("Conv_Deconv network output shape: {}".format(self.network_out.output_shape))
# self.input_pad, self.input_pad_out = self.build_pad_model(self.network_out)
# self.target_pad, self.target_pad_out = self.build_pad_model(InputLayer((batch_size, 3, 32, 32), input_var=target_var))
self.input_scaled, self.input_scaled_out = self.build_scaled_model(self.network_out)
self.target_scaled, self.target_scaled_out = self.build_scaled_model(InputLayer((batch_size, 3, 32, 32), input_var=target_var))
print("(Input) scaled network output shape: {}".format(self.input_scaled_out.output_shape))
print("(Target) scaled network output shape: {}".format(self.target_scaled_out.output_shape))
self.vgg_scaled_var = T.tensor4('scaled_vars')
self.vgg_model, self.vgg_model_out = self.build_vgg_model(self.vgg_scaled_var)
print("VGG model conv1_1 output shape: {}".format(self.vgg_model['conv1_1'].output_shape))
print("VGG model conv2_1 output shape: {}".format(self.vgg_model['conv2_1'].output_shape))
print("VGG model conv3_1 output shape: {}".format(self.vgg_model['conv3_1'].output_shape))
def build_vgg_model(self, input_var):
from lasagne.layers import InputLayer
from lasagne.layers import DenseLayer
from lasagne.layers import NonlinearityLayer
from lasagne.layers import DropoutLayer
from lasagne.layers import Pool2DLayer as PoolLayer
from lasagne.layers import TransposedConv2DLayer as Deconv2DLayer
from lasagne.nonlinearities import softmax, sigmoid, tanh
import cPickle as pickle
try:
from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer
except ImportError as e:
from lasagne.layers import Conv2DLayer as ConvLayer
print_warning("Cannot import 'lasagne.layers.dnn.Conv2DDNNLayer' as it requires GPU support and a functional cuDNN installation. Falling back on slower convolution function 'lasagne.layers.Conv2DLayer'.")
print_info("Building VGG-16 model...")
net = {}
net['input'] = InputLayer(shape = (None, 3, 224, 224), input_var = input_var, name = 'vgg_input')
net['conv1_1'] = ConvLayer(
net['input'], 64, 3, pad=1, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad=1, flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
net['conv2_1'] = ConvLayer(
net['pool1'], 128, 3, pad=1, flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad=1, flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
net['conv3_1'] = ConvLayer(
net['pool2'], 256, 3, pad=1, flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad=1, flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad=1, flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
net['conv4_1'] = ConvLayer(
net['pool3'], 512, 3, pad=1, flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad=1, flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad=1, flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
net['conv5_1'] = ConvLayer(
net['pool4'], 512, 3, pad=1, flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad=1, flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad=1, flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = DenseLayer(net['fc6_dropout'], num_units=4096)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['fc8'] = DenseLayer(
net['fc7_dropout'], num_units=1000, nonlinearity=None)
net['prob'] = NonlinearityLayer(net['fc8'], softmax)
net_output = net['prob']
print_info("Loading VGG16 pre-trained weights from file 'vgg16.pkl'...")
with open('vgg16.pkl', 'rb') as f:
params = pickle.load(f)
#net_output.initialize_layers()
lasagne.layers.set_all_param_values(net['prob'], params['param values'])
print_info("Alright, pre-trained VGG16 model is ready!")
return net, net['prob']
def __create_toplogy__(self, input_var_first=None, input_var_second=None):
# define network topology
if (self.conf.rep % 2 != 0):
raise ValueError(
"Representation size should be divisible by two as it's formed by combining two crossmodal translations",
self.conf.rep)
# input layers
l_in_first = InputLayer(shape=(self.conf.batch_size,
self.conf.mod1size),
input_var=input_var_first)
l_in_second = InputLayer(shape=(self.conf.batch_size,
self.conf.mod2size),
input_var=input_var_second)
# first -> second
l_hidden1_first = DenseLayer(l_in_first,
num_units=self.conf.hdn,
nonlinearity=self.conf.act,
W=GlorotUniform()) # enc1
l_hidden2_first = DenseLayer(l_hidden1_first,
num_units=self.conf.rep // 2,
nonlinearity=self.conf.act,
W=GlorotUniform()) # enc2
l_hidden2_first_d = DropoutLayer(l_hidden2_first, p=self.conf.dropout)
l_hidden3_first = DenseLayer(l_hidden2_first_d,
num_units=self.conf.hdn,
nonlinearity=self.conf.act,
W=GlorotUniform()) # dec1
l_out_first = DenseLayer(l_hidden3_first,
num_units=self.conf.mod2size,
nonlinearity=self.conf.act,
W=GlorotUniform()) # dec2
if self.conf.untied:
# FREE
l_hidden1_second = DenseLayer(l_in_second,
num_units=self.conf.hdn,
nonlinearity=self.conf.act,
W=GlorotUniform()) # enc1
l_hidden2_second = DenseLayer(l_hidden1_second,
num_units=self.conf.rep // 2,
nonlinearity=self.conf.act,
W=GlorotUniform()) # enc2
l_hidden2_second_d = DropoutLayer(l_hidden2_second,
p=self.conf.dropout)
l_hidden3_second = DenseLayer(l_hidden2_second_d,
num_units=self.conf.hdn,
nonlinearity=self.conf.act,
W=GlorotUniform()) # dec1
l_out_second = DenseLayer(l_hidden3_second,
num_units=self.conf.mod1size,
nonlinearity=self.conf.act,
W=GlorotUniform()) # dec2
else:
# TIED middle
l_hidden1_second = DenseLayer(l_in_second,
num_units=self.conf.hdn,
nonlinearity=self.conf.act,
W=GlorotUniform()) # enc1
l_hidden2_second = DenseLayer(l_hidden1_second,
num_units=self.conf.rep // 2,
nonlinearity=self.conf.act,
W=l_hidden3_first.W.T) # enc2
l_hidden2_second_d = DropoutLayer(l_hidden2_second,
p=self.conf.dropout)
l_hidden3_second = DenseLayer(l_hidden2_second_d,
num_units=self.conf.hdn,
nonlinearity=self.conf.act,
W=l_hidden2_first.W.T) # dec1
l_out_second = DenseLayer(l_hidden3_second,
num_units=self.conf.mod1size,
nonlinearity=self.conf.act,
W=GlorotUniform()) # dec2
l_out = concat([l_out_first, l_out_second])
return l_out, l_hidden2_first, l_hidden2_second
def LSTMCell(prev_cell,
prev_out,
input_or_inputs=tuple(),
num_units=None,
peepholes=True,
weight_init=init.Normal(),
bias_init=init.Constant(),
peepholes_W_init=init.Normal(),
forgetgate_nonlinearity=lasagne.nonlinearities.sigmoid,
inputgate_nonlinearity=lasagne.nonlinearities.sigmoid,
outputgate_nonlinearity=lasagne.nonlinearities.sigmoid,
cell_nonlinearity=lasagne.nonlinearities.tanh,
output_nonlinearity=lasagne.nonlinearities.tanh,
dropout=0.,
name=None,
grad_clipping=0.,
):
"""
Implements a one-step LSTM update. Note that LSTM requires both c_t (private memory) and h_t aka output.
:param prev_cell: input that denotes previous "private" state (shape must be (None, n_units) )
:type prev_cell: lasagne.layers.Layer
:param prev_out: input that denotes previous "public" state (shape must be (None,n_units))
:type prev_out: lasagne.layers.Layer
:param input_or_inputs: a single layer or a list/tuple of layers that go as inputs
:type input_or_inputs: lasagne.layers.Layer or list of such
:param num_units: how many recurrent cells to use. None means "as in prev_state"
:type num_units: int
:param peepholes: If True, the LSTM uses peephole connections.
When False, peepholes_W_init are ignored.
:type peepholes: bool
:param bias_init: either a lasagne initializer to use for every gate weights
or a list of 4 initializers for [input gate, forget gate, cell, output gate]
:param weight_init: either a lasagne initializer to use for every gate weights:
or a list of two initializers,
- first used for all weights from hidden -> <all>_gate and cell
- second used for all weights from input(s) -> <all>_gate weights and cell
or a list of two objects elements,
- second list is hidden -> input gate, forget gate, cell, output gate,
- second list of lists where list[i][0,1,2] = input[i] -> [input_gate, forget gate, cell,output gate ]
:param peepholes_W_init: either a lasagne initializer or a list of 3 initializers for
[input_gate, forget gate,output gate ] weights. If peepholes=False, this is ignored.
:param <any>_nonlinearity: which nonlinearity to use for a particular gate
:param dropout: dropout rate as per https://arxiv.org/pdf/1603.05118.pdf
:param grad_clipping: maximum gradient absolute value. 0 or None means "no clipping"
:returns: a tuple of (new_cell,new_output) layers
:rtype: (lasagne.layers.Layer,lasagne.layers.Layer)
for developers:
Works by stacking other lasagne layers;
is a function mock, not actual class.
"""
assert len(prev_cell.output_shape) == 2
# if required, infer num_units
if num_units is None:
num_units = prev_cell.output_shape[1]
# else check it
assert num_units == prev_cell.output_shape[1]
# gates and cell (before nonlinearities)
gates = GateLayer([prev_out] + check_list(input_or_inputs),
[num_units] * 4,
channel_names=["to_ingate", "to_forgetgate", "to_cell", "to_outgate"],
gate_nonlinearities=None,
bias_init=bias_init,
weight_init=weight_init,
name=name or "")
ingate, forgetgate, cell_input, outputgate = gates.values()
# clip grads #1
if grad_clipping:
ingate, forgetgate, cell_input, outputgate = [clip_grads(lyr, grad_clipping) for lyr in
[ingate, forgetgate, cell_input, outputgate]]
if peepholes:
# cast bias init to a list
peepholes_W_init = check_list(peepholes_W_init)
assert len(peepholes_W_init) in (1, 3)
if len(peepholes_W_init) == 1:
peepholes_W_init *= 3
W_cell_to_ingate_init,W_cell_to_forgetgate_init= peepholes_W_init[:2]
peep_ingate = lasagne.layers.ScaleLayer(prev_cell,W_cell_to_ingate_init,shared_axes=[0,],
name= (name or "") + ".W_cell_to_ingate_peephole")
peep_forgetgate = lasagne.layers.ScaleLayer(prev_cell,W_cell_to_forgetgate_init,shared_axes=[0,],
name= (name or "") + ".W_cell_to_forgetgate_peephole")
ingate = add(ingate,peep_ingate)
forgetgate = add(forgetgate,peep_forgetgate)
# nonlinearities
ingate = NonlinearityLayer(
ingate,
inputgate_nonlinearity,
name=(name or "")+".inputgate"
)
forgetgate = NonlinearityLayer(
forgetgate,
forgetgate_nonlinearity,
name=(name or "")+".forgetgate"
)
cell_input = NonlinearityLayer(cell_input,
nonlinearity=cell_nonlinearity,
name=(name or "")+'.cell_nonlinearity')
if dropout != 0:
cell_input = DropoutLayer(cell_input,p=dropout)
# cell = input * ingate + prev_cell * forgetgate
new_cell= add(mul(cell_input,ingate),
mul(prev_cell, forgetgate))
# output gate
if peepholes:
W_cell_to_outgate_init = peepholes_W_init[2]
peep_outgate = lasagne.layers.ScaleLayer(new_cell,W_cell_to_outgate_init,shared_axes=[0,],
name= (name or "") + ".W_cell_to_outgate_peephole")
outputgate = add(outputgate, peep_outgate)
outputgate = NonlinearityLayer(
outputgate,
outputgate_nonlinearity,
name=(name or "")+".outgate"
)
#cell output
new_output= NonlinearityLayer(new_cell,
output_nonlinearity,
name=(name or "")+'.outgate_nonlinearity')
new_output = mul(
outputgate,
new_output,
name=(name or "")+'.outgate'
)
return new_cell, new_output
def get_model(input_var, target_var, multiply_var):
# input layer with unspecified batch size
layer_input = InputLayer(
shape=(None, 12, 64, 64), input_var=input_var
) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var)
layer_0 = DimshuffleLayer(layer_input, (0, 'x', 1, 2, 3))
# Z-score?
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer_1 = batch_norm(
Conv3DDNNLayer(incoming=layer_0,
num_filters=16,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_2 = batch_norm(
Conv3DDNNLayer(incoming=layer_1,
num_filters=16,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_3 = MaxPool3DDNNLayer(layer_2,
pool_size=(2, 2, 2),
stride=(2, 2, 2),
pad=(1, 1, 1))
layer_4 = DropoutLayer(layer_3, p=0.25)
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer_5 = batch_norm(
Conv3DDNNLayer(incoming=layer_4,
num_filters=32,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_6 = batch_norm(
Conv3DDNNLayer(incoming=layer_5,
num_filters=32,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_7 = MaxPool3DDNNLayer(layer_6,
pool_size=(2, 2, 2),
stride=(2, 2, 2),
pad=(1, 1, 1))
layer_8 = DropoutLayer(layer_7, p=0.25)
# Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer
layer_5 = batch_norm(
Conv3DDNNLayer(incoming=layer_8,
num_filters=64,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_6 = batch_norm(
Conv3DDNNLayer(incoming=layer_5,
num_filters=64,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_7 = batch_norm(
Conv3DDNNLayer(incoming=layer_6,
num_filters=64,
filter_size=(3, 3, 3),
stride=(1, 1, 1),
pad='same',
nonlinearity=leaky_rectify))
layer_8 = MaxPool3DDNNLayer(layer_7,
pool_size=(2, 2, 2),
stride=(2, 2, 2),
pad=(1, 1, 1))
layer_9 = DropoutLayer(layer_8, p=0.25)
layer_flatten = FlattenLayer(layer_9)
# Output Layer
layer_hidden = DenseLayer(layer_flatten, 500, nonlinearity=linear)
layer_prediction = DenseLayer(layer_hidden, 2, nonlinearity=linear)
# Loss
prediction = get_output(layer_prediction) #/ multiply_var
loss = squared_error(prediction, target_var)
loss = loss.mean()
#Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum
params = get_all_params(layer_prediction, trainable=True)
# Create a loss expression for validation/testing. The crucial difference
# here is that we do a deterministic forward pass through the network, disabling dropout layers.
test_prediction = get_output(layer_prediction,
deterministic=True) # / multiply_var
test_loss = squared_error(test_prediction, target_var)
test_loss = test_loss.mean()
# crps estimate
crps = T.abs_(test_prediction - target_var).mean() / 600
return test_prediction, crps, loss, params
def GRUCell(prev_state,
input_or_inputs=tuple(),
num_units=None,
weight_init=init.Normal(),
bias_init=init.Constant(),
forgetgate_nonlinearity=lasagne.nonlinearities.sigmoid,
updategate_nonlinearity=lasagne.nonlinearities.sigmoid,
hidden_update_nonlinearity=lasagne.nonlinearities.tanh,
dropout=0,
name="YetAnotherGRULayer",
grad_clipping=0,
):
"""
Implements a one-step gated recurrent unit (GRU) with arbitrary number of units.
:param prev_state: input that denotes previous state (shape must be (None, n_units) )
:type prev_state: lasagne.layers.Layer
:param input_or_inputs: a single layer or a list/tuple of layers that go as inputs
:type input_or_inputs: lasagne.layers.Layer or list of such
:param num_units: how many recurrent cells to use. None means "as in prev_state"
:type num_units: int
:param weight_init: either a lasagne initializer to use for every gate weights
or a list of two initializers:
- first used for all weights from hidden -> <any>_gate and hidden update
- second used for all weights from input(s) -> <any>_gate weights and hidden update
or a list of two objects elements:
- second list is hidden -> forget gate, update gate, hidden update
- second list of lists where
list[i][0,1,2] = input[i] -> [forget gate, update gate, hidden update]
:param <any>_nonlinearity: which nonlinearity to use for a particular gate
:param dropout: dropout rate as per https://arxiv.org/abs/1603.05118
:param grad_clipping: maximum gradient absolute value. 0 or None means "no clipping"
:returns: updated memory layer
:rtype: lasagne.layers.Layer
for developers:
Works by stacking other lasagne layers;
is a function mock, not actual class.
"""
assert len(prev_state.output_shape) == 2
# if required, infer num_units
if num_units is None:
num_units = prev_state.output_shape[1]
# else check it
assert num_units == prev_state.output_shape[1]
inputs = check_list(input_or_inputs)
#handle weight init
weight_init = check_list(weight_init)
if len(weight_init) == 1:
weight_init *= 2
hidden_W_init, input_W_init = weight_init
# hidden to gates
hid_to_gates = GateLayer(prev_state, [num_units] * 3,
gate_nonlinearities=None,
channel_names=["to_resetgate","to_updategate", "to_hidden_update"],
bias_init=None,
weight_init=hidden_W_init,
name=name or "")
hid_forget, hid_update, hidden_update_hid = hid_to_gates.values()
# clip grads #1
if grad_clipping:
inputs = [clip_grads(lyr, grad_clipping) for lyr in inputs]
hid_forget, hid_update, hidden_update_hid = [clip_grads(lyr, grad_clipping) for lyr in
[hid_forget, hid_update, hidden_update_hid]]
# input to gates
inp_to_gates = GateLayer(inputs, [num_units] * 3,
gate_nonlinearities=None,
channel_names=["to_resetgate", "to_updategate", "to_hidden_update"],
bias_init = bias_init,
weight_init = input_W_init,
name=name or "")
inp_forget, inp_update, hidden_update_in = inp_to_gates.values()
# compute forget and update gates
forgetgate = NonlinearityLayer(
add(inp_forget, hid_forget),
forgetgate_nonlinearity,
name=(name or "")+".forgetgate"
)
updategate = NonlinearityLayer(
add(inp_update, hid_update),
updategate_nonlinearity,
name=(name or "")+".updategate"
)
inv_updategate = NonlinearityLayer(updategate,
lambda x: 1 - x,
name=(name or "")+".[1 - updategate]")
# compute hidden update
hidden_update = add(
hidden_update_in,
mul(forgetgate, hidden_update_hid),
name=(name or "")+".hid_update"
)
# clip grads #2
if grad_clipping:
hidden_update = clip_grads(hidden_update,
grad_clipping)
hidden_update = NonlinearityLayer(hidden_update,
hidden_update_nonlinearity)
if dropout != 0:
hidden_update = DropoutLayer(hidden_update,p=dropout)
# compute new hidden values
new_hid = add(
mul(inv_updategate, prev_state),
mul(updategate, hidden_update),
name=name
)
return new_hid
def build_densenet(input_shape=(None, 3, 32, 32),
input_var=None,
classes=10,
depth=40,
first_output=16,
growth_rate=12,
num_blocks=3,
dropout=0):
"""
Creates a DenseNet model in Lasagne.
Parameters
----------
input_shape : tuple
The shape of the input layer, as ``(batchsize, channels, rows, cols)``.
Any entry except ``channels`` can be ``None`` to indicate free size.
input_var : Theano expression or None
Symbolic input variable. Will be created automatically if not given.
classes : int
The number of classes of the softmax output.
depth : int
Depth of the network. Must be ``num_blocks * n + 1`` for some ``n``.
(Parameterizing by depth rather than n makes it easier to follow the
paper.)
first_output : int
Number of channels of initial convolution before entering the first
dense block, should be of comparable size to `growth_rate`.
growth_rate : int
Number of feature maps added per layer.
num_blocks : int
Number of dense blocks (defaults to 3, as in the original paper).
dropout : float
The dropout rate. Set to zero (the default) to disable dropout.
batchsize : int or None
The batch size to build the model for, or ``None`` (the default) to
allow any batch size.
inputsize : int, tuple of int or None
Returns
-------
network : Layer instance
Lasagne Layer instance for the output layer.
References
----------
.. [1] Gao Huang et al. (2016):
Densely Connected Convolutional Networks.
https://arxiv.org/abs/1608.06993
"""
if (depth - 1) % num_blocks != 0:
raise ValueError("depth must be num_blocks * n + 1 for some n")
# input and initial convolution
network = InputLayer(input_shape, input_var, name='input')
network = Conv2DLayer(network,
first_output,
3,
pad='same',
W=lasagne.init.HeNormal(gain='relu'),
b=None,
nonlinearity=None,
name='pre_conv')
if dropout:
network = DropoutLayer(network, dropout)
# dense blocks with transitions in between
n = (depth - 1) // num_blocks
for b in range(num_blocks):
network = dense_block(network,
n - 1,
growth_rate,
dropout,
name_prefix='block%d' % (b + 1))
if b < num_blocks - 1:
network = transition(network,
dropout,
name_prefix='block%d_trs' % (b + 1))
# post processing until prediction
network = BatchNormLayer(network, name='post_bn')
network = NonlinearityLayer(network,
nonlinearity=rectify,
name='post_relu')
network = GlobalPoolLayer(network, name='post_pool')
network = DenseLayer(network,
classes,
nonlinearity=softmax,
W=lasagne.init.HeNormal(gain=1),
name='output')
return network
def buildNetwork(CFG, params, vocab):
# {{{
"""
TODO document me
"""
# Use params to update CFG
CFG = get_CFG(CFG, params)
#-----------------------------------------------------------
# Setting up the image Embedding.
#-----------------------------------------------------------
l_input_sentence = InputLayer((CFG['BATCH_SIZE'], 1), name='l_input_sentence') # input (1 word)
l_sentence_embedding = lasagne.layers.EmbeddingLayer(l_input_sentence,
input_size=len(vocab),
output_size=CFG['EMBEDDING_SIZE'],
name='l_sentence_embedding')
# Setting up CNN in case of fine tuning.
if CFG['CNN_FINE_TUNE']:
cnn, l_input_cnn, l_input_img = build_CNN(CFG)
if CFG['CNN_MODEL'] == "vgg":
vgg16, resnet50 = cnn, None
elif CFG["CNN_MODEL"] == "resnet":
vgg16, resnet50 = None, cnn
if CFG['START_NORMALIZED'] == 1:
l_input_cnn = ExpressionLayer(l_input_cnn, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['START_NORMALIZED'] == 2:
l_input_cnn = ExpressionLayer(l_input_cnn, lambda X: X / T.sqrt(
T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_cnn = InputLayer((CFG['BATCH_SIZE'], CFG['CNN_FEATURE_SIZE']), name='l_input_cnn')
l_cnn_embedding = DenseLayer(l_input_cnn, num_units=CFG['EMBEDDING_SIZE'],
nonlinearity=lasagne.nonlinearities.identity, name='l_cnn_embedding')
l_cnn_embedding2 = ReshapeLayer(l_cnn_embedding, ([0], 1, [1]), name='l_cnn_embedding2')
l_rnn_input = InputLayer((CFG['BATCH_SIZE'], 1, CFG['EMBEDDING_SIZE']), name='l_rnn_input')
l_dropout_input = DropoutLayer(
l_rnn_input, p=0.5, name='l_dropout_input')
l_input_reg = None
l_out_reg = None
l_decoder = None
l_region_feedback = None
l_region = None
l_input_img2 = None
l_boxes = None
l_conv = None
l_loc = None
l_loc1 = None
l_input_loc = None
l_sel_region2 = None
l_weighted_region_prev = None
l_weighted_region = None
input_shape = (CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE'])
if CFG['MODE'] == 'normal':
# {{{1
l_cell_input = InputLayer(input_shape, name='l_cell_input')
l_prev_gru = InputLayer(input_shape, name="l_prev_gru")
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_input, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(
l_gru, p=0.5, name='l_dropout_output')
# decoder is a fully connected layer with one output unit for each word in the vocabulary
l_decoder = DenseLayer(l_dropout_output, num_units=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_decoder')
l_out = ReshapeLayer(
l_decoder, ([0], 1, [1]), name='l_out')
# }}}
elif CFG['MODE'] == 'tensor':
l_cell_input = InputLayer(input_shape, name='l_cell_input')
l_prev_gru = InputLayer(input_shape, name="l_prev_gru")
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'], l_cell_input, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(l_gru, p=0.5, name='l_dropout_output')
l_dropout_output = ReshapeLayer(l_dropout_output, ([0], 1, [1]), name='l_dropout_output')
# TODO put me back
if CFG['CNN_FINE_TUNE']:
l_input_regions, _input_regions, l_out_reg, l_input_img2, l_boxes, l_conv = build_finetune_proposals(CFG, vgg16, resnet50)
else:
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
# TODO a block.
#l_decoder = build_decoderLayer(l_dropout_output, l_input_regions, vocab, CFG)
if CFG.has_key('DISSECT') and CFG['DISSECT'] != 'No':
if CFG['DISSECT'] == 'wr':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_hr='skip', b_hr='skip')
elif CFG['DISSECT'] == 'rs':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_rw='skip', b_rw='skip')
if CFG['DISSECT'] == 'wr':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_hr='skip', b_hr='skip')
elif CFG['DISSECT'] == 'rs':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_rw='skip', b_rw='skip')
else:
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum(2), output_shape='auto', name='l_out') # sum over regions
elif CFG['MODE'] == 'transformer':
#{{{2
print(bcolors.OKGREEN + "Transformer mode." + bcolors.ENDC)
from TProd3 import TensorProdFactLayer, WeightedSumLayer, SubsampleLayer
# define a cell
l_cell_input = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_cell_input')
from agentnet.memory import GRUMemoryLayer
l_prev_gru = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name="l_prev_gru")
if CFG['TRANS_FEEDBACK']:
l_weighted_region_prev = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE']), name="l_weighted_region_prev")
if CFG['FEEDBACK'] == 2:
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_weighted_region_prev], axis=1, name='l_cell_concat')
else:
print("Are you sure you don't want to use feedback=2? I think you should. Change your mind, then come to see me again.")
else:
l_cell_concat = l_cell_input
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_concat, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(l_gru, p=CFG['RNN_DROPOUT'], name='l_dropout_output')
l_dropout_output = ReshapeLayer(l_dropout_output, ([0], 1, [1]), name='l_dropout_output')
if CFG['TRANS_USE_PRETRAINED']:
l_out_reg = vgg16['conv5_2']
#l_out_reg2 = vgg16['conv5_3']
else:
l_out_reg = vgg16['conv5_3']
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], 14, 14), name='l_input_reg')
l_input_regions = l_input_reg
if CFG['TRANS_USE_PRETRAINED']:
l_input_regions = l_input_regions
else:
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_regions = ExpressionLayer(l_input_regions, lambda X: X * 0.01, output_shape='auto')
factor = 2.0
W = lasagne.init.Constant(0.0)
b = lasagne.init.Constant(0.0)
if CFG['TRANS_MULTIPLE_BOXES']:
num_prop, l_loc = build_loc_net(CFG, l_gru, l_input_regions, 1, (
14, 14), (3, 3), CFG['TRANS_STRIDE'], CFG['TRANS_ZOOM'], W, b, name='')
if CFG['TRANS_ADD_BIG_PROPOSALS']:
num_prop_big, l_loc_big = build_loc_net(CFG, l_gru, l_input_regions, 1, (
14, 14), (3, 3), CFG['TRANS_STRIDE'], CFG['TRANS_ZOOM'] * 2, W, b, name='_big')
l_loc = ConcatLayer((l_loc, l_loc_big), axis=0)
num_prop += num_prop_big
l_sel_region2 = MultiTransformerLayer(l_input_regions, l_loc, kernel_size=(
3, 3), zero_padding=CFG['TRANS_ZEROPAD']) # 3x3
if CFG['TRANS_USE_PRETRAINED']:
Wvgg = vgg16['conv5_3'].W.reshape(
(CFG['REGION_SIZE'], CFG['REGION_SIZE'] * 3 * 3)).swapaxes(0, 1)
bvgg = vgg16['conv5_3'].b
l_sel_region = DenseLayer(
l_sel_region2, num_units=CFG['REGION_SIZE'], name='l_sel_region', W=Wvgg, b=bvgg)
if CFG['CONV_NORMALIZED'] == 1:
l_sel_region = ExpressionLayer(l_sel_region, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_sel_region = ExpressionLayer(l_sel_region, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_sel_region = l_sel_region
else:
l_sel_region = DenseLayer(
l_sel_region, num_units=CFG['REGION_SIZE'], name='l_sel_region')
l_sel_region = ReshapeLayer(
l_sel_region, (CFG['BATCH_SIZE'], num_prop, CFG['REGION_SIZE']))
l_sel_region = DimshuffleLayer(l_sel_region, (0, 2, 1))
l_sel_region = ReshapeLayer(
l_sel_region, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], num_prop))
else:
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 2
b[1, 1] = 2
b = b.flatten()
W = lasagne.init.Constant(0.0)
l_input_loc = l_gru
if CFG['TRANS_USE_STATE']:
l_input_im = ConvLayer(l_input_regions, num_filters=512, filter_size=(
3, 3), pad='same', name='l_reduce_im1')
l_input_im = lasagne.layers.MaxPool2DLayer(l_input_im, (2, 2))
l_input_im = ConvLayer(l_input_im, num_filters=512, filter_size=(
3, 3), pad='same', name='l_reduce_im2')
l_input_im = lasagne.layers.MaxPool2DLayer(l_input_im, (2, 2))
l_input_im = ReshapeLayer(l_input_im, (CFG['BATCH_SIZE'], 512))
l_input_loc = ConcatLayer((l_gru, l_input_im))
l_loc1 = DenseLayer(
l_input_loc, num_units=256, name='l_loc1')
l_loc = DenseLayer(
l_loc1, num_units=6, W=W, b=b, nonlinearity=None, name='l_loc2')
l_sel_region = TransformerLayer(
l_input_regions, l_loc, downsample_factor=2)
l_sel_region = DenseLayer(
l_sel_region, num_units=CFG['REGION_SIZE'], name='l_sel_region')
l_sel_region = ReshapeLayer(
l_sel_region, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], 1))
l_decoder = TensorProdFactLayer((l_dropout_output, l_sel_region), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), W=lasagne.init.Normal(std=0.001, mean=0.0), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor')
if CFG['TRANS_FEEDBACK']:
l_region = ExpressionLayer(l_decoder, lambda X: X.sum(3), output_shape='auto', name='l_region') # sum over regions
l_weighted_region = WeightedSumLayer([l_sel_region, l_region], name='l_weighted_region')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
#}}}
elif CFG['MODE'] == 'tensor-feedback':
# {{{2
# define a cell
l_cell_input = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_cell_input')
l_region_feedback = InputLayer((CFG['BATCH_SIZE'], CFG['NUM_REGIONS']), name='l_region_feedback')
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_region_feedback], axis=1, name='l_cell_concat')
from agentnet.memory import GRUMemoryLayer
l_prev_gru = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name="l_prev_gru")
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_concat, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(
l_gru, p=0.5, name='l_dropout_output')
l_dropout_output = ReshapeLayer(
l_dropout_output, ([0], 1, [1]), name='l_dropout_output')
from TProd3 import TensorProdFactLayer
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
l_tensor = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor')
l_region = ExpressionLayer(l_tensor, lambda X: X.sum(
3), output_shape='auto', name='l_region') # sum over
l_region = ReshapeLayer(
l_region, ([0], [2]), name='l_region')
l_out = ExpressionLayer(l_tensor, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
#}}}
elif CFG['MODE'] == 'tensor-feedback2':
# {{{2
l_feedback = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_feedback')
l_prev_gru = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name="l_prev_gru")
from TProd3 import TensorProdFactLayer, WeightedSumLayer
if CFG['PROPOSALS'] == 3: # use images at different resolution but without fully connected layers
import CNN
vgg16_det = CNN.build_model_RCNN(
CFG['NUM_REGIONS'], CFG['IM_SIZE'] * 1.5, pool_dims=3, dropout_value=CFG['RNN_DROPOUT'])
print "Loading pretrained VGG16 parameters for detection"
l_input_img2 = vgg16_det['input']
l_conv = vgg16_det['conv5_3']
l_boxes = vgg16_det['boxes']
l_input_regions = vgg16_det['reshape']
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X * 0.01, output_shape='auto')
l_cnn_embedding2 = DenseLayer(
l_input_regions, num_units=CFG['REGION_SIZE'], name='l_cnn_proposals')
l_input_regions = ReshapeLayer(
l_cnn_embedding2, (CFG['BATCH_SIZE'], CFG['NUM_REGIONS'], CFG['REGION_SIZE'], 1))
l_input_regions = lasagne.layers.DimshuffleLayer(
l_input_regions, (0, 2, 1, 3))
l_out_reg = l_input_regions
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS'], 1), name='l_input_reg')
l_input_regions = ReshapeLayer(
l_input_reg, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
# use images at different resolution but without fully connected layers
elif CFG['PROPOSALS'] == 4:
if CFG['CNN_MODEL'] == 'vgg':
import CNN
vgg16_det = CNN.build_model_RCNN(CFG['NUM_REGIONS'], int(
CFG['IM_SIZE'] * 1.5), pool_dims=1, dropout_value=CFG['RNN_DROPOUT'])
print "Loading pretrained VGG16 parameters for detection"
model_param_values = pickle.load(open('vgg16.pkl'))[
'param values']
lasagne.layers.set_all_param_values(
vgg16_det['conv5_3'], model_param_values[:-6])
l_input_img2 = vgg16_det['input']
l_conv = vgg16_det['conv5_3']
l_boxes = vgg16_det['boxes']
l_input_regions = vgg16_det['crop']
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'] * CFG['NUM_REGIONS'], CFG['REGION_SIZE']))
else:
resnet50_det = resnet_CNN.build_model_RCNN(
CFG['NUM_REGIONS'], im_size=CFG['IM_SIZE'] * 1.5, pool_dims=1, dropout_value=CFG['RNN_DROPOUT'])
print "Loading pretrained resnet50 parameters for detection"
# You can use this format to store other things for best effort
model_param_values = pickle.load(open('resnet50.pkl'))[
'param values']
from save_layers import add_names_layers_and_params
add_names_layers_and_params(resnet50_det)
#lasagne.layers.set_all_param_values(resnet50['prob'], model_param_values)
set_param_dict(
resnet50_det['pool5'], model_param_values, prefix='', show_layers=False, relax=False)
l_input_img2 = resnet50_det['input']
l_conv = resnet50_det['res4f_relu']
l_boxes = resnet50_det['boxes']
l_input_regions = resnet50_det['crop']
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'] * CFG['NUM_REGIONS'], CFG['REGION_SIZE']))
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
_input_regions = ExpressionLayer(
l_input_regions, lambda X: X * 0.01, output_shape='auto')
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'], CFG['NUM_REGIONS'], CFG['REGION_SIZE'], 1))
l_input_regions = lasagne.layers.DimshuffleLayer(
l_input_regions, (0, 2, 1, 3))
l_out_reg = l_input_regions
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS'], 1), name='l_input_reg')
l_input_regions = ReshapeLayer(
l_input_reg, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
else:
if CFG['CNN_MODEL'] == 'vgg':
l_out_reg = vgg16['conv5_3']
elif CFG['CNN_MODEL'] == 'resnet':
l_out_reg = resnet50['res4f_relu']
else:
print(bcolors.FAIL + "Unrecognized network" + bcolors.ENDC)
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], 14, 14), name='l_input_reg')
if CFG['CONV_REDUCED'] > 1:
# added a scaling factor of 100 to avoid exploding gradients
l_input_regions = ExpressionLayer(
l_input_reg, lambda X: X[:, :, ::CFG['CONV_REDUCED'], ::CFG['CONV_REDUCED']], output_shape='auto')
else:
l_input_regions = l_input_reg
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X * 0.01, output_shape='auto')
if CFG['TENSOR_ADD_CONV']:
l_input_regions = ConvLayer(l_input_regions, num_filters=CFG['REGION_SIZE'], filter_size=(
3, 3), pad='same', name='l_add_con')
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']))
if CFG['TENSOR_TIED']:
l_region_feedback = InputLayer((CFG['BATCH_SIZE'], CFG['NUM_REGIONS']), name='l_region_feedback')
l_region_feedback2 = ReshapeLayer(
l_region_feedback, ([0], 1, [1]), name='l_region_feedback2')
else:
l_shp2 = ReshapeLayer(
l_prev_gru, (CFG['BATCH_SIZE'], 1, CFG['EMBEDDING_SIZE']))
l_shp2 = DropoutLayer(
l_shp2, p=CFG['RNN_DROPOUT'], name='l_shp2')
l_tensor2 = TensorProdFactLayer((l_shp2, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor2')
l_region_feedback = ExpressionLayer(l_tensor2, lambda X: T.sum(
X, 3), output_shape='auto', name='l_region') # sum over
l_region_feedback2 = ReshapeLayer(
l_region_feedback, (CFG['BATCH_SIZE'], 1, CFG['NUM_REGIONS']))
l_weighted_region = WeightedSumLayer(
[l_input_regions, l_region_feedback2], name='l_weighted_region')
# define a cell
l_cell_input = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_cell_input')
if CFG['FEEDBACK'] == 0: # none
l_cell_concat = l_cell_input
elif CFG['FEEDBACK'] == 1: # none
l_region2 = ReshapeLayer(
l_region_feedback2, ([0], [2]))
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_region2], axis=1, name='l_cell_concat')
elif CFG['FEEDBACK'] == 2:
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_weighted_region], axis=1, name='l_cell_concat')
elif CFG['FEEDBACK'] == 3:
l_region2 = ReshapeLayer(
l_region_feedback2, ([0], [2]))
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_weighted_region, l_region2], axis=1, name='l_cell_concat')
elif CFG['FEEDBACK'] == 4:
# See RNNTraining.py for comments on this.
from TProd3 import WeightedImageLayer
l_weighted_image = WeightedImageLayer(
[l_input_regions, l_region_feedback2], name='l_weighted_image')
if CFG['IMGFEEDBACK_MECHANISM'] == 'highres':
l_weighted_image_reshaped = ReshapeLayer(
l_weighted_image, ([0], [1], 14, 14), name='l_weighted_image_reshaped')
l_weighted_image_conv_reduced = lasagne.layers.MaxPool2DLayer(
l_weighted_image_reshaped, (2, 2), name='l_weighted_image_conv_reduced')
l_feedback_co1 = lasagne.layers.Conv2DLayer(
incoming=l_weighted_image_conv_reduced, num_filters=512, filter_size=(3, 3), pad='same', name='l_feedback_co1')
else:
l_weighted_image_reshaped = ReshapeLayer(
l_weighted_image, ([0], [1], 7, 7), name='l_weighted_image_reshaped')
l_feedback_co1 = lasagne.layers.Conv2DLayer(
incoming=l_weighted_image_reshaped, num_filters=512, filter_size=(3, 3), pad='same', name='l_feedback_co1')
l_feedback_po1 = lasagne.layers.MaxPool2DLayer(
l_feedback_co1, (2, 2), name='l_feedback_po1')
l_feedback_co2 = lasagne.layers.Conv2DLayer(
incoming=l_feedback_po1, num_filters=512, filter_size=(3, 3), pad='same', name='l_feedback_co2')
l_feedback_po2 = lasagne.layers.MaxPool2DLayer(
l_feedback_co2, (2, 2), name='l_feedback_po2')
l_feedback_po2_reshaped = ReshapeLayer(
l_feedback_po2, ([0], [1]), name='l_feedback_po2_reshaped')
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_feedback_po2_reshaped], axis=1, name='l_cell_concat')
from agentnet.memory import GRUMemoryLayer
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_concat, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(
l_gru, p=0.5, name='l_dropout_output')
l_shp1 = ReshapeLayer(
l_dropout_output, ([0], 1, [1]), name='l_shp1')
if CFG.has_key('DISSECT') and CFG['DISSECT'] != 'No':
import pdb
pdb.set_trace() # XXX BREAKPOINT
if CFG['DISSECT'] == 'wr':
l_decoder = TensorProdFactLayer((l_shp1, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_hr='skip', b_hr='skip')
elif CFG['DISSECT'] == 'rs':
import pdb
pdb.set_trace() # XXX BREAKPOINT
l_decoder = TensorProdFactLayer((l_shp1, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_rw='skip', b_rw='skip')
else:
if CFG.has_key('DENSITY_TEMPERING') and CFG['DENSITY_TEMPERING']:
print("TEMPERING")
l_gamma = DenseLayer(
l_shp1, num_units=1, name='l_gamma')
l_gamma_shp = ReshapeLayer(
l_gamma, ([0], [1], 1, 1))
from TProd3 import TensorTemperatureLayer
l_decoder = TensorTemperatureLayer((l_shp1, l_input_regions, l_gamma_shp), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor')
else:
l_decoder = TensorProdFactLayer((l_shp1, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor')
if CFG['TENSOR_COND_WORD']:
from RNNTraining import get_Regions_cond_words
l_region = ExpressionLayer(
l_decoder, get_Regions_cond_words, output_shape='auto', name='l_region')
else:
l_region = ExpressionLayer(l_decoder, lambda X: X.sum(
3), output_shape='auto', name='l_region') # sum over
l_region = ReshapeLayer(
l_region, ([0], [2]), name='l_region')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
#}}}
elif CFG['MODE'] == 'tensor-reducedw':
# {{{2
from TProd3 import TensorProdFactLayer
# input: [h(batch,dimh),r(num_batch,r_dim,num_r)]
# output: [ h[0],r[2], dim_w ]
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regins')
if CFG.has_key('TENSOR_RECTIFY') and CFG['TENSOR_RECTIFY']:
l_tensor = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG[
'REGION_SIZE'], dim_w=CFG['EMBEDDING_WORDS'], nonlinearity=lasagne.nonlinearities.rectify, name='l_tensor')
else:
l_tensor = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG[
'REGION_SIZE'], dim_w=CFG['EMBEDDING_WORDS'], nonlinearity=lasagne.nonlinearities.identity, name='l_tensor')
# softmax does not accept non-flat layers, then flatten->softmax->reshape
l_flatten = ReshapeLayer(
l_decoder, (CFG['BATCH_SIZE'] * 1 * CFG['NUM_REGIONS'], CFG['EMBEDDING_WORDS']), name='l_flatten')
l_words = DenseLayer(l_flatten, num_units=len(vocab),
nonlinearity=lasagne.nonlinearities.identity, name='l_words')
l_reshape = ReshapeLayer(
l_words, (CFG['BATCH_SIZE'] * 1, CFG['NUM_REGIONS'] * len(vocab)), name='l_reshape')
l_softmax = lasagne.layers.NonlinearityLayer(
l_reshape, nonlinearity=lasagne.nonlinearities.softmax, name='l_softmax')
l_reshape1 = ReshapeLayer(
l_softmax, (CFG['BATCH_SIZE'], 1, CFG['NUM_REGIONS'], len(vocab)), name='l_reshape1')
l_out = ExpressionLayer(l_reshape1, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
# }}}
elif CFG['MODE'] == 'tensor-removedWrw':
from TProd3 import TensorProdFact2Layer
# input: [h(batch,dimh),r(num_batch,r_dim,num_r)]
# output: [ h[0],r[2], dim_w ]
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regins')
l_decoder = TensorProdFact2Layer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_decoder')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum( 2), output_shape='auto', name='l_out') # sum over regions
net_dictionnary = {'loc1': l_loc1, 'input_loc': l_input_loc, 'sel_region2': l_sel_region2,
'loc': l_loc, 'conv': l_conv, 'prev': l_prev_gru, 'input': l_cell_input,
'gru': l_gru, 'sent': l_input_sentence, 'img': l_input_img, 'img2': l_input_img2,
'reg_feedback2': l_region_feedback, 'reg_feedback': l_region, 'reg': l_input_reg,
'out_reg': l_out_reg, 'out': l_out, 'cnn': l_cnn_embedding, 'sent_emb': l_sentence_embedding,
'decoder': l_decoder, 'boxes': l_boxes, 'weighted_regions_prev': l_weighted_region_prev,
'weighted_regions': l_weighted_region}
return net_dictionnary
def build_autoencoder(layer,
nonlinearity='same',
b=init.Constant(0.),
learnable_conv=False):
"""
Unfolds a stack of layers into a symmetric autoencoder with tied weights.
Given a :class:`Layer` instance, this function builds a
symmetric autoencoder with tied weights.
Parameters
----------
layer : a :class:`Layer` instance or a tuple
The :class:`Layer` instance with respect to which a symmetric
autoencoder is built.
nonlinearity : 'same', list, callable, or None
The nonlinearities that are applied to the decoding layer.
If 'same', each decoder layer has the same nonlinearity as its
corresponding encoder layer. If a list is provided, it must contain
nonlinearities for each decoding layer. Otherwise, if a single
nonlinearity is provided, it is applied to all decoder layers.
If set to ``None``, all nonlinearities for the decoder layers are set
to lasagne.nonlinearities.identity.
b : callable, Theano shared variable, numpy array, list or None
An initializer for the decoder biases. By default, all decoder
biases are initialized to lasagne.init.Constant(0.). If a shared
variable or a numpy array is provided, the shape must match the
incoming shape (only in case all incoming shapes are the same).
Additianlly, a list containing initializers for the biases of each
decoder layer can be provided. If set to ``None``, the decoder
layers will have no biases, and pass through their input instead.
Returns
-------
layer: :class:`Layer` instance
The output :class:`Layer` of the symmetric autoencoder with
tied weights.
encoder: :class:`Layer` instance
The code :class:`Layer` of the autoencoder (see Notes)
Notes
-----
The encoder (input) :class:`Layer` is changed using
`unfold_bias_and_nonlinearity_layers`. Therefore, this layer is not the
code layer anymore, because it has got its bias and nonlinearity stripped
off.
Examples
--------
>>> from lasagne.layers import InputLayer, DenseLayer
>>> from lasagne.layers import build_autoencoder
>>> l_in = InputLayer((100, 20))
>>> l1 = DenseLayer(l_in, num_units=50)
>>> l2 = DenseLayer(l1, num_units=10)
>>> l_ae, l2 = build_autoencoder(l2, nonlinearity='same', b=None)
"""
if isinstance(nonlinearity, (tuple, list)):
n_idx = 0
if isinstance(b, (tuple, list)):
b_idx = 0
encoder = unfold_bias_and_nonlinearity_layers(layer)
layers = get_all_layers(encoder)
autoencoder_layers = [encoder]
kwargs_b = dict(b=None)
kwargs_n = dict(nonlinearity=nonlinearities.identity)
for i, layer in enumerate(layers[::-1]):
incoming = autoencoder_layers[-1]
if isinstance(layer, InputLayer):
continue
elif isinstance(layer, BiasLayer):
if b is None:
kwargs_b = dict(b=None)
elif isinstance(b, (tuple, list)):
kwargs_b = dict(b=b[b_idx])
b_idx += 1
else:
kwargs_b = dict(b=b)
elif isinstance(layer, NonlinearityLayer):
if nonlinearity == 'same':
kwargs_n = dict(nonlinearity=layer.nonlinearity)
elif nonlinearity is None:
kwargs_n = dict(nonlinearity=nonlinearities.identity)
elif isinstance(nonlinearity, (tuple, list)):
kwargs_n = dict(nonlinearity=nonlinearity[n_idx])
n_idx += 1
else:
kwargs_n = dict(nonlinearity=nonlinearity)
elif isinstance(layer, DropoutLayer):
a_layer = DropoutLayer(incoming=incoming,
p=layer.p,
rescale=layer.rescale)
autoencoder_layers.append(a_layer)
elif isinstance(layer, GaussianNoiseLayer):
a_layer = GaussianNoiseLayer(incoming=incoming, sigma=layer.sigma)
autoencoder_layers.append(a_layer)
elif isinstance(layer, L.Conv2DLayer):
a_layer = L.TransposedConv2DLayer(incoming=incoming,
num_filters=layer.input_shape[1],
filter_size=layer.filter_size,
stride=layer.stride,
crop=layer.pad,
untie_biases=layer.untie_biases,
b=None,
nonlinearity=None)
elif isinstance(layer, L.MaxPool2DLayer):
a_layer = L.Upscale2DLayer(incoming=incoming,
scale_factor=layer.pool_size)
elif isinstance(layer, L.BatchNormLayer):
a_layer = L.BatchNormLayer(incoming)
else:
a_layer = InverseLayer(incoming=incoming, layer=layer)
if hasattr(layer, 'b'):
a_layer = BiasLayer(incoming=a_layer, **kwargs_b)
if hasattr(layer, 'nonlinearity'):
a_layer = NonlinearityLayer(incoming=a_layer, **kwargs_n)
autoencoder_layers.append(a_layer)
return autoencoder_layers, encoder
def build_model_vgg16(input_shape, verbose):
'''
See Lasagne Modelzoo:
https://github.com/Lasagne/Recipes/blob/master/modelzoo/vgg16.py
'''
if verbose: print 'VGG16 (from lasagne model zoo)'
net = {}
net['input'] = InputLayer(input_shape)
net['conv1_1'] = ConvLayer(net['input'], 64, 3, pad=1, flip_filters=False)
net['conv1_2'] = ConvLayer(net['conv1_1'],
64,
3,
pad=1,
flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
net['conv2_1'] = ConvLayer(net['pool1'], 128, 3, pad=1, flip_filters=False)
net['conv2_2'] = ConvLayer(net['conv2_1'],
128,
3,
pad=1,
flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
net['conv3_1'] = ConvLayer(net['pool2'], 256, 3, pad=1, flip_filters=False)
net['conv3_2'] = ConvLayer(net['conv3_1'],
256,
3,
pad=1,
flip_filters=False)
net['conv3_3'] = ConvLayer(net['conv3_2'],
256,
3,
pad=1,
flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
net['conv4_1'] = ConvLayer(net['pool3'], 512, 3, pad=1, flip_filters=False)
net['conv4_2'] = ConvLayer(net['conv4_1'],
512,
3,
pad=1,
flip_filters=False)
net['conv4_3'] = ConvLayer(net['conv4_2'],
512,
3,
pad=1,
flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
net['conv5_1'] = ConvLayer(net['pool4'], 512, 3, pad=1, flip_filters=False)
net['conv5_2'] = ConvLayer(net['conv5_1'],
512,
3,
pad=1,
flip_filters=False)
net['conv5_3'] = ConvLayer(net['conv5_2'],
512,
3,
pad=1,
flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = DenseLayer(net['fc6_dropout'], num_units=4096)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['fc8'] = DenseLayer(net['fc7_dropout'],
num_units=1000,
nonlinearity=None)
net['prob'] = NonlinearityLayer(net['fc8'], softmax)
# for layer in net.values():
# print str(lasagne.layers.get_output_shape(layer))
return net
def __init__(self,
incoming,
axes='auto',
droprate=0.2,
epsilon=1e-4,
alpha=0.1,
sparsity=1.0,
beta=init.Constant(0),
gamma=init.Constant(1),
mean=init.Constant(0),
inv_std=init.Constant(1),
**kwargs):
super(BatchNormLayer, self).__init__(incoming, **kwargs)
if axes == 'auto':
# default: normalize over all but the second axis
axes = (0, ) + tuple(range(2, len(self.input_shape)))
elif isinstance(axes, int):
axes = (axes, )
self.axes = axes
if len(axes) == 1:
self.mean_axes = self.axes
else:
self.mean_axes = (axes[1], )
self.epsilon = epsilon
self.alpha = alpha
# create parameters, ignoring all dimensions in axes
shape = [
size for axis, size in enumerate(self.input_shape)
if axis not in self.axes
]
meanshape = [
size for axis, size in enumerate(self.input_shape)
if axis not in self.mean_axes
]
if any(size is None for size in shape):
raise ValueError("BatchNormLayer needs specified input sizes for "
"all axes not normalized over.")
if beta is None:
self.beta = None
else:
self.beta = self.add_param(beta,
shape,
'beta',
trainable=True,
regularizable=False)
if gamma is None:
self.gamma = None
else:
self.gamma = self.add_param(gamma,
shape,
'gamma',
trainable=True,
regularizable=True)
self.mean = self.add_param(mean,
meanshape,
'mean',
trainable=False,
regularizable=False)
self.inv_std = self.add_param(inv_std,
meanshape,
'inv_std',
trainable=False,
regularizable=False)
#print('here',len(self.input_shape))
self.sparsity = sparsity
if len(self.input_shape) == 3:
self.dropout = DropoutLayer(
(self.input_shape[0], self.input_shape[1],
self.input_shape[2]),
p=droprate,
shared_axes=(0, 1),
**kwargs)
else:
self.dropout = DropoutLayer(
(self.input_shape[0], self.input_shape[1]),
p=droprate,
shared_axes=(0, ),
**kwargs)
def build_model_vgg_cnn_s(input_shape, verbose):
'''
See Lasagne Modelzoo:
https://github.com/Lasagne/Recipes/blob/master/modelzoo/vgg_cnn_s.py
'''
if verbose: print 'VGG_cnn_s (from lasagne model zoo)'
net = {}
net['input'] = InputLayer(input_shape)
net['conv1'] = ConvLayer(net['input'],
num_filters=96,
filter_size=7,
stride=2,
flip_filters=False)
net['norm1'] = LRNLayer(
net['conv1'], alpha=0.0001) # caffe has alpha = alpha * pool_size
net['pool1'] = PoolLayer(net['norm1'],
pool_size=3,
stride=3,
ignore_border=False)
net['conv2'] = ConvLayer(net['pool1'],
num_filters=256,
filter_size=5,
flip_filters=False)
net['pool2'] = PoolLayer(net['conv2'],
pool_size=2,
stride=2,
ignore_border=False)
net['conv3'] = ConvLayer(net['pool2'],
num_filters=512,
filter_size=3,
pad=1,
flip_filters=False)
net['conv4'] = ConvLayer(net['conv3'],
num_filters=512,
filter_size=3,
pad=1,
flip_filters=False)
net['conv5'] = ConvLayer(net['conv4'],
num_filters=512,
filter_size=3,
pad=1,
flip_filters=False)
net['pool5'] = PoolLayer(net['conv5'],
pool_size=3,
stride=3,
ignore_border=False)
net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
net['drop6'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = DenseLayer(net['drop6'], num_units=4096)
net['drop7'] = DropoutLayer(net['fc7'], p=0.5)
net['fc8'] = DenseLayer(net['drop7'],
num_units=1000,
nonlinearity=lasagne.nonlinearities.softmax)
if verbose:
for layer in net.values():
print str(lasagne.layers.get_output_shape(layer))
return net
def build_cnn(input_var=None):
# This function returns network, network prediction
# Structure
# Input - Grayscale - Conv1 + Max pooling - Dense - Dropout -
# Unpool1 - Deconv1 - Output
conv1_filter_cnt = 100
conv1_filter_size = 5
maxpool1_size = 2
conv2_filter_cnt = 50
conv2_filter_size = 5
maxpool2_size = 2
dense_units_cnt = 3000
batch_size = input_var.shape[0]
image_size = 32
after_conv1 = image_size
after_pool1 = (after_conv1 + maxpool1_size - 1) // maxpool1_size
l_in = InputLayer(
shape=(None, 3, image_size, image_size),
input_var=input_var,
)
l_in_grayscale = GrayscaleLayer(incoming=l_in, )
print(lasagne.layers.get_output_shape(l_in_grayscale))
l_conv1 = Conv2DLayer(
incoming=l_in_grayscale,
num_filters=conv1_filter_cnt,
filter_size=conv1_filter_size,
stride=1,
pad='same',
nonlinearity=tanh,
)
print(lasagne.layers.get_output_shape(l_conv1))
l_maxpool1 = MaxPool2DLayer(
incoming=l_conv1,
pool_size=maxpool1_size,
stride=maxpool1_size,
)
print(lasagne.layers.get_output_shape(l_maxpool1))
l_dense1 = DenseLayer(
incoming=l_maxpool1,
num_units=dense_units_cnt,
nonlinearity=tanh,
)
print(lasagne.layers.get_output_shape(l_dense1))
l_drop1 = DropoutLayer(
incoming=l_dense1,
p=0.3,
)
print(lasagne.layers.get_output_shape(l_drop1))
l_pre_unpool1 = DenseLayer(
incoming=l_drop1,
num_units=conv1_filter_cnt * (after_pool1**2),
nonlinearity=tanh,
)
print(lasagne.layers.get_output_shape(l_pre_unpool1))
#l_drop2 = DropoutLayer(
# incoming = l_pre_unpool1,
# p = 0.3,
#)
l_pre_unpool1 = ReshapeLayer(
incoming=l_pre_unpool1,
shape=(batch_size, conv1_filter_cnt) + (after_pool1, after_pool1),
)
print(lasagne.layers.get_output_shape(l_pre_unpool1))
l_unpool1 = Unpool2DLayer(
incoming=l_pre_unpool1,
kernel_size=maxpool1_size,
)
print(lasagne.layers.get_output_shape(l_unpool1))
l_deconv1 = Conv2DLayer(
incoming=l_unpool1,
num_filters=3,
filter_size=conv1_filter_size,
stride=1,
pad='same',
nonlinearity=tanh,
)
print(lasagne.layers.get_output_shape(l_deconv1))
l_out = ReshapeLayer(incoming=l_deconv1, shape=input_var.shape)
print(lasagne.layers.get_output_shape(l_out))
return (l_out, lasagne.layers.get_output(l_out, deterministic=True))
# ==== Parameters ====
num_features = X.shape[1]
epochs = 20
hidden_layers = 4
hidden_units = 1024
dropout_p = 0.75
val_auc = np.zeros(epochs)
# ==== Defining the neural network shape ====
l_in = InputLayer(shape=(None, num_features))
l_hidden1 = DenseLayer(l_in, num_units=hidden_units)
l_hidden2 = DropoutLayer(l_hidden1, p=dropout_p)
l_current = l_hidden2
for k in range(hidden_layers - 1):
l_current = highway_dense(l_current)
l_current = DropoutLayer(l_current, p=dropout_p)
l_dropout = DropoutLayer(l_current, p=dropout_p)
l_out = DenseLayer(l_dropout, num_units=2, nonlinearity=softmax)
# ==== Neural network definition ====
net1 = NeuralNet(layers=l_out,
update=adadelta,
update_rho=0.95,
update_learning_rate=1.0,
objective_loss_function=categorical_crossentropy,
train_split=TrainSplit(eval_size=0),
def build(inputHeight, inputWidth, input_var,do_dropout=False):
#net = OrderedDict()
net = {'input': InputLayer((None, 3, inputHeight, inputWidth), input_var=input_var)}
#net['input'] = InputLayer((None, 3, inputHeight, inputWidth), input_var=input_var)
print "Input: {}".format(net['input'].output_shape[1:])
net['bgr'] = RGBtoBGRLayer(net['input'])
net['contr_1_1'] = batch_norm(ConvLayer(net['bgr'], 64, 3,pad='same',W=GlorotNormal(gain="relu")))
print "convtr1_1: {}".format(net['contr_1_1'].output_shape[1:])
net['contr_1_2'] = batch_norm(ConvLayer(net['contr_1_1'],64,3, pad='same',W=GlorotNormal(gain="relu")))
print "convtr1_2: {}".format(net['contr_1_2'].output_shape[1:])
net['pool1'] = Pool2DLayer(net['contr_1_2'], 2)
print"pool1: {}".format(net['pool1'].output_shape[1:])
net['contr_2_1'] = batch_norm(ConvLayer(net['pool1'], 128, 3, pad='same',W=GlorotNormal(gain="relu")))
print "convtr2_1: {}".format(net['contr_2_1'].output_shape[1:])
net['contr_2_2'] = batch_norm(ConvLayer(net['contr_2_1'], 128, 3, pad='same',W=GlorotNormal(gain="relu")))
print "convtr2_2: {}".format(net['contr_2_2'].output_shape[1:])
net['pool2'] = Pool2DLayer(net['contr_2_2'], 2)
print "pool2: {}".format(net['pool2'].output_shape[1:])
net['contr_3_1'] = batch_norm(ConvLayer(net['pool2'],256, 3, pad='same',W=GlorotNormal(gain="relu")))
print "convtr3_1: {}".format(net['contr_3_1'].output_shape[1:])
net['contr_3_2'] = batch_norm(ConvLayer(net['contr_3_1'], 256, 3, pad='same',W=GlorotNormal(gain="relu")))
print "convtr3_2: {}".format(net['contr_3_2'].output_shape[1:])
net['pool3'] = Pool2DLayer(net['contr_3_2'], 2)
print "pool3: {}".format(net['pool3'].output_shape[1:])
net['contr_4_1'] = batch_norm(ConvLayer(net['pool3'], 512, 3, pad='same',W=GlorotNormal(gain="relu")))
print "convtr4_1: {}".format(net['contr_4_1'].output_shape[1:])
net['contr_4_2'] = batch_norm(ConvLayer(net['contr_4_1'],512, 3,pad='same',W=GlorotNormal(gain="relu")))
print "convtr4_2: {}".format(net['contr_4_2'].output_shape[1:])
l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2)
print "pool4: {}".format(net['pool4'].output_shape[1:])
# the paper does not really describe where and how dropout is added. Feel free to try more options
if do_dropout:
l = DropoutLayer(l, p=0.4)
net['encode_1'] = batch_norm(ConvLayer(l,1024, 3,pad='same', W=GlorotNormal(gain="relu")))
print "encode_1: {}".format(net['encode_1'].output_shape[1:])
net['encode_2'] = batch_norm(ConvLayer(net['encode_1'], 1024, 3,pad='same', W=GlorotNormal(gain="relu")))
print "encode_2: {}".format(net['encode_2'].output_shape[1:])
net['upscale1'] = batch_norm(Deconv2DLayer(net['encode_2'],512, 2, 2, crop="valid", W=GlorotNormal(gain="relu")))
print "upscale1: {}".format(net['upscale1'].output_shape[1:])
net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']], cropping=(None, None, "center", "center"))
print "concat1: {}".format(net['concat1'].output_shape[1:])
net['expand_1_1'] = batch_norm(ConvLayer(net['concat1'], 512, 3,pad='same', W=GlorotNormal(gain="relu")))
print "expand_1_1: {}".format(net['expand_1_1'].output_shape[1:])
net['expand_1_2'] = batch_norm(ConvLayer(net['expand_1_1'],512, 3,pad='same',W=GlorotNormal(gain="relu")))
print "expand_1_2: {}".format(net['expand_1_2'].output_shape[1:])
net['upscale2'] = batch_norm(Deconv2DLayer(net['expand_1_2'], 256, 2, 2, crop="valid", W=GlorotNormal(gain="relu")))
print "upscale2: {}".format(net['upscale2'].output_shape[1:])
net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']], cropping=(None, None, "center", "center"))
print "concat2: {}".format(net['concat2'].output_shape[1:])
net['expand_2_1'] = batch_norm(ConvLayer(net['concat2'], 256, 3,pad='same',W=GlorotNormal(gain="relu")))
print "expand_2_1: {}".format(net['expand_2_1'].output_shape[1:])
net['expand_2_2'] = batch_norm(ConvLayer(net['expand_2_1'], 256, 3,pad='same',W=GlorotNormal(gain="relu")))
print "expand_2_2: {}".format(net['expand_2_2'].output_shape[1:])
net['upscale3'] = batch_norm(Deconv2DLayer(net['expand_2_2'],128, 2, 2, crop="valid",W=GlorotNormal(gain="relu")))
print "upscale3: {}".format(net['upscale3'].output_shape[1:])
net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']], cropping=(None, None, "center", "center"))
print "concat3: {}".format(net['concat3'].output_shape[1:])
net['expand_3_1'] = batch_norm(ConvLayer(net['concat3'], 128, 3,pad='same',W=GlorotNormal(gain="relu")))
print "expand_3_1: {}".format(net['expand_3_1'].output_shape[1:])
net['expand_3_2'] = batch_norm(ConvLayer(net['expand_3_1'],128, 3,pad='same', W=GlorotNormal(gain="relu")))
print "expand_3_2: {}".format(net['expand_3_2'].output_shape[1:])
net['upscale4'] = batch_norm(Deconv2DLayer(net['expand_3_2'], 64, 2, 2, crop="valid", W=GlorotNormal(gain="relu")))
print "upscale4: {}".format(net['upscale4'].output_shape[1:])
net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']], cropping=(None, None, "center", "center"))
print "concat4: {}".format(net['concat4'].output_shape[1:])
net['expand_4_1'] = batch_norm(ConvLayer(net['concat4'], 64, 3,pad='same', W=GlorotNormal(gain="relu")))
print "expand_4_1: {}".format(net['expand_4_1'].output_shape[1:])
net['expand_4_2'] = batch_norm(ConvLayer(net['expand_4_1'],64, 3,pad='same', W=GlorotNormal(gain="relu")))
print "expand_4_2: {}".format(net['expand_4_2'].output_shape[1:])
net['output'] = ConvLayer(net['expand_4_2'],1, 1, nonlinearity=sigmoid)
print "output: {}".format(net['output'].output_shape[1:])
# net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3))
# print "dimshuffle: {}".format(net['dimshuffle'].output_shape[1:])
# net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (2, -1))
# print "reshapeSeg: {}".format(net['reshapeSeg'].output_shape[1:])
# net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0))
# print "dimshuffle2: {}".format(net['dimshuffle2'].output_shape[1:])
# net['output_flattened'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax)
# print "output_flattened: {}".format(net['output_flattened'].output_shape[1:])
return net
def build_UNet(inputVar=None,
nonlinearity=lasagne.nonlinearities.elu,
input_dim=(128, 128),
base_n_filters=64,
do_dropout=False):
net = OrderedDict()
pad = "same"
if not inputVar:
net['input'] = InputLayer((None, 3, input_dim[0], input_dim[1]))
else:
net['input'] = InputLayer((None, 3, input_dim[0], input_dim[1]),
inputVar)
net['contr_1_1'] = batch_norm(
ConvLayer(net['input'],
base_n_filters,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['contr_1_2'] = batch_norm(
ConvLayer(net['contr_1_1'],
base_n_filters,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['pool1'] = Pool2DLayer(net['contr_1_2'], 2)
net['contr_2_1'] = batch_norm(
ConvLayer(net['pool1'],
base_n_filters * 2,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['contr_2_2'] = batch_norm(
ConvLayer(net['contr_2_1'],
base_n_filters * 2,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['pool2'] = Pool2DLayer(net['contr_2_2'], 2)
net['contr_3_1'] = batch_norm(
ConvLayer(net['pool2'],
base_n_filters * 4,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['contr_3_2'] = batch_norm(
ConvLayer(net['contr_3_1'],
base_n_filters * 4,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['pool3'] = Pool2DLayer(net['contr_3_2'], 2)
net['contr_4_1'] = batch_norm(
ConvLayer(net['pool3'],
base_n_filters * 8,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['contr_4_2'] = batch_norm(
ConvLayer(net['contr_4_1'],
base_n_filters * 8,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2)
# the paper does not really describe where and how dropout is added. Feel free to try more options
if do_dropout:
l = DropoutLayer(l, p=0.4)
net['encode_1'] = batch_norm(
ConvLayer(l,
base_n_filters * 16,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['encode_2'] = batch_norm(
ConvLayer(net['encode_1'],
base_n_filters * 16,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['upscale1'] = batch_norm(
Deconv2DLayer(net['encode_2'],
base_n_filters * 16,
2,
2,
crop="valid",
nonlinearity=nonlinearity,
W=HeNormal(gain="relu")))
net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']],
cropping=(None, None, "center", "center"))
net['expand_1_1'] = batch_norm(
ConvLayer(net['concat1'],
base_n_filters * 8,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['expand_1_2'] = batch_norm(
ConvLayer(net['expand_1_1'],
base_n_filters * 8,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['upscale2'] = batch_norm(
Deconv2DLayer(net['expand_1_2'],
base_n_filters * 8,
2,
2,
crop="valid",
nonlinearity=nonlinearity,
W=HeNormal(gain="relu")))
net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']],
cropping=(None, None, "center", "center"))
net['expand_2_1'] = batch_norm(
ConvLayer(net['concat2'],
base_n_filters * 4,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['expand_2_2'] = batch_norm(
ConvLayer(net['expand_2_1'],
base_n_filters * 4,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['upscale3'] = batch_norm(
Deconv2DLayer(net['expand_2_2'],
base_n_filters * 4,
2,
2,
crop="valid",
nonlinearity=nonlinearity,
W=HeNormal(gain="relu")))
net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']],
cropping=(None, None, "center", "center"))
net['expand_3_1'] = batch_norm(
ConvLayer(net['concat3'],
base_n_filters * 2,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['expand_3_2'] = batch_norm(
ConvLayer(net['expand_3_1'],
base_n_filters * 2,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['upscale4'] = batch_norm(
Deconv2DLayer(net['expand_3_2'],
base_n_filters * 2,
2,
2,
crop="valid",
nonlinearity=nonlinearity,
W=HeNormal(gain="relu")))
net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']],
cropping=(None, None, "center", "center"))
net['expand_4_1'] = batch_norm(
ConvLayer(net['concat4'],
base_n_filters,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['expand_4_2'] = batch_norm(
ConvLayer(net['expand_4_1'],
base_n_filters,
3,
nonlinearity=nonlinearity,
pad=pad,
W=HeNormal(gain="relu")))
net['output'] = ConvLayer(net['expand_4_2'], 3, 1, nonlinearity=None)
return net
def build_vgg_model():
net = {}
net['input'] = InputLayer((None, 3, 224, 224))
net['conv1_1'] = ConvLayer(net['input'],
64,
3,
pad=1,
flip_filters=False)
net['conv1_2'] = ConvLayer(net['conv1_1'],
64,
3,
pad=1,
flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
net['conv2_1'] = ConvLayer(net['pool1'],
128,
3,
pad=1,
flip_filters=False)
net['conv2_2'] = ConvLayer(net['conv2_1'],
128,
3,
pad=1,
flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
net['conv3_1'] = ConvLayer(net['pool2'],
256,
3,
pad=1,
flip_filters=False)
net['conv3_2'] = ConvLayer(net['conv3_1'],
256,
3,
pad=1,
flip_filters=False)
net['conv3_3'] = ConvLayer(net['conv3_2'],
256,
3,
pad=1,
flip_filters=False)
net['conv3_4'] = ConvLayer(net['conv3_3'],
256,
3,
pad=1,
flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_4'], 2)
net['conv4_1'] = ConvLayer(net['pool3'],
512,
3,
pad=1,
flip_filters=False)
net['conv4_2'] = ConvLayer(net['conv4_1'],
512,
3,
pad=1,
flip_filters=False)
net['conv4_3'] = ConvLayer(net['conv4_2'],
512,
3,
pad=1,
flip_filters=False)
net['conv4_4'] = ConvLayer(net['conv4_3'],
512,
3,
pad=1,
flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_4'], 2)
net['conv5_1'] = ConvLayer(net['pool4'],
512,
3,
pad=1,
flip_filters=False)
net['conv5_2'] = ConvLayer(net['conv5_1'],
512,
3,
pad=1,
flip_filters=False)
net['conv5_3'] = ConvLayer(net['conv5_2'],
512,
3,
pad=1,
flip_filters=False)
net['conv5_4'] = ConvLayer(net['conv5_3'],
512,
3,
pad=1,
flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_4'], 2)
net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = DenseLayer(net['fc6_dropout'], num_units=4096)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['fc8'] = DenseLayer(net['fc7_dropout'],
num_units=1000,
nonlinearity=None)
net['prob'] = NonlinearityLayer(net['fc8'], softmax)
# Remove the trainable argument from the layers that can potentially have it
for key, val in net.iteritems():
if not ('dropout' or 'pool' in key):
net[key].params[net[key].W].remove("trainable")
net[key].params[net[key].b].remove("trainable")
return net
def build_model(inp):
net = {}
net['input'] = InputLayer((None, 3, 224, 224), input_var=input_var)
net['conv1_1'] = batch_norm(
Conv2DLayer(net['input'], 64, 3, pad=1, flip_filters=False))
net['conv1_2'] = batch_norm(
Conv2DLayer(net['conv1_1'], 64, 3, pad=1, flip_filters=False))
net['pool1'] = MaxPool2DLayer(net['conv1_2'], 2)
net['conv2_1'] = batch_norm(
Conv2DLayer(net['pool1'], 128, 3, pad=1, flip_filters=False))
net['conv2_2'] = batch_norm(
Conv2DLayer(net['conv2_1'], 128, 3, pad=1, flip_filters=False))
net['pool2'] = MaxPool2DLayer(net['conv2_2'], 2)
net['conv3_1'] = batch_norm(
Conv2DLayer(net['pool2'], 256, 3, pad=1, flip_filters=False))
net['conv3_2'] = batch_norm(
Conv2DLayer(net['conv3_1'], 256, 3, pad=1, flip_filters=False))
net['conv3_3'] = batch_norm(
Conv2DLayer(net['conv3_2'], 256, 3, pad=1, flip_filters=False))
net['pool3'] = MaxPool2DLayer(net['conv3_3'], 2)
net['conv4_1'] = batch_norm(
Conv2DLayer(net['pool3'], 512, 3, pad=1, flip_filters=False))
net['conv4_2'] = batch_norm(
Conv2DLayer(net['conv4_1'], 512, 3, pad=1, flip_filters=False))
net['conv4_3'] = batch_norm(
Conv2DLayer(net['conv4_2'], 512, 3, pad=1, flip_filters=False))
net['pool4'] = MaxPool2DLayer(net['conv4_3'], 2)
net['conv5_1'] = batch_norm(
Conv2DLayer(net['pool4'], 512, 3, pad=1, flip_filters=False))
net['conv5_2'] = batch_norm(
Conv2DLayer(net['conv5_1'], 512, 3, pad=1, flip_filters=False))
net['conv5_3'] = batch_norm(
Conv2DLayer(net['conv5_2'], 512, 3, pad=1, flip_filters=False))
net['pool5'] = MaxPool2DLayer(net['conv5_3'], 2)
net['fc6'] = batch_norm(DenseLayer(net['pool5'], num_units=4096))
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = batch_norm(DenseLayer(net['fc6_dropout'], num_units=4096))
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['reshape'] = ReshapeLayer(net['fc7_dropout'], (-1, 4096, 1, 1))
net['deconv6'] = batch_norm(Deconv2DLayer(net['reshape'], 512, 7))
net['unpool5'] = Upscale2DLayer(net['deconv6'], 2)
net['deconv5_3'] = batch_norm(
Deconv2DLayer(net['unpool5'], 512, 3, crop=1, flip_filters=False))
net['deconv5_2'] = batch_norm(
Deconv2DLayer(net['deconv5_3'], 512, 3, crop=1, flip_filters=False))
net['deconv5_1'] = batch_norm(
Deconv2DLayer(net['deconv5_2'], 512, 3, crop=1, flip_filters=False))
net['unpool4'] = Upscale2DLayer(net['deconv5_1'], 2)
net['deconv4_3'] = batch_norm(
Deconv2DLayer(net['unpool4'], 512, 3, crop=1, flip_filters=False))
net['deconv4_2'] = batch_norm(
Deconv2DLayer(net['deconv4_3'], 512, 3, crop=1, flip_filters=False))
net['deconv4_1'] = batch_norm(
Deconv2DLayer(net['deconv4_2'], 512, 3, crop=1, flip_filters=False))
net['unpool3'] = Upscale2DLayer(net['deconv4_1'], 2)
net['deconv3_3'] = batch_norm(
Deconv2DLayer(net['unpool3'], 256, 3, crop=1, flip_filters=False))
net['deconv3_2'] = batch_norm(
Deconv2DLayer(net['deconv3_3'], 256, 3, crop=1, flip_filters=False))
net['deconv3_1'] = batch_norm(
Deconv2DLayer(net['deconv3_2'], 256, 3, crop=1, flip_filters=False))
net['unpool2'] = Upscale2DLayer(net['deconv3_1'], 2)
net['deconv2_2'] = batch_norm(
Deconv2DLayer(net['unpool2'], 128, 3, crop=1, flip_filters=False))
net['deconv2_1'] = batch_norm(
Deconv2DLayer(net['deconv2_2'], 128, 3, crop=1, flip_filters=False))
net['unpool1'] = Upscale2DLayer(net['deconv2_1'], 2)
net['deconv1_1'] = batch_norm(
Deconv2DLayer(net['unpool1'], 64, 3, crop=1, flip_filters=False))
net['deconv1_2'] = batch_norm(
Deconv2DLayer(net['deconv1_1'], 64, 3, crop=1, flip_filters=False))
net['out'] = NonlinearityLayer(DenseLayer(net['deconv1_2'], 21), softmax)
return net