def broadcast_dot_layer(l_pred, l_targets, feature_dim, id_tag):
l_broadcast = dimshuffle(l_pred, (0, 1, 'x'), name=id_tag + 'dot_broadcast')
l_forget = ForgetSizeLayer(l_broadcast, axis=2, name=id_tag + 'dot_nosize')
l_merge = ElemwiseMergeLayer((l_forget, l_targets), T.mul, name=id_tag + 'dot_elemwise_mul')
l_pool = FeaturePoolLayer(l_merge, pool_size=feature_dim, axis=1,
pool_function=T.sum, name=id_tag + 'dot_pool')
return reshape(l_pool, ([0], [2]), name=id_tag + 'broadcast_dot')
def create_attention(self, gru_con, in_con_mask, condition, batch_size,
n_hidden_con, **kwargs):
# (batch_size, n_attention)
gru_cond2 = non_flattening_dense_layer(gru_con,
self.in_con_mask,
self.n_attention,
nonlinearity=None)
gru_que2 = DenseLayer(condition, self.n_attention, nonlinearity=None)
gru_que2 = dimshuffle(gru_que2, (0, 'x', 1))
att = ElemwiseSumLayer([gru_cond2, gru_que2])
att = NonlinearityLayer(att, T.tanh)
att = SliceLayer(non_flattening_dense_layer(att,
self.in_con_mask,
1,
nonlinearity=None),
indices=0,
axis=2)
att_softmax = SequenceSoftmax(att, self.in_con_mask)
rep = ElemwiseMergeLayer(
[ForgetSizeLayer(dimshuffle(att_softmax,
(0, 1, 'x'))), gru_con], T.mul)
return ExpressionLayer(rep, lambda x: T.sum(x, axis=1), lambda s:
(s[0], ) + s[2:])
def build_convpool_max(input_vars, input_shape=None):
"""
Builds the complete network with maxpooling layer in time.
:param input_vars: list of EEG images (one image per time window)
:return: a pointer to the output of last layer
"""
convnets = []
W_init = None
# Build 7 parallel CNNs with shared weights
for i in range(input_shape[0]):
if i == 0:
convnet, W_init = build_cnn(input_vars[i], input_shape)
else:
convnet, _ = build_cnn(input_vars[i], input_shape, W_init)
convnets.append(convnet)
# convpooling using Max pooling over frames
convpool = ElemwiseMergeLayer(convnets, theano.tensor.maximum)
# A fully-connected layer of 512 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=512,
nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the output layer with 50% dropout on its inputs:
convpool = lasagne.layers.DenseLayer(
lasagne.layers.dropout(convpool, p=.5),
num_units=num_classes,
nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def build_convpool_max(input_vars, nb_classes, imsize=32, n_colors=3, n_timewin=3):
"""
Builds the complete network with maxpooling layer in time.
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 7 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i], imsize=imsize, n_colors=n_colors)
else:
convnet, _ = build_cnn(input_vars[i], w_init=w_init, imsize=imsize, n_colors=n_colors)
convnets.append(convnet)
# convpooling using Max pooling over frames
convpool = ElemwiseMergeLayer(convnets, theano.tensor.maximum)
# A fully-connected layer of 512 units with 50% dropout on its inputs:
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=512, nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the output layer with 50% dropout on its inputs:
convpool = lasagne.layers.DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=nb_classes, nonlinearity=lasagne.nonlinearities.softmax)
return convpool
def broadcast_sub_layer(l_pred, l_targets, feature_dim, id_tag):
l_broadcast = dimshuffle(l_pred, (0, 1, 'x'),
name=id_tag + 'sub_broadcast')
l_forget = ForgetSizeLayer(l_broadcast, axis=2, name=id_tag + 'sub_nosize')
return ElemwiseMergeLayer((l_forget, l_targets),
T.sub,
name=id_tag + 'broadcast_sub')
def apply_mask(layer_seq, layer_seq_mask):
"""
seq: layer of shape (batch_size, length_seq, n_features)
seq_mask: layer of shape (batch_size, length_seq)
"""
return ElemwiseMergeLayer(
[ForgetSizeLayer(dimshuffle(layer_seq_mask,
(0, 1, 'x'))), layer_seq], T.mul)
def add(*args, **kwargs):
"""Element-wise sum of layers"""
inp_names = [
layer.name or "layer" + str(i) for i, layer in enumerate(args)
]
kwargs["name"] = kwargs.get("name", "sum(%s)" % (', '.join(inp_names)))
return ElemwiseMergeLayer(args, T.add, **kwargs)
def cnn_fn(self):
l_in = InputLayer((None, self.max_length, self.vocab_size))
l_in_T = DimshuffleLayer(l_in, (0, 2, 1))
l_causal_conv = DilatedConv1DLayer(
l_in_T,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=tanh)
l_gate = DilatedConv1DLayer(l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_final = DimshuffleLayer(l_skip_sum, (0, 2, 1))
return l_final
def UNet_decoder_3(LR_conv1, LR_conv2, LR_conv3, LR_conv4, warp_conv1, warp_conv2, warp_conv3, warp_conv4):
# 80
mask4 = Conv2DLayer(ConcatLayer([LR_conv4, warp_conv4]), 64, 5, pad=2, W = W_init_SELU, b=Constant(0.), nonlinearity=sigmoid)
warp_conv4_m = ElemwiseMergeLayer([warp_conv4, mask4], T.mul)
warp_deconv4 = Deconv2DLayer(ConcatLayer([LR_conv4, warp_conv4_m]), num_filters=64, filter_size=4, stride=2, crop=1, W = W_init_SELU, b=Constant(0.), nonlinearity=SELU_activation)
# 160
mask3 = Conv2DLayer(ConcatLayer([warp_deconv4, LR_conv3, warp_conv3]), 64, 5, pad=2, W = W_init_SELU, b=Constant(0.), nonlinearity=sigmoid)
warp_conv3_m = ElemwiseMergeLayer([warp_conv3, mask3], T.mul)
warp_deconv3 = Deconv2DLayer(ConcatLayer([warp_deconv4, LR_conv3, warp_conv3_m]), num_filters=64, filter_size=4, stride=2, crop=1, W = W_init_SELU, b=Constant(0.), nonlinearity=SELU_activation)
# 320
mask2 = Conv2DLayer(ConcatLayer([warp_deconv3, LR_conv2, warp_conv2]), 64, 5, pad=2, W = W_init_SELU, b=Constant(0.), nonlinearity=sigmoid)
warp_conv2_m = ElemwiseMergeLayer([warp_conv2, mask2], T.mul)
warp_deconv2 = Deconv2DLayer(ConcatLayer([warp_deconv3, LR_conv2, warp_conv2_m]), num_filters=64, filter_size=4, stride=2, crop=1, W = W_init_SELU, b=Constant(0.), nonlinearity=SELU_activation)
# final
mask1 = Conv2DLayer(ConcatLayer([warp_deconv2, LR_conv1, warp_conv1]), 64, 5, pad=2, W = W_init_SELU, b=Constant(0.), nonlinearity=sigmoid)
warp_conv1_m = ElemwiseMergeLayer([warp_conv1, mask1], T.mul)
post_fusion1 = Conv2DLayer(ConcatLayer([warp_deconv2, LR_conv1, warp_conv1_m]), 64, 5, pad=2, W = W_init_SELU, b=Constant(0.), nonlinearity=SELU_activation)
post_fusion2 = Conv2DLayer(post_fusion1, 64, 5, pad=2, W = W_init_SELU, b=Constant(0.), nonlinearity=SELU_activation)
final = Conv2DLayer(post_fusion1, 3, 5, pad=2, W = W_init_linear, b=Constant(0.), nonlinearity=linear)
test = Conv2DLayer(final, 3, 5, pad=2, W = W_init_linear, b=Constant(0.), nonlinearity=linear)
return final
def build_test_model():
T_net = {}
T_net['input'] = InputLayer((None, 4, 224, 224))
#slice the input to get image and feat map part
T_net['input_map'] = SliceLayer(T_net['input'],
indices=slice(3, 4),
axis=1)
T_net['map112'] = PoolLayer(T_net['input_map'], 2)
T_net['map56'] = PoolLayer(T_net['map112'], 2)
T_net['map28'] = PoolLayer(T_net['map56'], 2)
T_net_buff56 = [T_net['map56'] for i in range(256)]
T_net['map56x256'] = concat(T_net_buff56)
T_net_buff28 = [T_net['map28'] for i in range(512)]
T_net['map28x512'] = concat(T_net_buff28)
T_net['input_im'] = SliceLayer(T_net['input'], indices=slice(0, 3), axis=1)
T_net['conv1_1'] = ConvLayer(T_net['input_im'],
64,
3,
pad=1,
flip_filters=False)
T_net['conv1_2'] = ConvLayer(T_net['conv1_1'],
64,
3,
pad=1,
flip_filters=False)
T_net['pool1'] = PoolLayer(T_net['conv1_2'], 2)
T_net['conv2_1'] = ConvLayer(T_net['pool1'],
128,
3,
pad=1,
flip_filters=False)
T_net['conv2_2'] = ConvLayer(T_net['conv2_1'],
128,
3,
pad=1,
flip_filters=False)
T_net['pool2'] = PoolLayer(T_net['conv2_2'], 2)
T_net['conv3_1'] = ConvLayer(T_net['pool2'],
256,
3,
pad=1,
flip_filters=False)
T_net['conv3_2'] = ConvLayer(T_net['conv3_1'],
256,
3,
pad=1,
flip_filters=False)
T_net['conv3_3'] = ConvLayer(T_net['conv3_2'],
256,
3,
pad=1,
flip_filters=False)
T_net['conv3_4'] = ConvLayer(T_net['conv3_3'],
256,
3,
pad=1,
flip_filters=False)
T_net['conv3_map'] = ElemwiseMergeLayer(
[T_net['conv3_1'], T_net['map56x256']], merge_function=T.mul)
T_net['conv3_all'] = ElemwiseSumLayer(
[T_net['conv3_4'], T_net['conv3_map']])
T_net['pool3'] = PoolLayer(T_net['conv3_all'], 2)
T_net['conv4_1'] = ConvLayer(T_net['pool3'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv4_2'] = ConvLayer(T_net['conv4_1'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv4_3'] = ConvLayer(T_net['conv4_2'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv4_4'] = ConvLayer(T_net['conv4_3'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv4_map'] = ElemwiseMergeLayer(
[T_net['conv4_1'], T_net['map28x512']], merge_function=T.mul)
T_net['conv4_all'] = ElemwiseSumLayer(
[T_net['conv4_4'], T_net['conv4_map']])
T_net['pool4'] = PoolLayer(T_net['conv4_all'], 2)
T_net['conv5_1'] = ConvLayer(T_net['pool4'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv5_2'] = ConvLayer(T_net['conv5_1'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv5_3'] = ConvLayer(T_net['conv5_2'],
512,
3,
pad=1,
flip_filters=False)
T_net['conv5_4'] = ConvLayer(T_net['conv5_3'],
512,
3,
pad=1,
flip_filters=False)
T_net['pool5'] = PoolLayer(T_net['conv5_4'], 2)
T_net['fc6'] = DenseLayer(T_net['pool5'], num_units=4096)
T_net['fc6_dropout'] = DropoutLayer(T_net['fc6'], p=0.)
T_net['fc7'] = DenseLayer(T_net['fc6_dropout'], num_units=4096)
T_net['fc7_dropout'] = DropoutLayer(T_net['fc7'], p=0.5)
T_net['fc8'] = DenseLayer(T_net['fc7_dropout'],
num_units=1000,
nonlinearity=None)
T_net['prob'] = NonlinearityLayer(T_net['fc8'], softmax)
# T_net['pos_fc_layer']=DenseLayer(T_net['fc6_dropout'],num_units=2048)
# T_net['pos_drop']=DropoutLayer(T_net['pos_fc_layer'],p=0.)
# T_net['pred_pos_layer']=DenseLayer(T_net['pos_drop'],num_units=40,nonlinearity=sigmoid)
#AU detection part
T_net['au_fc_layer'] = DenseLayer(T_net['fc6_dropout'], num_units=2048)
T_net['au_drop'] = DropoutLayer(T_net['au_fc_layer'], p=0.)
T_net['output_layer'] = DenseLayer(T_net['au_drop'],
num_units=12,
nonlinearity=sigmoid)
# T_net['final']=concat([T_net['pred_pos_layer'],T_net['output_layer']])
return T_net
def fused_convnets(self,
fusion_level,
fusion_type,
input_var1=None,
input_var2=None,
bottleneck_W=None,
weights_dir=None):
net = OrderedDict()
net['input_rgb'] = InputLayer((None, 4, 128, 128),
input_var=input_var1)
layer = 0
for i in range(fusion_level):
# Add convolution layers
net['conv_rgb{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2 and\
i != fusion_level - 1:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0 and i != fusion_level - 1:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7 and i != fusion_level - 1:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
net['input_depth'] = InputLayer((None, 1, 128, 128),
input_var=input_var2)
layer += 1
for i in range(fusion_level):
# Add convolution layers
net['conv_depth{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2 and\
i != fusion_level - 1:
# Add pooling layers
net['pool_depth{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0 and i != fusion_level - 1:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7 and i != fusion_level - 1:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
# Fuse ConvNets by fusion_level and fusion_type
if fusion_type == self.MAX:
net['merge'] =\
ElemwiseMergeLayer([net['conv_rgb{0:d}'.format(fusion_level)],
net['conv_depth{0:d}'.format(fusion_level)]
], T.maximum)
layer += 1
elif fusion_type == self.SUM:
net['merge'] =\
ElemwiseMergeLayer([net['conv_rgb{0:d}'.format(fusion_level)],
net['conv_depth{0:d}'.format(fusion_level)]
], T.add)
layer += 1
elif fusion_type == self.CONCAT:
net['merge'] = concat([
net['conv_rgb{0:d}'.format(fusion_level)],
net['conv_depth{0:d}'.format(fusion_level)]
])
layer += 1
elif fusion_type == self.CONCATCONV:
net['concat'] = concat([
net['conv_rgb{0:d}'.format(fusion_level)],
net['conv_depth{0:d}'.format(fusion_level)]
])
layer += 1
net['merge'] = Conv2DLayer(
net['concat'],
num_filters=self._net_specs_dict['num_conv_filters'][
fusion_level - 1],
filter_size=(1, 1),
nonlinearity=None)
layer += 1
# Max-pooling to the merged
if fusion_level in [2, 4, 7]:
net['pool_merged'] = MaxPool2DLayer(net['merge'], pool_size=(3, 3))
layer += 1
# Continue the rest of the convolutional part of the network,
# if the fusion took place before the last convolutional layer,
# else just connect the convolutional part with the fully connected
# part
if self._net_specs_dict['num_conv_layers'] > fusion_level:
for i in range(fusion_level,
self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_merged{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], )
* 2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_merged{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_merged{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_merged{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
# Add dropout layer
net['dropout1'] = dropout(net['fc1'], p=self._model_hp_dict['p'])
net['fc2'] = DenseLayer(net['dropout1'],
self._net_specs_dict['num_fc_units'][1])
# Add dropout layer
net['dropout2'] = dropout(net['fc2'], p=self._model_hp_dict['p'])
if bottleneck_W is not None:
# Add bottleneck layer
net['bottleneck'] = DenseLayer(net['dropout2'], 30)
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['bottleneck'],
3 * self._num_joints,
W=bottleneck_W[0:30],
nonlinearity=lasagne.nonlinearities.tanh)
else:
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['dropout2'],
3 * self._num_joints,
nonlinearity=lasagne.nonlinearities.tanh)
if weights_dir is not None:
lw = LoadWeights(weights_dir, net)
lw.load_weights_numpy()
return net
def dense_fused_convnets(self,
fusion_level,
fusion_type,
input_var1=None,
input_var2=None,
bottleneck_W=None,
weights_dir=None):
net = OrderedDict()
net['input_rgb'] = InputLayer((None, 4, 128, 128),
input_var=input_var1)
layer = 0
for i in range(self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_rgb{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_rgb{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1_rgb'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
layer += 1
if fusion_level == 2:
# Add dropout layer
net['dropout1_rgb'] = dropout(net['fc1_rgb'],
p=self._model_hp_dict['p'])
layer += 1
net['fc2_rgb'] = DenseLayer(
net['dropout1_rgb'], self._net_specs_dict['num_fc_units'][1])
layer += 1
net['input_depth'] = InputLayer((None, 1, 128, 128),
input_var=input_var2)
layer += 1
for i in range(self._net_specs_dict['num_conv_layers']):
# Add convolution layers
net['conv_depth{0:d}'.format(i + 1)] = Conv2DLayer(
net.values()[layer],
num_filters=self._net_specs_dict['num_conv_filters'][i],
filter_size=(self._net_specs_dict['conv_filter_size'][i], ) *
2,
pad='same')
layer += 1
if self._net_specs_dict['num_conv_layers'] <= 2:
# Add pooling layers
net['pool_depth{0:d}'.format(i + 1)] = MaxPool2DLayer(
net.values()[layer], pool_size=(3, 3))
layer += 1
else:
if i < 4:
if (i + 1) % 2 == 0:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
else:
if (i + 1) == 7:
# Add pooling layers
net['pool_depth{0:d}'.format(i+1)] =\
MaxPool2DLayer(net.values()[layer],
pool_size=(3, 3))
layer += 1
# Fc-layers
net['fc1_depth'] = DenseLayer(net.values()[layer],
self._net_specs_dict['num_fc_units'][0])
layer += 1
if fusion_level == 2:
# Add dropout layer
net['dropout1_depth'] = dropout(net['fc1_depth'],
p=self._model_hp_dict['p'])
layer += 1
net['fc2_depth'] = DenseLayer(
net['dropout1_depth'], self._net_specs_dict['num_fc_units'][1])
layer += 1
# Fuse ConvNets by fusion_level and fusion_type
if fusion_type == self.MAX:
net['merge'] =\
ElemwiseMergeLayer([net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]],
T.maximum)
layer += 1
elif fusion_type == self.SUM:
net['merge'] =\
ElemwiseMergeLayer([net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]],
T.add)
layer += 1
elif fusion_type == self.CONCAT:
net['merge'] = concat([
net['fc%i_rgb' % fusion_level],
net['fc%i_depth' % fusion_level]
])
layer += 1
elif fusion_type == self.CONCATCONV:
net['fc%i_rgb_res' % fusion_level] =\
reshape(net['fc%i_rgb' % fusion_level], ([0], 1, [1]))
layer += 1
net['fc%i_depth_res' % fusion_level] =\
reshape(net['fc%i_depth' % fusion_level], ([0], 1, [1]))
layer += 1
net['concat'] = concat([
net['fc%i_rgb_res' % fusion_level],
net['fc%i_depth_res' % fusion_level]
])
layer += 1
net['merge_con'] = Conv1DLayer(net['concat'],
num_filters=1,
filter_size=(1, ),
nonlinearity=None)
layer += 1
net['merge'] = reshape(net['merge_con'], ([0], [2]))
layer += 1
if fusion_level == 1:
# Add dropout layer
net['dropout1'] = dropout(net['merge'], p=self._model_hp_dict['p'])
layer += 1
net['fc2'] = DenseLayer(net['dropout1'],
self._net_specs_dict['num_fc_units'][1])
layer += 1
# Add dropout layer
net['dropout2'] = dropout(net['fc2'], p=self._model_hp_dict['p'])
layer += 1
else:
# Add dropout layer
net['dropout2'] = dropout(net['merge'], p=self._model_hp_dict['p'])
layer += 1
# Add output layer(linear activation because it's regression)
if bottleneck_W is not None:
# Add bottleneck layer
net['bottleneck'] = DenseLayer(net['dropout2'], 30)
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['bottleneck'],
3 * self._num_joints,
W=bottleneck_W[0:30],
nonlinearity=lasagne.nonlinearities.tanh)
else:
# Add output layer(linear activation because it's regression)
net['output'] = DenseLayer(
net['dropout2'],
3 * self._num_joints,
nonlinearity=lasagne.nonlinearities.tanh)
if weights_dir is not None:
lw = LoadWeights(weights_dir, net)
lw.load_weights_numpy()
return net
def __init__(self,
input_shape=(None, 3, None, None),
n_filters=48,
n_pool=4,
n_layers_per_block=5,
dropout_p=0.2):
"""
This code implements the Fully Convolutional DenseNet described in https://arxiv.org/abs/1611.09326
The network consist of a downsampling path, where dense blocks and transition down are applied, followed
by an upsampling path where transition up and dense blocks are applied.
Skip connections are used between the downsampling path and the upsampling path
Each layer is a composite function of BN - ReLU - Conv and the last layer is a softmax layer.
:param input_shape: shape of the input batch. Only the first dimension (n_channels) is needed
:param n_classes: number of classes
:param n_filters_first_conv: number of filters for the first convolution applied
:param n_pool: number of pooling layers = number of transition down = number of transition up
:param growth_rate: number of new feature maps created by each layer in a dense block
:param n_layers_per_block: number of layers per block. Can be an int or a list of size 2 * n_pool + 1
:param dropout_p: dropout rate applied after each convolution (0. for not using)
"""
if type(n_layers_per_block) == list:
assert (len(n_layers_per_block) == 2 * n_pool + 1)
elif type(n_layers_per_block) == int:
n_layers_per_block = [n_layers_per_block] * (2 * n_pool + 1)
else:
raise ValueError
# Theano variables
self.input_var = T.tensor4('input_var', dtype='float32') # input image
self.target_var = T.tensor4('target_var', dtype='float32') # target
#####################
# First Convolution #
#####################
inputs = InputLayer(input_shape, self.input_var)
# We perform a first convolution. All the features maps will be stored in the tensor called stack (the Tiramisu)
stack = Conv2DLayer(inputs,
n_filters[0],
filter_size=1,
pad='same',
W=HeUniform(gain='relu'),
nonlinearity=linear,
flip_filters=False)
#####################
# Downsampling path #
#####################
skip_connection_list = []
for i in range(n_pool):
# Dense Block
for j in range(n_layers_per_block[i]):
# Compute new feature maps
l = BN_ReLU_Conv(stack, n_filters[i], dropout_p=dropout_p)
# add new outputs
stack = ElemwiseMergeLayer([stack, l], T.add)
# At the end of the block, the current stack is stored in the skip_connections list
skip_connection_list.append(stack)
# Transition Down
stack = TransitionDown(stack, n_filters[i + 1], dropout_p)
skip_connection_list = skip_connection_list[::-1]
#####################
# Bottleneck #
#####################
# We store now the output of the next dense block in a list. We will only upsample these new feature maps
block_to_upsample = []
# Dense Block
for j in range(n_layers_per_block[n_pool]):
l = BN_ReLU_Conv(stack, n_filters[n_pool], dropout_p=dropout_p)
stack = ElemwiseMergeLayer([stack, l], T.add)
#######################
# Upsampling path #
#######################
for i in range(n_pool):
# Transition Up ( Upsampling + concatenation with the skip connection)
stack = TransitionUpRes(skip_connection_list[i],
stack,
n_filters[n_pool + i + 1],
dropout_p=dropout_p)
# Dense Block
block_to_upsample = []
for j in range(n_layers_per_block[n_pool + i + 1]):
l = BN_ReLU_Conv(stack,
n_filters[n_pool + i + 1],
dropout_p=dropout_p)
stack = ElemwiseMergeLayer([stack, l], T.add)
#####################
# Sigmoid #
#####################
self.output_layer = SpatialSoftmaxLayer(stack)
set4 = get_multiple_block(set3,num_filt=128, pooling_size=(8,8)) # Dense Layers follow. h_flat = flatten(set4) ## 5 Way Max-Out Layer (DenseMaxout) ''' Reference - https://github.com/fchollet/keras/pull/3128 ''' h_dense = [] for _ in xrange(5): h_dense.append( DenseLayer(h_flat,500,W = lasagne.init.GlorotUniform(), nonlinearity = lasagne.nonlinearities.linear)) h17 = ElemwiseMergeLayer( h_dense, merge_function=T.maximum()) h17 = BatchNormLayer(h17) h17_drop = dropout(h17,0.2) # Softmax Layer network = DenseLayer(h17_drop,nb_classes, nonlinearity = lasagne.nonlinearities.softmax, W = lasagne.init.GlorotUniform()) net_output = lasagne.layers.get_output(network) true_output = T.matrix() all_params = lasagne.layers.get_all_params(network,trainable=True) loss = T.mean(lasagne.objectives.categorical_crossentropy(net_output,true_output)) updates = lasagne.updates.adam(loss,all_params) train = theano.function(inputs= [l_in.input_var,true_output] , outputs=[net_output,loss], updates = updates)
def __init__(self, config):
self.clouds = T.tensor3(dtype='float32')
self.norms = [
T.tensor3(dtype='float32') for step in xrange(config['steps'])
]
self.target = T.vector(dtype='int64')
KDNet = {}
if config['input_features'] == 'no':
KDNet['input'] = InputLayer((None, 1, 2**config['steps']),
input_var=self.clouds)
else:
KDNet['input'] = InputLayer((None, 3, 2**config['steps']),
input_var=self.clouds)
for i in xrange(config['steps']):
KDNet['norm{}_r'.format(i + 1)] = InputLayer(
(None, 3, 2**(config['steps'] - 1 - i)),
input_var=self.norms[i])
KDNet['norm{}_l'.format(i + 1)] = ExpressionLayer(
KDNet['norm{}_r'.format(i + 1)], lambda X: -X)
KDNet['norm{}_l_X-'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_l'.format(i + 1)], '-', 0, config['n_f'][i + 1])
KDNet['norm{}_l_Y-'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_l'.format(i + 1)], '-', 1, config['n_f'][i + 1])
KDNet['norm{}_l_Z-'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_l'.format(i + 1)], '-', 2, config['n_f'][i + 1])
KDNet['norm{}_l_X+'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_l'.format(i + 1)], '+', 0, config['n_f'][i + 1])
KDNet['norm{}_l_Y+'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_l'.format(i + 1)], '+', 1, config['n_f'][i + 1])
KDNet['norm{}_l_Z+'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_l'.format(i + 1)], '+', 2, config['n_f'][i + 1])
KDNet['norm{}_r_X-'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_r'.format(i + 1)], '-', 0, config['n_f'][i + 1])
KDNet['norm{}_r_Y-'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_r'.format(i + 1)], '-', 1, config['n_f'][i + 1])
KDNet['norm{}_r_Z-'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_r'.format(i + 1)], '-', 2, config['n_f'][i + 1])
KDNet['norm{}_r_X+'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_r'.format(i + 1)], '+', 0, config['n_f'][i + 1])
KDNet['norm{}_r_Y+'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_r'.format(i + 1)], '+', 1, config['n_f'][i + 1])
KDNet['norm{}_r_Z+'.format(i + 1)] = SPTNormReshapeLayer(
KDNet['norm{}_r'.format(i + 1)], '+', 2, config['n_f'][i + 1])
KDNet['cloud{}'.format(i+1)] = SharedDotLayer(KDNet['input'], config['n_f'][i]) if i == 0 else \
ElemwiseSumLayer([KDNet['cloud{}_l_X-_masked'.format(i)],
KDNet['cloud{}_l_Y-_masked'.format(i)],
KDNet['cloud{}_l_Z-_masked'.format(i)],
KDNet['cloud{}_l_X+_masked'.format(i)],
KDNet['cloud{}_l_Y+_masked'.format(i)],
KDNet['cloud{}_l_Z+_masked'.format(i)],
KDNet['cloud{}_r_X-_masked'.format(i)],
KDNet['cloud{}_r_Y-_masked'.format(i)],
KDNet['cloud{}_r_Z-_masked'.format(i)],
KDNet['cloud{}_r_X+_masked'.format(i)],
KDNet['cloud{}_r_Y+_masked'.format(i)],
KDNet['cloud{}_r_Z+_masked'.format(i)]])
KDNet['cloud{}_bn'.format(i + 1)] = BatchNormDNNLayer(
KDNet['cloud{}'.format(i + 1)])
KDNet['cloud{}_relu'.format(i + 1)] = NonlinearityLayer(
KDNet['cloud{}_bn'.format(i + 1)], rectify)
KDNet['cloud{}_r'.format(i + 1)] = ExpressionLayer(
KDNet['cloud{}_relu'.format(i + 1)], lambda X: X[:, :, 1::2],
(None, config['n_f'][i], 2**(config['steps'] - i - 1)))
KDNet['cloud{}_l'.format(i + 1)] = ExpressionLayer(
KDNet['cloud{}_relu'.format(i + 1)], lambda X: X[:, :, ::2],
(None, config['n_f'][i], 2**(config['steps'] - i - 1)))
KDNet['cloud{}_l_X-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_l'.format(i + 1)], config['n_f'][i + 1])
KDNet['cloud{}_l_Y-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_l'.format(i + 1)], config['n_f'][i + 1])
KDNet['cloud{}_l_Z-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_l'.format(i + 1)], config['n_f'][i + 1])
KDNet['cloud{}_l_X+'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_l'.format(i + 1)], config['n_f'][i + 1])
KDNet['cloud{}_l_Y+'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_l'.format(i + 1)], config['n_f'][i + 1])
KDNet['cloud{}_l_Z+'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_l'.format(i + 1)], config['n_f'][i + 1])
KDNet['cloud{}_r_X-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_X-'.format(i + 1)].W,
b=KDNet['cloud{}_l_X-'.format(i + 1)].b)
KDNet['cloud{}_r_X-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_X-'.format(i + 1)].W,
b=KDNet['cloud{}_l_X-'.format(i + 1)].b)
KDNet['cloud{}_r_Y-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_Y-'.format(i + 1)].W,
b=KDNet['cloud{}_l_Y-'.format(i + 1)].b)
KDNet['cloud{}_r_Z-'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_Z-'.format(i + 1)].W,
b=KDNet['cloud{}_l_Z-'.format(i + 1)].b)
KDNet['cloud{}_r_X+'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_X+'.format(i + 1)].W,
b=KDNet['cloud{}_l_X+'.format(i + 1)].b)
KDNet['cloud{}_r_Y+'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_Y+'.format(i + 1)].W,
b=KDNet['cloud{}_l_Y+'.format(i + 1)].b)
KDNet['cloud{}_r_Z+'.format(i + 1)] = SharedDotLayer(
KDNet['cloud{}_r'.format(i + 1)],
config['n_f'][i + 1],
W=KDNet['cloud{}_l_Z+'.format(i + 1)].W,
b=KDNet['cloud{}_l_Z+'.format(i + 1)].b)
KDNet['cloud{}_l_X-_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_l_X-'.format(i + 1)],
KDNet['norm{}_l_X-'.format(i + 1)]
], T.mul)
KDNet['cloud{}_l_Y-_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_l_Y-'.format(i + 1)],
KDNet['norm{}_l_Y-'.format(i + 1)]
], T.mul)
KDNet['cloud{}_l_Z-_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_l_Z-'.format(i + 1)],
KDNet['norm{}_l_Z-'.format(i + 1)]
], T.mul)
KDNet['cloud{}_l_X+_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_l_X+'.format(i + 1)],
KDNet['norm{}_l_X+'.format(i + 1)]
], T.mul)
KDNet['cloud{}_l_Y+_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_l_Y+'.format(i + 1)],
KDNet['norm{}_l_Y+'.format(i + 1)]
], T.mul)
KDNet['cloud{}_l_Z+_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_l_Z+'.format(i + 1)],
KDNet['norm{}_l_Z+'.format(i + 1)]
], T.mul)
KDNet['cloud{}_r_X-_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_r_X-'.format(i + 1)],
KDNet['norm{}_r_X-'.format(i + 1)]
], T.mul)
KDNet['cloud{}_r_Y-_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_r_Y-'.format(i + 1)],
KDNet['norm{}_r_Y-'.format(i + 1)]
], T.mul)
KDNet['cloud{}_r_Z-_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_r_Z-'.format(i + 1)],
KDNet['norm{}_r_Z-'.format(i + 1)]
], T.mul)
KDNet['cloud{}_r_X+_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_r_X+'.format(i + 1)],
KDNet['norm{}_r_X+'.format(i + 1)]
], T.mul)
KDNet['cloud{}_r_Y+_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_r_Y+'.format(i + 1)],
KDNet['norm{}_r_Y+'.format(i + 1)]
], T.mul)
KDNet['cloud{}_r_Z+_masked'.format(i + 1)] = ElemwiseMergeLayer([
KDNet['cloud{}_r_Z+'.format(i + 1)],
KDNet['norm{}_r_Z+'.format(i + 1)]
], T.mul)
KDNet['cloud_fin'] = ElemwiseSumLayer([
KDNet['cloud{}_l_X-_masked'.format(config['steps'])],
KDNet['cloud{}_l_Y-_masked'.format(config['steps'])],
KDNet['cloud{}_l_Z-_masked'.format(config['steps'])],
KDNet['cloud{}_l_X+_masked'.format(config['steps'])],
KDNet['cloud{}_l_Y+_masked'.format(config['steps'])],
KDNet['cloud{}_l_Z+_masked'.format(config['steps'])],
KDNet['cloud{}_r_X-_masked'.format(config['steps'])],
KDNet['cloud{}_r_Y-_masked'.format(config['steps'])],
KDNet['cloud{}_r_Z-_masked'.format(config['steps'])],
KDNet['cloud{}_r_X+_masked'.format(config['steps'])],
KDNet['cloud{}_r_Y+_masked'.format(config['steps'])],
KDNet['cloud{}_r_Z+_masked'.format(config['steps'])]
])
KDNet['cloud_fin_bn'] = BatchNormDNNLayer(KDNet['cloud_fin'])
KDNet['cloud_fin_relu'] = NonlinearityLayer(KDNet['cloud_fin_bn'],
rectify)
KDNet['cloud_fin_reshape'] = ReshapeLayer(KDNet['cloud_fin_relu'],
(-1, config['n_f'][-1]))
KDNet['output'] = DenseLayer(KDNet['cloud_fin_reshape'],
config['num_classes'],
nonlinearity=softmax)
prob = get_output(KDNet['output'])
prob_det = get_output(KDNet['output'], deterministic=True)
weights = get_all_params(KDNet['output'], trainable=True)
l2_pen = regularize_network_params(KDNet['output'], l2)
loss = categorical_crossentropy(
prob, self.target).mean() + config['l2'] * l2_pen
accuracy = categorical_accuracy(prob, self.target).mean()
lr = theano.shared(np.float32(config['learning_rate']))
updates = adam(loss, weights, learning_rate=lr)
self.train_fun = theano.function([self.clouds] + self.norms +
[self.target], [loss, accuracy],
updates=updates)
self.prob_fun = theano.function([self.clouds] + self.norms +
[self.target], [loss, prob_det])
self.KDNet = KDNet
def build_model():
net = {}
net['input'] = InputLayer((None, 4, 224, 224))
#slice the input to get image and feat map part
net['input_map']=SliceLayer(net['input'],indices=slice(3,4),axis=1)
net['map112']=PoolLayer(net['input_map'],2)
net['map56']=PoolLayer(net['map112'],2)
net['map28']=PoolLayer(net['map56'],2)
net_buff56=[net['map56'] for i in range(256)]
net['map56x256']=concat(net_buff56)
net_buff28=[net['map28'] for i in range(512)]
net['map28x512']=concat(net_buff28)
net['input_im']=SliceLayer(net['input'],indices=slice(0,3),axis=1)
net['conv1_1'] = ConvLayer(
net['input_im'], 64, 3, pad=1, flip_filters=False,trainable=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad=1, flip_filters=False,trainable=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
net['conv2_1'] = ConvLayer(
net['pool1'], 128, 3, pad=1, flip_filters=False,trainable=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad=1, flip_filters=False,trainable=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
net['conv3_1'] = ConvLayer(
net['pool2'], 256, 3, pad=1, flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad=1, flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad=1, flip_filters=False)
net['conv3_4'] = ConvLayer(
net['conv3_3'], 256, 3, pad=1, flip_filters=False)
net['conv3_map']=ElemwiseMergeLayer([net['conv3_1'],net['map56x256']],merge_function=T.mul)
net['conv3_all']=ElemwiseSumLayer([net['conv3_4'],net['conv3_map']])
net['pool3'] = PoolLayer(net['conv3_all'], 2)
net['conv4_1'] = ConvLayer(
net['pool3'], 512, 3, pad=1, flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad=1, flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad=1, flip_filters=False)
net['conv4_4'] = ConvLayer(
net['conv4_3'], 512, 3, pad=1, flip_filters=False)
net['conv4_map']=ElemwiseMergeLayer([net['conv4_1'],net['map28x512']],merge_function=T.mul)
net['conv4_all']=ElemwiseSumLayer([net['conv4_4'],net['conv4_map']])
net['pool4'] = PoolLayer(net['conv4_all'], 2)
net['conv5_1'] = ConvLayer(
net['pool4'], 512, 3, pad=1, flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad=1, flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad=1, flip_filters=False)
net['conv5_4'] = ConvLayer(
net['conv5_3'], 512, 3, pad=1, flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_4'], 2)
net['fc6'] = DenseLayer(net['pool5'], num_units=4096)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=0.5)
net['fc7'] = DenseLayer(net['fc6_dropout'], num_units=4096)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['fc8'] = DenseLayer(net['fc7_dropout'], num_units=1000, nonlinearity=None)
net['prob'] = NonlinearityLayer(net['fc8'], softmax)
return net
def buildFCN8(nb_in_channels,
input_var,
path_weights='/Tmp/romerosa/itinf/models/' +
'camvid/fcn8_model.npz',
n_classes=21,
load_weights=False,
void_labels=[],
trainable=True,
layer=['probs_dimshuffle'],
pascal=False,
temperature=1.0):
'''
Build fcn8 model
'''
net = {}
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None), input_var)
# pool 1
net['conv1_1'] = ConvLayer(net['input'],
64,
3,
pad=100,
flip_filters=False)
net['conv1_2'] = ConvLayer(net['conv1_1'],
64,
3,
pad='same',
flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
# pool 2
net['conv2_1'] = ConvLayer(net['pool1'],
128,
3,
pad='same',
flip_filters=False)
net['conv2_2'] = ConvLayer(net['conv2_1'],
128,
3,
pad='same',
flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
# pool 3
net['conv3_1'] = ConvLayer(net['pool2'],
256,
3,
pad='same',
flip_filters=False)
net['conv3_2'] = ConvLayer(net['conv3_1'],
256,
3,
pad='same',
flip_filters=False)
net['conv3_3'] = ConvLayer(net['conv3_2'],
256,
3,
pad='same',
flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
# pool 4
net['conv4_1'] = ConvLayer(net['pool3'],
512,
3,
pad='same',
flip_filters=False)
net['conv4_2'] = ConvLayer(net['conv4_1'],
512,
3,
pad='same',
flip_filters=False)
net['conv4_3'] = ConvLayer(net['conv4_2'],
512,
3,
pad='same',
flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
# pool 5
net['conv5_1'] = ConvLayer(net['pool4'],
512,
3,
pad='same',
flip_filters=False)
net['conv5_2'] = ConvLayer(net['conv5_1'],
512,
3,
pad='same',
flip_filters=False)
net['conv5_3'] = ConvLayer(net['conv5_2'],
512,
3,
pad='same',
flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
# fc6
net['fc6'] = ConvLayer(net['pool5'],
4096,
7,
pad='valid',
flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'])
# fc7
net['fc7'] = ConvLayer(net['fc6_dropout'],
4096,
1,
pad='valid',
flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['score_fr'] = ConvLayer(net['fc7_dropout'],
n_classes,
1,
pad='valid',
flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'],
n_classes,
4,
stride=2,
crop='valid',
nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1, pad='same')
net['score_fused'] = ElemwiseSumLayer(
(net['score2'], net['score_pool4']),
cropping=[None, None, 'center', 'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'],
n_classes,
4,
stride=2,
crop='valid',
nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1, pad='valid')
net['score_final'] = ElemwiseSumLayer(
(net['score4'], net['score_pool3']),
cropping=[None, None, 'center', 'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'],
n_classes,
16,
stride=8,
crop='valid',
nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None, None),
input_var)
net['score'] = ElemwiseMergeLayer(
(net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv: deconv,
cropping=[None, None, 'center', 'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = \
lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = \
lasagne.layers.ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs'])
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(
net['probs'], (laySize[0], laySize[1], laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
# Apply temperature
if load_weights:
soft_value = net['upsample'].W.get_value() / temperature
net['upsample'].W.set_value(soft_value)
soft_value = net['upsample'].b.get_value() / temperature
net['upsample'].b.set_value(soft_value)
return [net[el] for el in layer]
def buildFCN8_DAE(input_concat_h_vars, input_mask_var, n_classes, nb_in_channels=3,
path_weights='/Tmp/romerosa/itinf/models/',
model_name='fcn8_model.npz', trainable=False,
load_weights=False, pretrained=False, freeze=False,
pretrained_path='/data/lisatmp4/romerosa/itinf/models/camvid/',
pascal=False, return_layer='probs_dimshuffle',
concat_h=['input'], noise=0.1, dropout=0.5):
'''
Build fcn8 model as DAE
'''
net = {}
pos = 0
assert all([el in ['pool1', 'pool2', 'pool3', 'pool4', 'input']
for el in concat_h])
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None),
input_mask_var)
# Add noise
# Noise
if noise > 0:
# net['noisy_input'] = GaussianNoiseLayerSoftmax(net['input'],
# sigma=noise)
net['noisy_input'] = GaussianNoiseLayer(net['input'], sigma=noise)
in_layer = 'noisy_input'
else:
in_layer = 'input'
pos, out = model_helpers.concatenate(net, in_layer, concat_h,
input_concat_h_vars, pos,
net['input'].output_shape[1])
# pool 1
net['conv1_1'] = ConvLayer(
net[out], 64, 3, pad=100, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad='same', flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
pos, out = model_helpers.concatenate(net, 'pool1', concat_h,
input_concat_h_vars, pos,
net['pool1'].output_shape[1])
# pool 2
net['conv2_1'] = ConvLayer(
net[out], 128, 3, pad='same', flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad='same', flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
pos, out = model_helpers.concatenate(net, 'pool2', concat_h,
input_concat_h_vars, pos,
net['pool2'].output_shape[1])
# pool 3
net['conv3_1'] = ConvLayer(
net[out], 256, 3, pad='same', flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad='same', flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad='same', flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
pos, out = model_helpers.concatenate(net, 'pool3', concat_h,
input_concat_h_vars, pos,
net['pool3'].output_shape[1])
# pool 4
net['conv4_1'] = ConvLayer(
net[out], 512, 3, pad='same', flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad='same', flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad='same', flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
pos, out = model_helpers.concatenate(net, 'pool4', concat_h,
input_concat_h_vars, pos,
net['pool4'].output_shape[1])
# pool 5
net['conv5_1'] = ConvLayer(
net[out], 512, 3, pad='same', flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad='same', flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad='same', flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
pos, out = model_helpers.concatenate(net, 'pool5', concat_h,
input_concat_h_vars, pos,
net['pool5'].output_shape[1])
# fc6
net['fc6'] = ConvLayer(
net[out], 4096, 7, pad='valid', flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=dropout)
# fc7
net['fc7'] = ConvLayer(
net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=dropout)
net['score_fr'] = ConvLayer(
net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2,
crop='valid', nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,
pad='same')
net['score_fused'] = ElemwiseSumLayer((net['score2'],
net['score_pool4']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4,
stride=2, crop='valid', nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,
pad='valid')
net['score_final'] = ElemwiseSumLayer((net['score4'],
net['score_pool3']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16,
stride=8, crop='valid', nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None,
None), input_mask_var)
net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv:
deconv,
cropping=[None, None, 'center',
'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = \
lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = \
lasagne.layers.ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Load weights
if load_weights:
pretrained = False
with np.load(os.path.join(path_weights, model_name)) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs'], param_values)
# In case we want to re-use the weights of an FCN8 model pretrained from images (not GT)
if pretrained:
print 'Loading pretrained weights'
if pascal:
path_weights = '/data/lisatmp4/romerosa/itinf/models/camvid/pascal-fcn8s-tvg-dag.mat'
if 'tvg' in path_weights:
str_filter = 'f'
str_bias = 'b'
else:
str_filter = '_filter'
str_bias = '_bias'
W = sio.loadmat(path_weights)
# Load the parameter values into the net
num_params = W.get('params').shape[1]
str_ind = [''.join(x for x in concat if x.isdigit()) for concat in concat_h]
list_of_lays = ['conv' + str(int(x)+1) + '_1' for x in str_ind if x]
list_of_lays += ['conv1_1'] if nb_in_channels != 3 or 'input' in concat_h else []
print list_of_lays
for i in range(num_params):
# Get layer name from the saved model
name = str(W.get('params')[0][i][0])[3:-2]
# Get parameter value
param_value = W.get('params')[0][i][1]
# Load weights
if name.endswith(str_filter):
raw_name = name[:-len(str_filter)]
if raw_name not in list_of_lays:
print 'Copying weights for ' + raw_name
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
# print 'Initializing layer ' + raw_name
param_value = param_value.T
param_value = np.swapaxes(param_value, 2, 3)
net[raw_name].W.set_value(param_value)
else:
print 'Ignoring ' + raw_name
# Load bias terms
if name.endswith(str_bias):
raw_name = name[:-len(str_bias)]
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
param_value = np.squeeze(param_value)
net[raw_name].b.set_value(param_value)
else:
with np.load(os.path.join(pretrained_path, model_name)) as f:
start = 0 if nb_in_channels == f['arr_%d' % 0].shape[1] \
else 2
param_values = [f['arr_%d' % i] for i in range(start,
len(f.files))]
all_layers = lasagne.layers.get_all_layers(net['probs'])
all_layers = [l for l in all_layers if (not isinstance(l, InputLayer) and not isinstance(l, GaussianNoiseLayerSoftmax) and not isinstance(l,GaussianNoiseLayer))]
all_layers = all_layers[1:] if start > 0 else all_layers
# Freeze parameters after last concatenation layer
last_concat = [idx for idx,l in enumerate(all_layers) if isinstance(l,ConcatLayer)][-1]
count = 0
for ixd, layer in enumerate(all_layers):
layer_params = layer.get_params()
for p in layer_params:
if hasattr(layer, 'input_layer') and not isinstance(layer.input_layer, ConcatLayer):
p.set_value(param_values[count])
if freeze:
model_helpers.freezeParameters(layer, single=True)
if isinstance(layer.input_layer, ConcatLayer) and idx == last_concat:
print('freezing')
freeze = True
count += 1
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs'])
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(net['probs'], (laySize[0], laySize[1],
laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
return net[return_layer]
def _build_disc(self):
inputs = OrderedDict()
inputs['x'] = InputLayer((None, 4, 64, 64))
inputs['c'] = InputLayer((None, 843))
inputs['v'] = InputLayer((None, 4))
inputs['t'] = InputLayer((None, 8))
layer_c = inputs['c']
layer_c = DenseLayer(layer_c, 512, nonlinearity=leaky_rectify)
layer_c.params[layer_c.W].add('dense')
layer_c = (DenseLayer(layer_c, 512, nonlinearity=leaky_rectify))
layer_c.params[layer_c.W].add('dense')
layer_v = inputs['v']
layer_v = DenseLayer(layer_v, 512, nonlinearity=leaky_rectify)
layer_v.params[layer_v.W].add('dense')
layer_v = (DenseLayer(layer_v, 512, nonlinearity=leaky_rectify))
layer_v.params[layer_v.W].add('dense')
layer_t = inputs['t']
layer_t = DenseLayer(layer_t, 512, nonlinearity=leaky_rectify)
layer_t.params[layer_t.W].add('dense')
layer_t = (DenseLayer(layer_t, 512, nonlinearity=leaky_rectify))
layer_t.params[layer_t.W].add('dense')
layer_i = ConcatLayer([layer_c, layer_v, layer_t])
layer_i = DenseLayer(layer_i, 1024, nonlinearity=leaky_rectify)
layer_i.params[layer_i.W].add('dense')
layer_i = DenseLayer(layer_i, 1024, nonlinearity=None)
layer_i.params[layer_i.W].add('dense')
layer_i = NonlinearityLayer(
layer_i,
lambda x: x / T.sqrt(T.sum(T.square(x), axis=1, keepdims=True)))
layer_x = inputs['x']
layer_x_n = layer_x
layer_x = weight_norm(
Conv2DLayer(layer_x_n, 64, 5, 2, 'same', nonlinearity=None,
b=None))
if self.reg: layer_x = dropout(layer_x)
layer_x = NonlinearityLayer(layer_x, leaky_rectify)
layer_x = weight_norm(
Conv2DLayer(layer_x, 64, 5, 2, 'same', nonlinearity=None, b=None))
if self.reg: layer_x = dropout(layer_x)
layer_x = NonlinearityLayer(layer_x, leaky_rectify)
layer_x = weight_norm(
Conv2DLayer(layer_x, 128, 5, 2, 'same', nonlinearity=None, b=None))
if self.reg: layer_x = dropout(layer_x)
layer_x = NonlinearityLayer(layer_x, leaky_rectify)
layer_x = weight_norm(
Conv2DLayer(layer_x, 256, 5, 2, 'same', nonlinearity=None, b=None))
layer_x = NonlinearityLayer(layer_x, leaky_rectify)
layer_x = FlattenLayer(layer_x)
layer_x = DenseLayer(layer_x, 1024, nonlinearity=leaky_rectify)
layer_x.params[layer_x.W].add('dense')
layer_x = DenseLayer(layer_x, 1024, nonlinearity=None)
layer_x.params[layer_x.W].add('dense')
layer_x = NonlinearityLayer(
layer_x,
lambda x: x / T.sqrt(T.sum(T.square(x), axis=1, keepdims=True)))
layer = ElemwiseMergeLayer([layer_i, layer_x], T.mul)
layer = ConcatLayer([layer, layer_x, layer_i])
layer = DenseLayer(layer, 1024, nonlinearity=leaky_rectify)
layer.params[layer.W].add('dense')
layer_r = DenseLayer(layer, 1024, nonlinearity=leaky_rectify)
layer_r.params[layer_r.W].add('dense')
layer_r = DenseLayer(layer_r, 1, nonlinearity=None)
layer_r.params[layer_r.W].add('dense')
layer_r_0 = NonlinearityLayer(layer_r, nonlinearity=sigmoid)
layer_r_1 = NonlinearityLayer(
layer_r, nonlinearity=lambda x: x - T.log(1 + T.exp(x)))
layer_r_2 = NonlinearityLayer(
layer_r, nonlinearity=lambda x: -T.log(1 + T.exp(x)))
layer_s = DenseLayer(layer, 1024, nonlinearity=leaky_rectify)
layer_s.params[layer_s.W].add('dense')
layer_s = DenseLayer(layer_s, 1, nonlinearity=None)
layer_s.params[layer_s.W].add('dense')
layer_s_0 = NonlinearityLayer(layer_s, nonlinearity=sigmoid)
layer_s_1 = NonlinearityLayer(
layer_s, nonlinearity=lambda x: x - T.log(1 + T.exp(x)))
layer_s_2 = NonlinearityLayer(
layer_s, nonlinearity=lambda x: -T.log(1 + T.exp(x)))
outputs = OrderedDict()
outputs['s'] = layer_s_0
outputs['log(s)'] = layer_s_1
outputs['log(1-s)'] = layer_s_2
outputs['r'] = layer_r_0
outputs['log(r)'] = layer_r_1
outputs['log(1-r)'] = layer_r_2
self.disc_inputs = inputs
self.disc_outputs = outputs
def network_convpool_cnn_max(input_vars,
nb_classes,
imsize=[32, 32],
n_colors=3,
n_timewin=5,
n_layers=(4, 2),
n_filters_first=32,
dense_num_unit=[512, 512],
shared_weights=True,
batch_norm_dense=False,
batch_norm_conv=False):
"""
Builds the complete network with maxpooling layer in time.
:param input_vars: list of EEG images (one image per time window)
:param nb_classes: number of classes
:param imsize: size of the input image (assumes a square input)
:param n_colors: number of color channels in the image
:param n_timewin: number of time windows in the snippet
:return: a pointer to the output of last layer
"""
convnets = []
w_init = None
# Build 5 parallel CNNs with shared weights
for i in range(n_timewin):
if i == 0:
convnet, w_init = build_cnn(input_vars[i],
imsize=imsize,
n_colors=n_colors,
n_filters_first=n_filters_first,
n_layers=n_layers,
batch_norm_conv=batch_norm_conv)
if not shared_weights:
w_init = None
else:
convnet, _ = build_cnn(input_vars[i],
w_init=w_init,
imsize=imsize,
n_colors=n_colors,
n_filters_first=n_filters_first,
n_layers=n_layers,
batch_norm_conv=batch_norm_conv)
convnets.append(FlattenLayer(convnet))
# convpooling using Max pooling over frames
convpool = ElemwiseMergeLayer(convnets, theano.tensor.maximum)
# A fully-connected layer of 256 units with 50% dropout on its inputs:
for i in range(len(dense_num_unit)):
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=dense_num_unit[i],
nonlinearity=lasagne.nonlinearities.rectify)
if batch_norm_dense:
convpool = batch_norm(convpool)
# And, finally, the 2-unit output layer with 50% dropout on its inputs:
if nb_classes == 1:
nonlinearity = lasagne.nonlinearities.sigmoid
else:
nonlinearity = lasagne.nonlinearities.softmax
convpool = DenseLayer(lasagne.layers.dropout(convpool, p=.5),
num_units=nb_classes,
nonlinearity=nonlinearity)
return convpool
def nn_fn(self):
l_in_x = InputLayer((None, self.embedding_dim, self.max_length))
l_in_z = InputLayer((None, self.z_dim))
l_causal_conv = DilatedConv1DLayer(
l_in_x,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_x_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=None,
b=None)
l_z_filter = DenseLayer(l_in_z,
num_units=self.nn_dilation_channels,
nonlinearity=None,
b=None)
l_z_filter_reshape = ReshapeLayer(l_z_filter, (
[0],
[1],
1,
))
l_z_filter_rep = RepeatLayer(l_z_filter_reshape,
self.max_length,
axis=-1,
ndim=3)
l_filter = NonlinearityLayer(ElemwiseSumLayer(
[l_x_filter, l_z_filter_rep]),
nonlinearity=tanh)
l_x_gate = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=None,
b=None)
l_z_gate = DenseLayer(l_in_z,
num_units=self.nn_dilation_channels,
nonlinearity=None,
b=None)
l_z_gate_reshape = ReshapeLayer(l_z_gate, (
[0],
[1],
1,
))
l_z_gate_rep = RepeatLayer(l_z_gate_reshape,
self.max_length,
axis=-1,
ndim=3)
l_gate = NonlinearityLayer(ElemwiseSumLayer(
[l_x_gate, l_z_gate_rep]),
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None,
b=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.embedding_dim,
filter_size=1,
nonlinearity=None,
b=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_prev = l_skip_sum
for h in range(2):
l_h = Conv1DLayer(l_prev,
num_filters=self.embedding_dim,
filter_size=1,
nonlinearity=None,
b=None)
l_z = DenseLayer(l_in_z,
num_units=self.embedding_dim,
nonlinearity=None,
b=None)
l_z_reshape = ReshapeLayer(l_z, (
[0],
[1],
1,
))
l_z_reshape_rep = RepeatLayer(l_z_reshape,
self.max_length,
axis=-1,
ndim=3)
l_sum = NonlinearityLayer(ElemwiseSumLayer([l_h, l_z_reshape_rep]),
nonlinearity=elu)
l_prev = l_sum
l_out = DimshuffleLayer(l_prev, (0, 2, 1))
return (l_in_x, l_in_z), l_out
def buildFCN8(nb_in_channels, input_var,
path_weights='/Tmp/romerosa/itinf/models/' +
'camvid/fcn8_model.npz',
n_classes=21, load_weights=True,
void_labels=[], trainable=False,
layer=['probs_dimshuffle']):
'''
Build fcn8 model (generator)
'''
net = {}
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None),
input_var)
# pool 1
net['conv1_1'] = ConvLayer(
net['input'], 64, 3, pad=100, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad='same', flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
# pool 2
net['conv2_1'] = ConvLayer(
net['pool1'], 128, 3, pad='same', flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad='same', flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
# pool 3
net['conv3_1'] = ConvLayer(
net['pool2'], 256, 3, pad='same', flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad='same', flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad='same', flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
# pool 4
net['conv4_1'] = ConvLayer(
net['pool3'], 512, 3, pad='same', flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad='same', flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad='same', flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
# pool 5
net['conv5_1'] = ConvLayer(
net['pool4'], 512, 3, pad='same', flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad='same', flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad='same', flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
# fc6
net['fc6'] = ConvLayer(
net['pool5'], 4096, 7, pad='valid', flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'])
# fc7
net['fc7'] = ConvLayer(
net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['score_fr'] = ConvLayer(
net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2,
crop='valid', nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,
pad='same')
net['score_fused'] = ElemwiseSumLayer((net['score2'],
net['score_pool4']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4,
stride=2, crop='valid', nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,
pad='valid')
net['score_final'] = ElemwiseSumLayer((net['score4'],
net['score_pool3']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16,
stride=8, crop='valid', nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None,
None), input_var)
net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv:
deconv,
cropping=[None, None, 'center',
'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Load weights
if load_weights:
with np.load(path_weights) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs'], param_values)
if not trainable:
freezeParameters(net['probs'], single=False)
if any(void_labels):
layVoid = lasagne.layers.get_output(net['probs']).shape
input_discrim_var = T.zeros((layVoid[0], 1))
net['input_void'] = InputLayer((None, 1), input_discrim_var)
net['concat'] = ConcatLayer([net['probs'], net['input_void']],
axis=1, cropping=None)
n_classes = n_classes + 1
else:
net['concat'] = net['probs']
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(net['concat'], (laySize[0], laySize[1],
laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
return [net[el] for el in layer]
def buildFCN8(nb_in_channels, input_var,
path_weights='/Tmp/romerosa/itinf/models/' +
'camvid/new_fcn8_model_best.npz',
n_classes=21, load_weights=True,
void_labels=[], trainable=False,
layer=['probs_dimshuffle'], pascal=False,
temperature=1.0, dropout=0.5):
'''
Build fcn8 model
'''
net = {}
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None),
input_var)
# pool 1
net['conv1_1'] = ConvLayer(
net['input'], 64, 3, pad=100, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad='same', flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
# pool 2
net['conv2_1'] = ConvLayer(
net['pool1'], 128, 3, pad='same', flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad='same', flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
# pool 3
net['conv3_1'] = ConvLayer(
net['pool2'], 256, 3, pad='same', flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad='same', flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad='same', flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
# pool 4
net['conv4_1'] = ConvLayer(
net['pool3'], 512, 3, pad='same', flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad='same', flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad='same', flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
# pool 5
net['conv5_1'] = ConvLayer(
net['pool4'], 512, 3, pad='same', flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad='same', flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad='same', flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
# fc6
net['fc6'] = ConvLayer(
net['pool5'], 4096, 7, pad='valid', flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=dropout)
# fc7
net['fc7'] = ConvLayer(
net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=dropout)
net['score_fr'] = ConvLayer(
net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2,
crop='valid', nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,
pad='same')
net['score_fused'] = ElemwiseSumLayer((net['score2'],
net['score_pool4']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4,
stride=2, crop='valid', nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,
pad='valid')
net['score_final'] = ElemwiseSumLayer((net['score4'],
net['score_pool3']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16,
stride=8, crop='valid', nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None,
None), input_var)
net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv:
deconv,
cropping=[None, None, 'center',
'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = \
lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = \
lasagne.layers.ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Load weights
if load_weights:
if pascal:
path_weights = '/data/lisatmp4/erraqabi/data/att-segm/' + \
'pre_trained_weights/pascal-fcn8s-tvg-dag.mat'
if 'tvg' in path_weights:
str_filter = 'f'
str_bias = 'b'
else:
str_filter = '_filter'
str_bias = '_bias'
W = sio.loadmat(path_weights)
# Load the parameter values into the net
num_params = W.get('params').shape[1]
for i in range(num_params):
# Get layer name from the saved model
name = str(W.get('params')[0][i][0])[3:-2]
# Get parameter value
param_value = W.get('params')[0][i][1]
# Load weights
if name.endswith(str_filter):
raw_name = name[:-len(str_filter)]
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
# print 'Initializing layer ' + raw_name
param_value = param_value.T
param_value = np.swapaxes(param_value, 2, 3)
net[raw_name].W.set_value(param_value)
# Load bias terms
if name.endswith(str_bias):
raw_name = name[:-len(str_bias)]
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
param_value = np.squeeze(param_value)
net[raw_name].b.set_value(param_value)
else:
with np.load(path_weights) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs'], param_values)
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs'], single=False)
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(net['probs'], (laySize[0], laySize[1],
laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
# Apply temperature
if load_weights:
soft_value = net['upsample'].W.get_value() / temperature
net['upsample'].W.set_value(soft_value)
soft_value = net['upsample'].b.get_value() / temperature
net['upsample'].b.set_value(soft_value)
return [net[el] for el in layer]
def cnn_fn(self, max_length):
"""Build the theano tensor
Using the attributes of the class, build the theano graph
using lasagne to get the neural networks which can
then be passed into theano.function to get the values
given input.
:param max_len: (int) max length of the language we are using
:return l_final: (tensor) theano tensor specifying the output of the cnn"""
l_in = InputLayer((None, max_length, self.vocab_size))
l_in_T = DimshuffleLayer(l_in, (0, 2, 1))
l_causal_conv = DilatedConv1DLayer(
l_in_T,
num_filters=self.nn_residual_channels,
dilation=1,
nonlinearity=None)
l_prev = l_causal_conv
skip_layers = []
for h in range(len(self.nn_dilations)):
l_filter = DilatedConv1DLayer(
l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=tanh)
l_gate = DilatedConv1DLayer(l_prev,
num_filters=self.nn_dilation_channels,
dilation=self.nn_dilations[h],
nonlinearity=sigmoid)
l_merge = ElemwiseMergeLayer([l_filter, l_gate],
merge_function=T.mul)
l_dense = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
l_residual = ElemwiseSumLayer([l_prev, l_dense])
l_skip = Conv1DLayer(l_merge,
num_filters=self.nn_residual_channels,
filter_size=1,
nonlinearity=None)
skip_layers.append(l_skip)
l_prev = l_residual
l_skip_sum = NonlinearityLayer(ElemwiseSumLayer(skip_layers),
nonlinearity=elu)
l_final = DimshuffleLayer(l_skip_sum, (0, 2, 1))
return l_final