def buildFCN8_DAE(input_concat_h_vars, input_mask_var, n_classes, nb_in_channels=3,
path_weights='/Tmp/romerosa/itinf/models/',
model_name='fcn8_model.npz', trainable=False,
load_weights=False, pretrained=False, freeze=False,
pretrained_path='/data/lisatmp4/romerosa/itinf/models/camvid/',
pascal=False, return_layer='probs_dimshuffle',
concat_h=['input'], noise=0.1, dropout=0.5):
'''
Build fcn8 model as DAE
'''
net = {}
pos = 0
assert all([el in ['pool1', 'pool2', 'pool3', 'pool4', 'input']
for el in concat_h])
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None),
input_mask_var)
# Add noise
# Noise
if noise > 0:
# net['noisy_input'] = GaussianNoiseLayerSoftmax(net['input'],
# sigma=noise)
net['noisy_input'] = GaussianNoiseLayer(net['input'], sigma=noise)
in_layer = 'noisy_input'
else:
in_layer = 'input'
pos, out = model_helpers.concatenate(net, in_layer, concat_h,
input_concat_h_vars, pos,
net['input'].output_shape[1])
# pool 1
net['conv1_1'] = ConvLayer(
net[out], 64, 3, pad=100, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad='same', flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
pos, out = model_helpers.concatenate(net, 'pool1', concat_h,
input_concat_h_vars, pos,
net['pool1'].output_shape[1])
# pool 2
net['conv2_1'] = ConvLayer(
net[out], 128, 3, pad='same', flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad='same', flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
pos, out = model_helpers.concatenate(net, 'pool2', concat_h,
input_concat_h_vars, pos,
net['pool2'].output_shape[1])
# pool 3
net['conv3_1'] = ConvLayer(
net[out], 256, 3, pad='same', flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad='same', flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad='same', flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
pos, out = model_helpers.concatenate(net, 'pool3', concat_h,
input_concat_h_vars, pos,
net['pool3'].output_shape[1])
# pool 4
net['conv4_1'] = ConvLayer(
net[out], 512, 3, pad='same', flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad='same', flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad='same', flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
pos, out = model_helpers.concatenate(net, 'pool4', concat_h,
input_concat_h_vars, pos,
net['pool4'].output_shape[1])
# pool 5
net['conv5_1'] = ConvLayer(
net[out], 512, 3, pad='same', flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad='same', flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad='same', flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
pos, out = model_helpers.concatenate(net, 'pool5', concat_h,
input_concat_h_vars, pos,
net['pool5'].output_shape[1])
# fc6
net['fc6'] = ConvLayer(
net[out], 4096, 7, pad='valid', flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=dropout)
# fc7
net['fc7'] = ConvLayer(
net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=dropout)
net['score_fr'] = ConvLayer(
net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2,
crop='valid', nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,
pad='same')
net['score_fused'] = ElemwiseSumLayer((net['score2'],
net['score_pool4']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4,
stride=2, crop='valid', nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,
pad='valid')
net['score_final'] = ElemwiseSumLayer((net['score4'],
net['score_pool3']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16,
stride=8, crop='valid', nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None,
None), input_mask_var)
net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv:
deconv,
cropping=[None, None, 'center',
'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = \
lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = \
lasagne.layers.ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Load weights
if load_weights:
pretrained = False
with np.load(os.path.join(path_weights, model_name)) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs'], param_values)
# In case we want to re-use the weights of an FCN8 model pretrained from images (not GT)
if pretrained:
print 'Loading pretrained weights'
if pascal:
path_weights = '/data/lisatmp4/romerosa/itinf/models/camvid/pascal-fcn8s-tvg-dag.mat'
if 'tvg' in path_weights:
str_filter = 'f'
str_bias = 'b'
else:
str_filter = '_filter'
str_bias = '_bias'
W = sio.loadmat(path_weights)
# Load the parameter values into the net
num_params = W.get('params').shape[1]
str_ind = [''.join(x for x in concat if x.isdigit()) for concat in concat_h]
list_of_lays = ['conv' + str(int(x)+1) + '_1' for x in str_ind if x]
list_of_lays += ['conv1_1'] if nb_in_channels != 3 or 'input' in concat_h else []
print list_of_lays
for i in range(num_params):
# Get layer name from the saved model
name = str(W.get('params')[0][i][0])[3:-2]
# Get parameter value
param_value = W.get('params')[0][i][1]
# Load weights
if name.endswith(str_filter):
raw_name = name[:-len(str_filter)]
if raw_name not in list_of_lays:
print 'Copying weights for ' + raw_name
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
# print 'Initializing layer ' + raw_name
param_value = param_value.T
param_value = np.swapaxes(param_value, 2, 3)
net[raw_name].W.set_value(param_value)
else:
print 'Ignoring ' + raw_name
# Load bias terms
if name.endswith(str_bias):
raw_name = name[:-len(str_bias)]
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
param_value = np.squeeze(param_value)
net[raw_name].b.set_value(param_value)
else:
with np.load(os.path.join(pretrained_path, model_name)) as f:
start = 0 if nb_in_channels == f['arr_%d' % 0].shape[1] \
else 2
param_values = [f['arr_%d' % i] for i in range(start,
len(f.files))]
all_layers = lasagne.layers.get_all_layers(net['probs'])
all_layers = [l for l in all_layers if (not isinstance(l, InputLayer) and not isinstance(l, GaussianNoiseLayerSoftmax) and not isinstance(l,GaussianNoiseLayer))]
all_layers = all_layers[1:] if start > 0 else all_layers
# Freeze parameters after last concatenation layer
last_concat = [idx for idx,l in enumerate(all_layers) if isinstance(l,ConcatLayer)][-1]
count = 0
for ixd, layer in enumerate(all_layers):
layer_params = layer.get_params()
for p in layer_params:
if hasattr(layer, 'input_layer') and not isinstance(layer.input_layer, ConcatLayer):
p.set_value(param_values[count])
if freeze:
model_helpers.freezeParameters(layer, single=True)
if isinstance(layer.input_layer, ConcatLayer) and idx == last_concat:
print('freezing')
freeze = True
count += 1
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs'])
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(net['probs'], (laySize[0], laySize[1],
laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
return net[return_layer]
def buildFCN8(nb_in_channels,
input_var,
path_weights='/Tmp/romerosa/itinf/models/' +
'camvid/fcn8_model.npz',
n_classes=21,
load_weights=False,
void_labels=[],
trainable=True,
layer=['probs_dimshuffle'],
pascal=False,
temperature=1.0):
'''
Build fcn8 model
'''
net = {}
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None), input_var)
# pool 1
net['conv1_1'] = ConvLayer(net['input'],
64,
3,
pad=100,
flip_filters=False)
net['conv1_2'] = ConvLayer(net['conv1_1'],
64,
3,
pad='same',
flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
# pool 2
net['conv2_1'] = ConvLayer(net['pool1'],
128,
3,
pad='same',
flip_filters=False)
net['conv2_2'] = ConvLayer(net['conv2_1'],
128,
3,
pad='same',
flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
# pool 3
net['conv3_1'] = ConvLayer(net['pool2'],
256,
3,
pad='same',
flip_filters=False)
net['conv3_2'] = ConvLayer(net['conv3_1'],
256,
3,
pad='same',
flip_filters=False)
net['conv3_3'] = ConvLayer(net['conv3_2'],
256,
3,
pad='same',
flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
# pool 4
net['conv4_1'] = ConvLayer(net['pool3'],
512,
3,
pad='same',
flip_filters=False)
net['conv4_2'] = ConvLayer(net['conv4_1'],
512,
3,
pad='same',
flip_filters=False)
net['conv4_3'] = ConvLayer(net['conv4_2'],
512,
3,
pad='same',
flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
# pool 5
net['conv5_1'] = ConvLayer(net['pool4'],
512,
3,
pad='same',
flip_filters=False)
net['conv5_2'] = ConvLayer(net['conv5_1'],
512,
3,
pad='same',
flip_filters=False)
net['conv5_3'] = ConvLayer(net['conv5_2'],
512,
3,
pad='same',
flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
# fc6
net['fc6'] = ConvLayer(net['pool5'],
4096,
7,
pad='valid',
flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'])
# fc7
net['fc7'] = ConvLayer(net['fc6_dropout'],
4096,
1,
pad='valid',
flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['score_fr'] = ConvLayer(net['fc7_dropout'],
n_classes,
1,
pad='valid',
flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'],
n_classes,
4,
stride=2,
crop='valid',
nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1, pad='same')
net['score_fused'] = ElemwiseSumLayer(
(net['score2'], net['score_pool4']),
cropping=[None, None, 'center', 'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'],
n_classes,
4,
stride=2,
crop='valid',
nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1, pad='valid')
net['score_final'] = ElemwiseSumLayer(
(net['score4'], net['score_pool3']),
cropping=[None, None, 'center', 'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'],
n_classes,
16,
stride=8,
crop='valid',
nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None, None),
input_var)
net['score'] = ElemwiseMergeLayer(
(net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv: deconv,
cropping=[None, None, 'center', 'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = \
lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = \
lasagne.layers.ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs'])
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(
net['probs'], (laySize[0], laySize[1], laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
# Apply temperature
if load_weights:
soft_value = net['upsample'].W.get_value() / temperature
net['upsample'].W.set_value(soft_value)
soft_value = net['upsample'].b.get_value() / temperature
net['upsample'].b.set_value(soft_value)
return [net[el] for el in layer]
def buildFCN8(nb_in_channels, input_var,
path_weights='/Tmp/romerosa/itinf/models/' +
'camvid/fcn8_model.npz',
n_classes=21, load_weights=True,
void_labels=[], trainable=False,
layer=['probs_dimshuffle']):
'''
Build fcn8 model (generator)
'''
net = {}
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None),
input_var)
# pool 1
net['conv1_1'] = ConvLayer(
net['input'], 64, 3, pad=100, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad='same', flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
# pool 2
net['conv2_1'] = ConvLayer(
net['pool1'], 128, 3, pad='same', flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad='same', flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
# pool 3
net['conv3_1'] = ConvLayer(
net['pool2'], 256, 3, pad='same', flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad='same', flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad='same', flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
# pool 4
net['conv4_1'] = ConvLayer(
net['pool3'], 512, 3, pad='same', flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad='same', flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad='same', flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
# pool 5
net['conv5_1'] = ConvLayer(
net['pool4'], 512, 3, pad='same', flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad='same', flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad='same', flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
# fc6
net['fc6'] = ConvLayer(
net['pool5'], 4096, 7, pad='valid', flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'])
# fc7
net['fc7'] = ConvLayer(
net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=0.5)
net['score_fr'] = ConvLayer(
net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2,
crop='valid', nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,
pad='same')
net['score_fused'] = ElemwiseSumLayer((net['score2'],
net['score_pool4']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4,
stride=2, crop='valid', nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,
pad='valid')
net['score_final'] = ElemwiseSumLayer((net['score4'],
net['score_pool3']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16,
stride=8, crop='valid', nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None,
None), input_var)
net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv:
deconv,
cropping=[None, None, 'center',
'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Load weights
if load_weights:
with np.load(path_weights) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs'], param_values)
if not trainable:
freezeParameters(net['probs'], single=False)
if any(void_labels):
layVoid = lasagne.layers.get_output(net['probs']).shape
input_discrim_var = T.zeros((layVoid[0], 1))
net['input_void'] = InputLayer((None, 1), input_discrim_var)
net['concat'] = ConcatLayer([net['probs'], net['input_void']],
axis=1, cropping=None)
n_classes = n_classes + 1
else:
net['concat'] = net['probs']
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(net['concat'], (laySize[0], laySize[1],
laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
return [net[el] for el in layer]
def buildDAE(input_concat_h_vars,
input_mask_var,
n_classes,
nb_features_to_concat,
padding,
ae_h=False,
void_labels=[],
path_weights='/Tmp/romerosa/itinf/models/',
model_name='dae_model.npz',
trainable=False,
load_weights=False,
out_nonlin=linear,
concat_h=['input'],
noise=0.1,
n_filters=64,
conv_before_pool=1,
additional_pool=0,
dropout=0.,
skip=False,
unpool_type='standard',
bn=0):
'''
Build score model
'''
# Build fcn contracting path
fcn_down, last_layer_down = buildFCN_down(
input_mask_var,
input_concat_h_vars,
nb_features_to_concat=nb_features_to_concat,
padding=padding,
n_classes=n_classes,
concat_layers=concat_h,
noise=noise,
n_filters=n_filters,
conv_before_pool=conv_before_pool,
additional_pool=additional_pool,
ae_h=ae_h,
dropout=dropout,
bn=bn)
dae = fcn_down
# Count the number of pooling layers to know how many upsamplings we
# should perform
if 'pool' in concat_h[-1]:
n_pool = int(concat_h[-1][-1]) + additional_pool
else:
n_pool = additional_pool
# Unpooling
fcn_up = buildFCN_up(dae,
last_layer_down,
n_pool,
skip=skip,
n_classes=n_classes,
unpool_type=unpool_type,
out_nonlin=out_nonlin,
ae_h=ae_h,
additional_pool=additional_pool,
dropout=dropout,
bn=bn)
dae.update(fcn_up)
# Load weights
if load_weights:
with np.load(os.path.join(path_weights, model_name)) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(dae['probs_dimshuffle'],
param_values)
# Do not train
if not trainable:
model_helpers.freezeParameters(dae['probs_dimshuffle'], single=False)
return dae['probs_dimshuffle']
def buildFCN8(nb_in_channels, input_var,
path_weights='/Tmp/romerosa/itinf/models/' +
'camvid/new_fcn8_model_best.npz',
n_classes=21, load_weights=True,
void_labels=[], trainable=False,
layer=['probs_dimshuffle'], pascal=False,
temperature=1.0, dropout=0.5):
'''
Build fcn8 model
'''
net = {}
# Contracting path
net['input'] = InputLayer((None, nb_in_channels, None, None),
input_var)
# pool 1
net['conv1_1'] = ConvLayer(
net['input'], 64, 3, pad=100, flip_filters=False)
net['conv1_2'] = ConvLayer(
net['conv1_1'], 64, 3, pad='same', flip_filters=False)
net['pool1'] = PoolLayer(net['conv1_2'], 2)
# pool 2
net['conv2_1'] = ConvLayer(
net['pool1'], 128, 3, pad='same', flip_filters=False)
net['conv2_2'] = ConvLayer(
net['conv2_1'], 128, 3, pad='same', flip_filters=False)
net['pool2'] = PoolLayer(net['conv2_2'], 2)
# pool 3
net['conv3_1'] = ConvLayer(
net['pool2'], 256, 3, pad='same', flip_filters=False)
net['conv3_2'] = ConvLayer(
net['conv3_1'], 256, 3, pad='same', flip_filters=False)
net['conv3_3'] = ConvLayer(
net['conv3_2'], 256, 3, pad='same', flip_filters=False)
net['pool3'] = PoolLayer(net['conv3_3'], 2)
# pool 4
net['conv4_1'] = ConvLayer(
net['pool3'], 512, 3, pad='same', flip_filters=False)
net['conv4_2'] = ConvLayer(
net['conv4_1'], 512, 3, pad='same', flip_filters=False)
net['conv4_3'] = ConvLayer(
net['conv4_2'], 512, 3, pad='same', flip_filters=False)
net['pool4'] = PoolLayer(net['conv4_3'], 2)
# pool 5
net['conv5_1'] = ConvLayer(
net['pool4'], 512, 3, pad='same', flip_filters=False)
net['conv5_2'] = ConvLayer(
net['conv5_1'], 512, 3, pad='same', flip_filters=False)
net['conv5_3'] = ConvLayer(
net['conv5_2'], 512, 3, pad='same', flip_filters=False)
net['pool5'] = PoolLayer(net['conv5_3'], 2)
# fc6
net['fc6'] = ConvLayer(
net['pool5'], 4096, 7, pad='valid', flip_filters=False)
net['fc6_dropout'] = DropoutLayer(net['fc6'], p=dropout)
# fc7
net['fc7'] = ConvLayer(
net['fc6_dropout'], 4096, 1, pad='valid', flip_filters=False)
net['fc7_dropout'] = DropoutLayer(net['fc7'], p=dropout)
net['score_fr'] = ConvLayer(
net['fc7_dropout'], n_classes, 1, pad='valid', flip_filters=False)
# Upsampling path
# Unpool
net['score2'] = DeconvLayer(net['score_fr'], n_classes, 4, stride=2,
crop='valid', nonlinearity=linear)
net['score_pool4'] = ConvLayer(net['pool4'], n_classes, 1,
pad='same')
net['score_fused'] = ElemwiseSumLayer((net['score2'],
net['score_pool4']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['score4'] = DeconvLayer(net['score_fused'], n_classes, 4,
stride=2, crop='valid', nonlinearity=linear)
net['score_pool3'] = ConvLayer(net['pool3'], n_classes, 1,
pad='valid')
net['score_final'] = ElemwiseSumLayer((net['score4'],
net['score_pool3']),
cropping=[None, None, 'center',
'center'])
# Unpool
net['upsample'] = DeconvLayer(net['score_final'], n_classes, 16,
stride=8, crop='valid', nonlinearity=linear)
upsample_shape = lasagne.layers.get_output_shape(net['upsample'])[1]
net['input_tmp'] = InputLayer((None, upsample_shape, None,
None), input_var)
net['score'] = ElemwiseMergeLayer((net['input_tmp'], net['upsample']),
merge_function=lambda input, deconv:
deconv,
cropping=[None, None, 'center',
'center'])
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = \
lasagne.layers.DimshuffleLayer(net['score'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = \
lasagne.layers.ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]),
laySize[3]))
net['probs'] = lasagne.layers.NonlinearityLayer(net['final_reshape'],
nonlinearity=softmax)
# Load weights
if load_weights:
if pascal:
path_weights = '/data/lisatmp4/erraqabi/data/att-segm/' + \
'pre_trained_weights/pascal-fcn8s-tvg-dag.mat'
if 'tvg' in path_weights:
str_filter = 'f'
str_bias = 'b'
else:
str_filter = '_filter'
str_bias = '_bias'
W = sio.loadmat(path_weights)
# Load the parameter values into the net
num_params = W.get('params').shape[1]
for i in range(num_params):
# Get layer name from the saved model
name = str(W.get('params')[0][i][0])[3:-2]
# Get parameter value
param_value = W.get('params')[0][i][1]
# Load weights
if name.endswith(str_filter):
raw_name = name[:-len(str_filter)]
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
# print 'Initializing layer ' + raw_name
param_value = param_value.T
param_value = np.swapaxes(param_value, 2, 3)
net[raw_name].W.set_value(param_value)
# Load bias terms
if name.endswith(str_bias):
raw_name = name[:-len(str_bias)]
if 'score' not in raw_name and \
'upsample' not in raw_name and \
'final' not in raw_name and \
'probs' not in raw_name:
param_value = np.squeeze(param_value)
net[raw_name].b.set_value(param_value)
else:
with np.load(path_weights) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs'], param_values)
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs'], single=False)
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(net['probs'], (laySize[0], laySize[1],
laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
# Apply temperature
if load_weights:
soft_value = net['upsample'].W.get_value() / temperature
net['upsample'].W.set_value(soft_value)
soft_value = net['upsample'].b.get_value() / temperature
net['upsample'].b.set_value(soft_value)
return [net[el] for el in layer]
def buildFCN_down(input_var,
concat_h_vars,
nb_features_to_concat,
padding,
n_classes=21,
concat_layers=['pool5'],
noise=0.1,
n_filters=64,
conv_before_pool=1,
additional_pool=0,
dropout=0.,
bn=0,
ae_h=False):
'''
Build fcn contracting path. The contracting path is built by combining
convolution and pooling layers, at least until the last concatenation is
reached. The last concatenation can be eventually followed by extra
convolution and pooling layers.
Parameters
----------
input_var: theano tensor, input of the network
concat_h_vars: list of theano tensors, intermediate inputs of the network
nb_features_to_concat: number of feature maps that the layer that we want to
concatenate has
padding: padding of the input layer
n_classes: int, number of classes
concat_layers: list intermediate layers names (layers we want to
concatenate)
noise: float, noise
n_filters: int, number of filters of each convolution (increases every time
resolution is downsampled)
conv_before_pool: int, number of convolutions before a pooling layer
additional_pool: int, number of poolings following the concatenation of the
last layer layer in `concat_h_vars` and `concat_layers`
dropout: float, dropout probability
'''
assert all([
el in ['pool1', 'pool2', 'pool3', 'pool4', 'input', 'pool5']
for el in concat_layers
])
if 'pool' in concat_layers[-1]:
n_pool = int(concat_layers[-1][-1])
else:
n_pool = 0
net = {}
pos = 0
#
# Contracting path
#
# input
net['input'] = InputLayer((None, n_classes, None, None), input_var)
# Noise
if noise > 0:
# net['noisy_input'] = GaussianNoiseLayerSoftmax(net['input'],
# sigma=noise)
net['noisy_input'] = GaussianNoiseLayer(net['input'], sigma=noise)
# net['noisy_input'] = GaussianNoiseLayerClip(net['input'], sigma=noise) # TODO: Be careful!!!
in_next = 'noisy_input'
else:
in_next = 'input'
# check whether we need to concatenate concat_h
pos, out = model_helpers.concatenate(net, in_next, concat_layers,
concat_h_vars, pos,
nb_features_to_concat)
if concat_layers[-1] == 'input' and additional_pool == 0:
raise ValueError('It seems your DAE will have no conv/pooling layers!')
# start building the network
for p in range(n_pool + additional_pool):
# add conv + pool
# freeze params of the pre-h layers
if ae_h and p == n_pool and net != {} and 'pool' in concat_layers[-1]:
model_helpers.freezeParameters(net['pool' + str(p)])
for i in range(1, conv_before_pool + 1):
# Choose padding type: this is defined according to the
# layers:
# - if have several concats, we padding 100 in the first
# layer for fcn8 compatibility
# - if concatenation is only performed at the input, we
# don't pad
if p == 0 and i == 1 and len(concat_layers) == 1 and \
concat_layers[-1] != 'input' and padding > 0:
pad_type = padding
else:
pad_type = 'same'
# Define conv (follow vgg scheme, limited to 6 due to memory
# constraints)
if p < 6:
filters_conv = n_filters * (2**p)
# add conv layer
net['conv' + str(p + 1) + '_' + str(i)] = ConvLayer(
net[out], filters_conv, 3, pad=pad_type, flip_filters=False)
out = 'conv' + str(p + 1) + '_' + str(i)
# add dropout layer
# if p > n_pool and dropout > 0.:
if dropout > 0:
net[out + '_drop'] = DropoutLayer(net[out], p=dropout)
out += '_drop'
if bn:
net[out + '_bn'] = BatchNormLayer(net[out])
out += '_bn'
# Define pool
if p == n_pool - 1:
layer_name = 'h_to_recon'
else:
layer_name = None
# add pooling layer
net['pool' + str(p + 1)] = PoolLayer(net[out], 2, name=layer_name)
out = 'pool' + str(p + 1)
laySize = net['pool' + str(p + 1)].input_shape
n_cl = laySize[1]
print('Number of feature maps (out):', n_cl)
# check whether concatenation is required
if p < n_pool:
pos, out = model_helpers.concatenate(net, 'pool' + str(p + 1),
concat_layers, concat_h_vars,
pos, nb_features_to_concat)
last_layer = out
return net, last_layer
def buildDAE_contextmod(input_concat_h_vars,
input_mask_var,
n_classes,
path_weights='/Tmp/romerosa/itinf/models/',
model_name='dae_model.npz',
trainable=False,
load_weights=False,
out_nonlin=linear,
concat_h=['input'],
noise=0.1):
'''
Build context module
Parameters
----------
input_concat_h_vars: list of theano tensors, variables to concatenate
input_mask_var: theano tensor, input to context module
n_classes: int, number of classes
path_weights: string, path to weights directory
trainable: bool, whether the model is trainable (freeze parameters or not)
load_weights: bool, whether to load pretrained weights
out_nonlin: output nonlinearity
concat_h: list of strings, names of layers we want to concatenate
noise: float, noise
'''
# context module does not reduce the image resolution
assert all([el in ['input'] for el in concat_h])
net = {}
pos = 0
# Contracting path
net['input'] = InputLayer((None, n_classes, None, None), input_mask_var)
# Noise
if noise > 0:
# net['noisy_input'] = GaussianNoiseLayerSoftmax(net['input'],
# sigma=noise)
net['noisy_input'] = GaussianNoiseLayer(net['input'], sigma=noise)
in_next = 'noisy_input'
else:
in_next = 'input'
pos, out = model_helpers.concatenate(net, in_next, concat_h,
input_concat_h_vars, pos, 3)
class IdentityInit(Initializer):
""" We adapt the same initializiation method than in the paper"""
def sample(self, shape):
n_filters, n_filters2, filter_size, filter_size2 = shape
assert ((n_filters == n_filters2) & (filter_size == filter_size2))
assert (filter_size % 2 == 1)
W = np.zeros(shape, dtype='float32')
for i in range(n_filters):
W[i, i, filter_size / 2, filter_size / 2] = 1.
return W
net['conv1'] = Conv2DLayer(net[out],
n_classes,
3,
pad='same',
nonlinearity=rectify,
flip_filters=False)
net['pad1'] = PadLayer(net['conv1'], width=32, val=0, batch_ndim=2)
net['dilconv1'] = DilatedConv2DLayer(net['pad1'],
n_classes,
3,
1,
W=IdentityInit(),
nonlinearity=rectify)
net['dilconv2'] = DilatedConv2DLayer(net['dilconv1'],
n_classes,
3,
2,
W=IdentityInit(),
nonlinearity=rectify)
net['dilconv3'] = DilatedConv2DLayer(net['dilconv2'],
n_classes,
3,
4,
W=IdentityInit(),
nonlinearity=rectify)
net['dilconv4'] = DilatedConv2DLayer(net['dilconv3'],
n_classes,
3,
8,
W=IdentityInit(),
nonlinearity=rectify)
net['dilconv5'] = DilatedConv2DLayer(net['dilconv4'],
n_classes,
3,
16,
W=IdentityInit(),
nonlinearity=rectify)
net['dilconv6'] = DilatedConv2DLayer(net['dilconv5'],
n_classes,
3,
1,
W=IdentityInit(),
nonlinearity=rectify)
net['dilconv7'] = DilatedConv2DLayer(net['dilconv6'],
n_classes,
1,
1,
W=IdentityInit(),
nonlinearity=linear)
# Final dimshuffle, reshape and softmax
net['final_dimshuffle'] = DimshuffleLayer(net['dilconv7'], (0, 2, 3, 1))
laySize = lasagne.layers.get_output(net['final_dimshuffle']).shape
net['final_reshape'] = ReshapeLayer(net['final_dimshuffle'],
(T.prod(laySize[0:3]), laySize[3]))
net['probs'] = NonlinearityLayer(net['final_reshape'],
nonlinearity=out_nonlin)
# Go back to 4D
net['probs_reshape'] = ReshapeLayer(
net['probs'], (laySize[0], laySize[1], laySize[2], n_classes))
net['probs_dimshuffle'] = DimshuffleLayer(net['probs_reshape'],
(0, 3, 1, 2))
# print('Input to last layer: ', net['probs_dimshuffle'].input_shape)
print(net.keys())
# Load weights
if load_weights:
with np.load(os.path.join(path_weights, model_name)) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(net['probs_dimshuffle'],
param_values)
# Do not train
if not trainable:
model_helpers.freezeParameters(net['probs_dimshuffle'], single=False)
return net['probs_dimshuffle']