def create_classifier(body, data, n_classes, l2_reg=0.):
# Include last layers
top = BatchNormalization(mode=0, axis=channel_idx, name="bn7")(body)
top = Activation('relu', name="relu7")(top)
top = AtrousConvolution2D(512,
3,
3,
'he_normal',
atrous_rate=(12, 12),
border_mode='same',
name="conv6a",
W_regularizer=l2(l2_reg))(top)
top = Activation('relu', name="conv6a_relu")(top)
name = "hyperplane_num_cls_%d_branch_%d" % (n_classes, 12)
def my_init(shape, name=None, dim_ordering='th'):
return initializations.normal(shape, scale=0.01, name=name)
top = AtrousConvolution2D(n_classes,
3,
3,
my_init,
atrous_rate=(12, 12),
border_mode='same',
name=name,
W_regularizer=l2(l2_reg))(top)
top = Deconvolution2D(n_classes,
16,
16,
top._keras_shape,
bilinear_init,
'linear',
border_mode='valid',
subsample=(8, 8),
bias=False,
name="upscaling_" + str(n_classes),
W_regularizer=l2(l2_reg))(top)
top = CropLayer2D(data, name='score')(top)
top = NdSoftmax()(top)
return top
def transition_up(x, skip_connection, keep_filters, weight_decay=1e-4, tu_id=""):
"""Apply BatchNorm, Relu 1x1Conv2D, optional dropout and Maxpooling2D
:param x: keras model
:param skip_connection: skip connection feature map from downsampling path
:param keep_filters: number of filters to be convolved with
:param weight_decay: int -- weight decay factor
:param tu_id: str -- transition up identifier
:returns: model
:rtype: keras model, after applying batch_norm, relu-conv, dropout, maxpool
"""
# Transposed convolution
deconv = Deconvolution2D(keep_filters, 3, 3, x._keras_shape,
border_mode='same',
subsample=(2, 2),
W_regularizer=l2(weight_decay),
b_regularizer=l2(weight_decay),
init=bilinear_init,
name='{}_deconv'.format(tu_id))(x)
# Add 1 extra pixel (0) on top and right borders to match dimensions
deconv = ZeroPadding2D({'bottom_pad': 1, 'right_pad': 1})(deconv)
return merge([deconv, skip_connection], mode='concat', concat_axis=concat_axis, name='{}_merge'.format(tu_id))
def transition_up_Layer(skip_connection, block_to_upsample, n_filters_keep):
if K.image_dim_ordering() == 'th':
concat_axis = 1
# sizeSC = [skip_connection._keras_shape[2], skip_connection._keras_shape[3]]
# sizeX = [x._keras_shape[2], x._keras_shape[3]]
elif K.image_dim_ordering() == 'tf':
concat_axis = -1
# sizeSC = [skip_connection._keras_shape[1], skip_connection._keras_shape[2]]
# sizeX = [x._keras_shape[1], x._keras_shape[2]]
x = merge(block_to_upsample, mode='concat', concat_axis=concat_axis)
print('shape_x:' + str(x._keras_shape))
x = Deconvolution2D(n_filters_keep,
3,
3,
input_shape=x._keras_shape,
activation='linear',
border_mode='valid',
subsample=(2, 2))(x)
print('shape_x_deconv:' + str(x._keras_shape))
x_crop = CropLayer2D(skip_connection)(x)
print('shape:' + str(x_crop._keras_shape))
x = merge([x_crop, skip_connection],
mode='concat',
concat_axis=concat_axis)
# print('shape_skip_connection:' + str(skip_connection._keras_shape))
# newSkip = CropLayer2D(x)(skip_connection)
#
#
#
# x = merge([x, newSkip], mode = 'concat', concat_axis = concat_axis)
print('shape_merge:' + str(x._keras_shape))
return x
def build_dilation(img_shape=(3, None, None),
nclasses=11,
l2_reg=0.,
init='glorot_uniform',
path_weights=None,
load_pretrained=False,
freeze_layers_from=None,
vgg_weights=True):
# Build network
if K.image_dim_ordering() == 'tf':
bn_axis = 3
else:
bn_axis = 1
# CONTRACTING PATH
# Input layer
inputs = Input(img_shape)
# padded = ZeroPadding2D(padding=(100, 100), name='pad100')(inputs)
# Block1
conv1_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='same',
name='block1_conv1',
W_regularizer=l2(l2_reg))(inputs)
conv1_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='same',
name='block1_conv2',
W_regularizer=l2(l2_reg))(conv1_1)
pool1 = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(conv1_2)
# Block2
conv2_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='same',
name='block2_conv1',
W_regularizer=l2(l2_reg))(pool1)
conv2_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='same',
name='block2_conv2',
W_regularizer=l2(l2_reg))(conv2_1)
pool2 = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(conv2_2)
# Block3
conv3_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='block3_conv1',
W_regularizer=l2(l2_reg))(pool2)
conv3_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='block3_conv2',
W_regularizer=l2(l2_reg))(conv3_1)
conv3_3 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='block3_conv3',
W_regularizer=l2(l2_reg))(conv3_2)
pool3 = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(conv3_3)
# Block4
conv4_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='block4_conv1',
W_regularizer=l2(l2_reg))(pool3)
conv4_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='block4_conv2',
W_regularizer=l2(l2_reg))(conv4_1)
conv4_3 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='block4_conv3',
W_regularizer=l2(l2_reg))(conv4_2)
vgg_base_model = Model(input=inputs, output=conv4_3)
vgg_base_in = vgg_base_model.output
# Block5
conv5_bn = BatchNormalization(axis=bn_axis, name='block5_bn')(vgg_base_in)
conv5_1 = AtrousConvolution2D(512,
3,
3,
atrous_rate=(2, 2),
name='atrous_conv_5_1',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv5_bn)
conv5_1_relu = Activation('relu')(conv5_1)
conv5_bn_2 = BatchNormalization(axis=bn_axis,
name='block5_bn2')(conv5_1_relu)
conv5_2 = AtrousConvolution2D(512,
3,
3,
atrous_rate=(2, 2),
name='atrous_conv_5_2',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv5_bn_2)
conv5_2_relu = Activation('relu')(conv5_2)
conv5_bn3 = BatchNormalization(axis=bn_axis,
name='block5_bn3')(conv5_2_relu)
conv5_3 = AtrousConvolution2D(512,
3,
3,
atrous_rate=(2, 2),
name='atrous_conv_5_3',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv5_bn3)
conv5_3_relu = Activation('relu')(conv5_3)
# Block6
conv6_bn = BatchNormalization(axis=bn_axis, name='block6_bn')(conv5_3_relu)
conv6 = AtrousConvolution2D(1024,
7,
7,
atrous_rate=(4, 4),
name='atrous_conv_6',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv6_bn)
conv6_relu = Activation('relu')(conv6)
conv6_relu = Dropout(0.5)(conv6_relu)
# Block7
conv7_bn = BatchNormalization(axis=bn_axis, name='block7_bn')(conv6_relu)
conv7 = AtrousConvolution2D(4096,
1,
1,
atrous_rate=(1, 1),
name='atrous_conv_7',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(conv7_bn)
conv7_relu = Activation('relu')(conv7)
conv7_relu = Dropout(0.5)(conv7_relu)
# Final block
convf_bn = BatchNormalization(axis=bn_axis, name='blockf_bn')(conv7_relu)
x = AtrousConvolution2D(nclasses,
1,
1,
atrous_rate=(1, 1),
name='final_block',
border_mode='same',
dim_ordering=dim_ordering,
init=identity_init)(convf_bn)
# Appending context block
upsampling = 8
context_out = context_block(x, [1, 1, 2, 4, 8, 16, 1],
nclasses,
init=identity_init)
deconv_out = Deconvolution2D(
nclasses,
upsampling,
upsampling,
init=bilinear_init,
subsample=(upsampling, upsampling),
input_shape=context_out._keras_shape)(context_out)
# Softmax
prob = NdSoftmax()(deconv_out)
# Complete model
model = Model(input=vgg_base_model.input, output=prob)
# Load pretrained weights VGG part of the model
if vgg_weights == True:
if K.image_dim_ordering() == 'th':
weights_path = get_file(
'vgg19_weights_th_dim_ordering_th_kernels_notop.h5',
TH_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
else:
weights_path = get_file(
'vgg19_weights_tf_dim_ordering_tf_kernels_notop.h5',
TF_WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
model.load_weights(weights_path, by_name=True)
print('Loaded pre-trained VGG weights')
if path_weights:
print('Loaded pre-trained weights for the full network')
model.load_weights(weights_path, by_name=True)
# load_matcovnet(model, path_weights, n_classes=nclasses)
# Freeze some layers
if freeze_layers_from is not None:
freeze_layers(model, freeze_layers_from)
return model
def build_unet(img_shape=(3, None, None),
nclasses=8,
l2_reg=0.,
init='glorot_uniform',
freeze_layers_from=None,
path_weights=None):
# Regularization warning
if l2_reg > 0.:
print("Regularizing the weights: " + str(l2_reg))
# Build network
# CONTRACTING PATH
# Input layer
inputs = Input(img_shape)
padded1 = ZeroPadding2D(padding=(100, 100), name='padded1')(inputs)
# Block 1
conv1_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv1_1',
W_regularizer=l2(l2_reg))(padded1)
conv1_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv1_2',
W_regularizer=l2(l2_reg))(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2), name='pool1')(conv1_2)
# Block 2
conv2_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='conv2_1',
W_regularizer=l2(l2_reg))(pool1)
conv2_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='conv2_2',
W_regularizer=l2(l2_reg))(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2), name='pool2')(conv2_2)
# Block 3
conv3_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='conv3_1',
W_regularizer=l2(l2_reg))(pool2)
conv3_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='conv3_2',
W_regularizer=l2(l2_reg))(conv3_1)
pool3 = MaxPooling2D(pool_size=(2, 2), name='pool3')(conv3_2)
# Block 4
conv4_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='conv4_1',
W_regularizer=l2(l2_reg))(pool3)
conv4_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='conv4_2',
W_regularizer=l2(l2_reg))(conv4_1)
conv4_drop = Dropout(0.5, name='conv4_drop')(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2), name='pool4')(conv4_drop)
# Block 5
bottom_conv1 = Convolution2D(1024,
3,
3,
init,
'relu',
border_mode='valid',
name='bottom_conv1',
W_regularizer=l2(l2_reg))(pool4)
bottom_conv2 = Convolution2D(1024,
3,
3,
init,
'relu',
border_mode='valid',
name='bottom_conv2',
W_regularizer=l2(l2_reg))(bottom_conv1)
bottom_drop = Dropout(0.5, name='bottom_drop')(bottom_conv2)
#uncoder blocks
# Block 6
deconv4 = Deconvolution2D(512,
2,
2,
bottom_drop._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='deconv4',
W_regularizer=l2(l2_reg))(bottom_drop)
conv4_crop = CropLayer2D(deconv4, name='conv4_crop')(conv4_drop)
deconv4_crop = CropLayer2D(deconv4, name='deconv4_crop')(deconv4)
deconv4_concat = merge([deconv4_crop, conv4_crop],
mode='concat',
concat_axis=3,
name='deconv4_concat')
deconv4_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv4_1',
W_regularizer=l2(l2_reg))(deconv4_concat)
deconv4_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv4_2',
W_regularizer=l2(l2_reg))(deconv4_1)
# Block 7
deconv3 = Deconvolution2D(256,
2,
2,
deconv4_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='deconv3',
W_regularizer=l2(l2_reg))(deconv4_2)
deconv3_crop = CropLayer2D(deconv3, name='deconv3_crop')(conv3_2)
deconv3_concat = merge([deconv3_crop, deconv3],
mode='concat',
concat_axis=3,
name='deconv3_concat')
deconv3_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv3_1',
W_regularizer=l2(l2_reg))(deconv3_concat)
deconv3_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv3_2',
W_regularizer=l2(l2_reg))(deconv3_1)
# Block 8
deconv2 = Deconvolution2D(128,
2,
2,
deconv3_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='deconv2',
W_regularizer=l2(l2_reg))(deconv3_2)
deconv2_crop = CropLayer2D(deconv2, name='deconv2_crop')(conv2_2)
deconv2_concat = merge([deconv2_crop, deconv2],
mode='concat',
concat_axis=3,
name='deconv2_concat')
deconv2_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv2_1',
W_regularizer=l2(l2_reg))(deconv2_concat)
deconv2_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv2_2',
W_regularizer=l2(l2_reg))(deconv2_1)
# Block 9
deconv1 = Deconvolution2D(64,
2,
2,
deconv2_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='deconv1',
W_regularizer=l2(l2_reg))(deconv2_2)
deconv1_crop = CropLayer2D(deconv1, name='deconv1_crop')(conv1_2)
deconv1_concat = merge([deconv1_crop, deconv1],
mode='concat',
concat_axis=3,
name='deconv1_concat')
deconv1_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv1_1',
W_regularizer=l2(l2_reg))(deconv1_concat)
deconv1_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='deconv1_2',
W_regularizer=l2(l2_reg))(deconv1_1)
l1 = Convolution2D(nclasses, 1, 1, border_mode='valid',
name='logits')(deconv1_2)
score = CropLayer2D(inputs, name='score')(l1)
# Softmax
softmax_segnet = NdSoftmax()(score)
# Complete model
model = Model(input=inputs, output=softmax_segnet)
# Load pretrained Model
if path_weights:
load_matcovnet(model, n_classes=nclasses)
# Freeze some layers
if freeze_layers_from is not None:
freeze_layers(model, freeze_layers_from)
return model
def TinyYOLT(input_shape=(3, 416, 416), num_classes=80, num_priors=5):
"""Tiny TinyYOLT architecture
# Arguments
input_shape: Shape of the input image
num_classes: Number of classes (excluding background)
# References
https://arxiv.org/abs/1612.08242
https://arxiv.org/abs/1506.02640
"""
K.set_image_dim_ordering('th')
input_tensor = Input(shape=input_shape)
net = {}
net['input'] = input_tensor
# Scale 1
net['input_scale1'] = (YOLOConvolution2D(16,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['input'])
# Scale 2
net['deconv'] = Deconvolution2D(3,
4,
4,
net['input']._keras_shape,
'glorot_uniform',
'linear',
border_mode='valid',
subsample=(2, 2),
name='deconv')(net['input'])
net['deconv'] = UpSampling2D(size=(2, 2))((net['input']))
net['conv1'] = (YOLOConvolution2D(16,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['deconv'])
net['input_scale2'] = (MaxPooling2D(pool_size=(2, 2),
border_mode='valid'))(net['conv1'])
# Merge scales
# net['merge'] = (concat())([net['input_scale1'], net['input_scale2']])
net['merge'] = merge([net['input_scale1'], net['input_scale2']],
mode='concat',
name='merge',
concat_axis=1)
net['relu1'] = (LeakyReLU(alpha=0.1))(net['merge'])
net['pool1'] = (MaxPooling2D(pool_size=(2, 2),
border_mode='valid'))(net['relu1'])
net['conv2'] = (YOLOConvolution2D(32,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['pool1'])
net['relu2'] = (LeakyReLU(alpha=0.1))(net['conv2'])
net['pool2'] = (MaxPooling2D(pool_size=(2, 2),
border_mode='valid'))(net['relu2'])
net['conv3'] = (YOLOConvolution2D(64,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['pool2'])
net['relu3'] = (LeakyReLU(alpha=0.1))(net['conv3'])
net['pool3'] = (MaxPooling2D(pool_size=(2, 2),
border_mode='valid'))(net['relu3'])
net['conv4'] = (YOLOConvolution2D(128,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['pool3'])
net['relu4'] = (LeakyReLU(alpha=0.1))(net['conv4'])
net['pool4'] = (MaxPooling2D(pool_size=(2, 2),
border_mode='valid'))(net['relu4'])
net['conv5'] = (YOLOConvolution2D(256,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['pool4'])
net['relu5'] = (LeakyReLU(alpha=0.1))(net['conv5'])
net['pool5'] = (MaxPooling2D(pool_size=(2, 2),
border_mode='valid'))(net['relu5'])
net['conv6'] = (YOLOConvolution2D(512,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['pool5'])
net['relu6'] = (LeakyReLU(alpha=0.1))(net['conv6'])
net['pool6'] = (MaxPooling2D(pool_size=(2, 2),
strides=(1, 1),
border_mode='same'))(net['relu6'])
net['conv7'] = (YOLOConvolution2D(1024,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['pool6'])
net['relu7'] = (LeakyReLU(alpha=0.1))(net['conv7'])
net['conv8'] = (YOLOConvolution2D(1024,
3,
3,
border_mode='same',
subsample=(1, 1),
epsilon=0.000001))(net['relu7'])
net['relu8'] = (LeakyReLU(alpha=0.1))(net['conv8'])
net['output'] = (Convolution2D(num_priors * (4 + num_classes + 1),
1,
1,
border_mode='same',
subsample=(1, 1)))(net['relu8'])
model = Model(net['input'], net['output'])
return model
def build_unet(img_shape=(3, None, None),
nclasses=8,
l2_reg=0.,
init='glorot_uniform',
path_weights=None,
freeze_layers_from=None,
padding=100,
dropout=True):
# Regularization warning
if l2_reg > 0.:
print("Regularizing the weights: " + str(l2_reg))
# Input
inputs = Input(img_shape, name='input')
padded = ZeroPadding2D(padding=(padding, padding), name='padded')(inputs)
# Block 1
conv1_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv1_1',
W_regularizer=l2(l2_reg))(padded)
conv1_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv1_2',
W_regularizer=l2(l2_reg))(conv1_1)
pool1 = MaxPooling2D((2, 2), (2, 2), name='pool1')(conv1_2)
# Block 2
conv2_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='conv2_1',
W_regularizer=l2(l2_reg))(pool1)
conv2_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='conv2_2',
W_regularizer=l2(l2_reg))(conv2_1)
pool2 = MaxPooling2D((2, 2), (2, 2), name='pool2')(conv2_2)
# Block 3
conv3_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='conv3_1',
W_regularizer=l2(l2_reg))(pool2)
conv3_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='conv3_2',
W_regularizer=l2(l2_reg))(conv3_1)
pool3 = MaxPooling2D((2, 2), (2, 2), name='pool3')(conv3_2)
# Block 4
conv4_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='conv4_1',
W_regularizer=l2(l2_reg))(pool3)
conv4_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='conv4_2',
W_regularizer=l2(l2_reg))(conv4_1)
if dropout:
conv4_2 = Dropout(0.5, name='drop1')(conv4_2)
pool4 = MaxPooling2D((2, 2), (2, 2), name='pool4')(conv4_2)
# Block 5
conv5_1 = Convolution2D(1024,
3,
3,
init,
'relu',
border_mode='valid',
name='conv5_1',
W_regularizer=l2(l2_reg))(pool4)
conv5_2 = Convolution2D(1024,
3,
3,
init,
'relu',
border_mode='valid',
name='conv5_2',
W_regularizer=l2(l2_reg))(conv5_1)
if dropout:
conv5_2 = Dropout(0.5, name='drop2')(conv5_2)
# pool5 = MaxPooling2D((2, 2), (2, 2), name='pool4')(conv5_2)
# Upsampling 1
upconv4 = Deconvolution2D(512,
2,
2,
conv5_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='upconv4',
W_regularizer=l2(l2_reg))(conv5_2)
conv4_2_crop = CropLayer2D(upconv4, name='conv4_2_crop')(conv4_2)
upconv4_crop = CropLayer2D(upconv4, name='upconv4_crop')(upconv4)
Concat_4 = merge([conv4_2_crop, upconv4_crop],
mode='concat',
concat_axis=3,
name='Concat_4')
conv6_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='conv6_1',
W_regularizer=l2(l2_reg))(Concat_4)
conv6_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='valid',
name='conv6_2',
W_regularizer=l2(l2_reg))(conv6_1)
# Upsampling 2
upconv3 = Deconvolution2D(256,
2,
2,
conv6_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='upconv3',
W_regularizer=l2(l2_reg))(conv6_2)
conv3_2_crop = CropLayer2D(upconv3, name='conv3_2_crop')(conv3_2)
Concat_3 = merge([conv3_2_crop, upconv3], mode='concat', name='Concat_3')
conv7_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='conv7_1',
W_regularizer=l2(l2_reg))(Concat_3)
conv7_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='valid',
name='conv7_2',
W_regularizer=l2(l2_reg))(conv7_1)
# Upsampling 3
upconv2 = Deconvolution2D(128,
2,
2,
conv7_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='upconv2',
W_regularizer=l2(l2_reg))(conv7_2)
conv2_2_crop = CropLayer2D(upconv2, name='conv2_2_crop')(conv2_2)
Concat_2 = merge([conv2_2_crop, upconv2], mode='concat', name='Concat_2')
conv8_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='conv8_1',
W_regularizer=l2(l2_reg))(Concat_2)
conv8_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='valid',
name='conv8_2',
W_regularizer=l2(l2_reg))(conv8_1)
# Upsampling 4
upconv1 = Deconvolution2D(64,
2,
2,
conv8_2._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='upconv1',
W_regularizer=l2(l2_reg))(conv8_2)
conv1_2_crop = CropLayer2D(upconv1, name='conv1_2_crop')(conv1_2)
Concat_1 = merge([conv1_2_crop, upconv1], mode='concat', name='Concat_1')
conv9_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv9_1',
W_regularizer=l2(l2_reg))(Concat_1)
conv9_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv9_2',
W_regularizer=l2(l2_reg))(conv9_1)
conv10 = Convolution2D(nclasses,
1,
1,
init,
'linear',
border_mode='valid',
name='conv10',
W_regularizer=l2(l2_reg))(conv9_2)
# Crop
final_crop = CropLayer2D(inputs, name='final_crop')(conv10)
# Softmax
softmax_unet = NdSoftmax()(final_crop)
# Complete model
model = Model(input=inputs, output=softmax_unet)
# Load pretrained Model
if path_weights:
pass
# Freeze some layers
if freeze_layers_from is not None:
freeze_layers(model, freeze_layers_from)
return model
def build_fcn8(img_shape=(3, None, None),
nclasses=8,
l2_reg=0.,
init='glorot_uniform',
path_weights=None,
freeze_layers_from=None):
# Regularization warning
if l2_reg > 0.:
print("Regularizing the weights: " + str(l2_reg))
# Build network
# CONTRACTING PATH
# Input layer
inputs = Input(img_shape)
padded = ZeroPadding2D(padding=(100, 100), name='pad100')(inputs)
# Block 1
conv1_1 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='valid',
name='conv1_1',
W_regularizer=l2(l2_reg))(padded)
conv1_2 = Convolution2D(64,
3,
3,
init,
'relu',
border_mode='same',
name='conv1_2',
W_regularizer=l2(l2_reg))(conv1_1)
pool1 = MaxPooling2D((2, 2), (2, 2), name='pool1')(conv1_2)
# Block 2
conv2_1 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='same',
name='conv2_1',
W_regularizer=l2(l2_reg))(pool1)
conv2_2 = Convolution2D(128,
3,
3,
init,
'relu',
border_mode='same',
name='conv2_2',
W_regularizer=l2(l2_reg))(conv2_1)
pool2 = MaxPooling2D((2, 2), (2, 2), name='pool2')(conv2_2)
# Block 3
conv3_1 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='conv3_1',
W_regularizer=l2(l2_reg))(pool2)
conv3_2 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='conv3_2',
W_regularizer=l2(l2_reg))(conv3_1)
conv3_3 = Convolution2D(256,
3,
3,
init,
'relu',
border_mode='same',
name='conv3_3',
W_regularizer=l2(l2_reg))(conv3_2)
pool3 = MaxPooling2D((2, 2), (2, 2), name='pool3')(conv3_3)
# Block 4
conv4_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='conv4_1',
W_regularizer=l2(l2_reg))(pool3)
conv4_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='conv4_2',
W_regularizer=l2(l2_reg))(conv4_1)
conv4_3 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='conv4_3',
W_regularizer=l2(l2_reg))(conv4_2)
pool4 = MaxPooling2D((2, 2), (2, 2), name='pool4')(conv4_3)
# Block 5
conv5_1 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='conv5_1',
W_regularizer=l2(l2_reg))(pool4)
conv5_2 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='conv5_2',
W_regularizer=l2(l2_reg))(conv5_1)
conv5_3 = Convolution2D(512,
3,
3,
init,
'relu',
border_mode='same',
name='conv5_3',
W_regularizer=l2(l2_reg))(conv5_2)
pool5 = MaxPooling2D((2, 2), (2, 2), name='pool5')(conv5_3)
# Block 6 (fully conv)
fc6 = Convolution2D(4096,
7,
7,
init,
'relu',
border_mode='valid',
name='fc6',
W_regularizer=l2(l2_reg))(pool5)
fc6 = Dropout(0.5)(fc6)
# Block 7 (fully conv)
fc7 = Convolution2D(
4096,
1,
1,
init,
'relu',
border_mode='valid',
name='fc7',
W_regularizer=l2(l2_reg),
)(fc6)
fc7 = Dropout(0.5)(fc7)
score_fr = Convolution2D(nclasses,
1,
1,
init,
'relu',
border_mode='valid',
name='score_fr')(fc7)
# DECONTRACTING PATH
# Unpool 1
score_pool4 = Convolution2D(nclasses,
1,
1,
init,
'relu',
border_mode='same',
name='score_pool4',
W_regularizer=l2(l2_reg))(pool4)
score2 = Deconvolution2D(nclasses,
4,
4,
score_fr._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
name='score2',
W_regularizer=l2(l2_reg))(score_fr)
score_pool4_crop = CropLayer2D(score2,
name='score_pool4_crop')(score_pool4)
score_fused = merge([score_pool4_crop, score2],
mode=custom_sum,
output_shape=custom_sum_shape,
name='score_fused')
# Unpool 2
score_pool3 = Convolution2D(nclasses,
1,
1,
init,
'relu',
border_mode='valid',
name='score_pool3',
W_regularizer=l2(l2_reg))(pool3)
score4 = Deconvolution2D(
nclasses,
4,
4,
score_fused._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(2, 2),
bias=True, # TODO: No bias??
name='score4',
W_regularizer=l2(l2_reg))(score_fused)
score_pool3_crop = CropLayer2D(score4,
name='score_pool3_crop')(score_pool3)
score_final = merge([score_pool3_crop, score4],
mode=custom_sum,
output_shape=custom_sum_shape,
name='score_final')
upsample = Deconvolution2D(
nclasses,
16,
16,
score_final._keras_shape,
init,
'linear',
border_mode='valid',
subsample=(8, 8),
bias=False, # TODO: No bias??
name='upsample',
W_regularizer=l2(l2_reg))(score_final)
score = CropLayer2D(inputs, name='score')(upsample)
# Softmax
softmax_fcn8 = NdSoftmax()(score)
# Complete model
model = Model(input=inputs, output=softmax_fcn8)
# Load pretrained Model
if path_weights:
load_matcovnet(model, n_classes=nclasses)
# Freeze some layers
if freeze_layers_from is not None:
freeze_layers(model, freeze_layers_from)
return model
def build_densenet_segmentation(in_shape=(3, 224, 224),
n_classes=1000,
weight_decay=0.,
freeze_layers_from='base_model',
path_weights=None):
#####################
# First Convolution #
#####################
print('Input shape:' + str(in_shape))
inp = Input(shape=in_shape)
n_filter = 48
x = Convolution2D(n_filter, 3, 3, subsample=(1, 1),
border_mode='same')(inp)
#####################
# Downsampling path #
#####################
growth_rate = 16
dropout_fraction = 0.2
n_layers_down = [4, 5, 7, 10, 12]
skip_connection_list = []
for i in range(len(n_layers_down)):
#Dense block
x, n_filter = denseBlock_down(x, n_layers_down[i], growth_rate,
n_filter, dropout_fraction)
print('number of filters = ' + str(x._keras_shape[-1]))
x = transition_down_Layer(x, n_filter, dropout_fraction)
# At the end of the dense block, the current output is stored in the skip_connections list
skip_connection_list.append(x)
# print('Shape: ' + str(x._keras_shape))
skip_connection_list = skip_connection_list[::-1]
#####################
# Bottleneck #
#####################
n_layers = 15
# We store now the output of the next dense block in a list(block_to_upsample).
# We will only upsample these new feature maps
x, n_filter, block_to_upsample = denseBlock_up(x, n_layers, growth_rate,
n_filter, dropout_fraction)
print('number of filters = ' + str(x._keras_shape[-1]))
# Add dense blocks of upsampling path
n_layers_up = [15, 12, 10, 7, 5, 4]
for j in range(1, len(n_layers_up)):
# Transition Up ( Upsampling + concatenation with the skip connection)
n_filters_keep = growth_rate * n_layers_up[j - 1]
x = transition_up_Layer(skip_connection_list[j - 1], block_to_upsample,
n_filters_keep)
x, n_filter, block_to_upsample = denseBlock_up(x, n_layers_up[j],
growth_rate, n_filter,
dropout_fraction)
print('number of filters = ' + str(x._keras_shape[-1]))
x = Deconvolution2D(n_filters_keep,
3,
3,
input_shape=x._keras_shape,
activation='linear',
border_mode='valid',
subsample=(2, 2))(x)
x = CropLayer2D(inp)(x)
#Last convolution
x = Convolution2D(n_classes, 1, 1, subsample=(1, 1), border_mode='same')(x)
#####################
# Softmax #
#####################
predictions = NdSoftmax()(x)
print('Predictions_shape: ' + str(predictions._keras_shape))
# This is the model we will train
model = Model(input=inp, output=predictions)
model.summary()
# Freeze some layers
if freeze_layers_from is not None:
if freeze_layers_from == 'base_model':
print(' Freezing base model layers')
for layer in model.layers:
layer.trainable = False
else:
for i, layer in enumerate(model.layers):
print(i, layer.name)
print(' Freezing from layer 0 to ' + str(freeze_layers_from))
for layer in model.layers[:freeze_layers_from]:
layer.trainable = False
for layer in model.layers[freeze_layers_from:]:
layer.trainable = True
return model
def build_fcn8(img_shape,
x_shape=None,
dim_ordering='th',
l2_reg=0.,
nclasses=8,
x_test_val=None,
weights_file=False,
**kwargs):
# For Theano debug prouposes
if x_test_val is not None:
inputs.tag.test_value = x_test_val
theano.config.compute_test_value = "warn"
do = dim_ordering
# Regularization warning
if l2_reg > 0.:
print("Regularizing the weights: " + str(l2_reg))
# Build network
# CONTRACTING PATH
# Input layer
inputs = Input(img_shape)
sh = inputs._keras_shape
padded = ZeroPadding2D(padding=(100, 100), dim_ordering=do,
name='pad100')(inputs)
# Block 1
conv1_1 = Convolution2D(64,
3,
3,
activation='relu',
border_mode='valid',
dim_ordering=do,
name='conv1_1',
W_regularizer=l2(l2_reg),
trainable=True)(padded)
conv1_2 = Convolution2D(64,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv1_2',
W_regularizer=l2(l2_reg),
trainable=True)(conv1_1)
pool1 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
dim_ordering=do,
name='pool1')(conv1_2)
# Block 2
conv2_1 = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv2_1',
W_regularizer=l2(l2_reg),
trainable=True)(pool1)
conv2_2 = Convolution2D(128,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv2_2',
W_regularizer=l2(l2_reg),
trainable=True)(conv2_1)
pool2 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
dim_ordering=do,
name='pool2')(conv2_2)
# Block 3
conv3_1 = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv3_1',
W_regularizer=l2(l2_reg),
trainable=True)(pool2)
conv3_2 = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv3_2',
W_regularizer=l2(l2_reg),
trainable=True)(conv3_1)
conv3_3 = Convolution2D(256,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv3_3',
W_regularizer=l2(l2_reg),
trainable=True)(conv3_2)
pool3 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
dim_ordering=do,
name='pool3')(conv3_3)
# Block 4
conv4_1 = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv4_1',
W_regularizer=l2(l2_reg),
trainable=True)(pool3)
conv4_2 = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv4_2',
W_regularizer=l2(l2_reg),
trainable=True)(conv4_1)
conv4_3 = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv4_3',
W_regularizer=l2(l2_reg),
trainable=True)(conv4_2)
pool4 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
dim_ordering=do,
name='pool4')(conv4_3)
# Block 5
conv5_1 = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv5_1',
W_regularizer=l2(l2_reg),
trainable=True)(pool4)
conv5_2 = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv5_2',
W_regularizer=l2(l2_reg),
trainable=True)(conv5_1)
conv5_3 = Convolution2D(512,
3,
3,
activation='relu',
border_mode='same',
dim_ordering=do,
name='conv5_3',
W_regularizer=l2(l2_reg),
trainable=True)(conv5_2)
pool5 = MaxPooling2D(pool_size=(2, 2),
strides=(2, 2),
dim_ordering=do,
name='pool5')(conv5_3)
# Block 6 (fully conv)
fc6 = Convolution2D(4096,
7,
7,
activation='relu',
border_mode='valid',
dim_ordering=do,
name='fc6',
W_regularizer=l2(l2_reg),
trainable=True)(pool5)
fc6 = Dropout(0.5)(fc6)
# Block 7 (fully conv)
fc7 = Convolution2D(4096,
1,
1,
activation='relu',
border_mode='valid',
dim_ordering=do,
name='fc7',
W_regularizer=l2(l2_reg),
trainable=True)(fc6)
fc7 = Dropout(0.5)(fc7)
score_fr = Convolution2D(nclasses,
1,
1,
activation='relu',
border_mode='valid',
dim_ordering=do,
name='score_fr')(fc7)
# DECONTRACTING PATH
# Unpool 1
score_pool4 = Convolution2D(nclasses,
1,
1,
activation='relu',
border_mode='same',
dim_ordering=do,
name='score_pool4',
W_regularizer=l2(l2_reg),
trainable=True)(pool4)
score2 = Deconvolution2D(nb_filter=nclasses,
nb_row=4,
nb_col=4,
input_shape=score_fr._keras_shape,
subsample=(2, 2),
border_mode='valid',
activation='linear',
W_regularizer=l2(l2_reg),
dim_ordering=do,
trainable=True,
name='score2')(score_fr)
score_pool4_crop = CropLayer2D(score2,
dim_ordering=do,
name='score_pool4_crop')(score_pool4)
score_fused = merge([score_pool4_crop, score2],
mode=custom_sum,
output_shape=custom_sum_shape,
name='score_fused')
# Unpool 2
score_pool3 = Convolution2D(nclasses,
1,
1,
activation='relu',
border_mode='valid',
dim_ordering=do,
W_regularizer=l2(l2_reg),
trainable=True,
name='score_pool3')(pool3)
score4 = Deconvolution2D(nb_filter=nclasses,
nb_row=4,
nb_col=4,
input_shape=score_fused._keras_shape,
subsample=(2, 2),
border_mode='valid',
activation='linear',
W_regularizer=l2(l2_reg),
dim_ordering=do,
trainable=True,
name='score4',
bias=False)(score_fused) # TODO: No bias??
score_pool3_crop = CropLayer2D(score4,
dim_ordering=do,
name='score_pool3_crop')(score_pool3)
score_final = merge([score_pool3_crop, score4],
mode=custom_sum,
output_shape=custom_sum_shape,
name='score_final')
# Unpool 3
upsample = Deconvolution2D(nb_filter=nclasses,
nb_row=16,
nb_col=16,
input_shape=score_final._keras_shape,
subsample=(8, 8),
border_mode='valid',
activation='linear',
W_regularizer=l2(l2_reg),
dim_ordering=do,
trainable=True,
name='upsample',
bias=False)(score_final) # TODO: No bias??
score = CropLayer2D(inputs, dim_ordering=do, name='score')(upsample)
# Softmax
if do == 'th':
softmax_fcn8 = NdSoftmax(1)(score)
else:
softmax_fcn8 = NdSoftmax(3)(score)
# Complete model
net = Model(input=inputs, output=softmax_fcn8)
# Load weights
if weights_file:
print(' > Loading weights from pretrained model: ' + weights_file)
net.load_weights(weights_file)
return net
def inceptionSeg(inp, kernel, concat_axis, n_classes):
# Encoding layers: inception layers
# print('Entra1')
# base_model = InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None)
# print('Entra2')
# x = base_model.layers[-2].output
conv1_7x7_s2 = Convolution2D(64,7,7,subsample=(2,2),border_mode='same',activation='relu',name='conv1/7x7_s2')(inp)
# conv1_zero_pad = ZeroPadding2D(padding=(1, 1))(conv1_7x7_s2)
#
# pool1_helper = PoolHelper()(conv1_zero_pad)
# pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool1/3x3_s2')(pool1_helper)
pool1_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool1/3x3_s2')(conv1_7x7_s2)
#pool1_norm1 = LRN(name='pool1/norm1')(pool1_3x3_s2)
conv2_3x3_reduce = Convolution2D(64,1,1,border_mode='same',activation='relu',name='conv2/3x3_reduce')(pool1_3x3_s2)
conv2_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='conv2/3x3')(conv2_3x3_reduce)
#conv2_norm2 = LRN(name='conv2/norm2')(conv2_3x3)
conv2_zero_pad = ZeroPadding2D(padding=(1, 1))(conv2_3x3)
pool2_helper = PoolHelper()(conv2_zero_pad)
pool2_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool2/3x3_s2')(pool2_helper)
# First Inception module
inception_3a_1x1 = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3a/1x1')(pool2_3x3_s2)
inception_3a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_3a/3x3_reduce')(pool2_3x3_s2)
inception_3a_3x3 = Convolution2D(128,3,3,border_mode='same',activation='relu',name='inception_3a/3x3')(inception_3a_3x3_reduce)
inception_3a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_3a/5x5_reduce')(pool2_3x3_s2)
inception_3a_5x5 = Convolution2D(32,5,5,border_mode='same',activation='relu',name='inception_3a/5x5')(inception_3a_5x5_reduce)
inception_3a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3a/pool')(pool2_3x3_s2)
inception_3a_pool_proj = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3a/pool_proj')(inception_3a_pool)
inception_3a_output = merge([inception_3a_1x1,inception_3a_3x3,inception_3a_5x5,inception_3a_pool_proj],mode='concat',concat_axis=1,name='inception_3a/output')
# Second Inception module
inception_3b_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/1x1')(inception_3a_output)
inception_3b_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_3b/3x3_reduce')(inception_3a_output)
inception_3b_3x3 = Convolution2D(192,3,3,border_mode='same',activation='relu',name='inception_3b/3x3')(inception_3b_3x3_reduce)
inception_3b_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_3b/5x5_reduce' )(inception_3a_output)
inception_3b_5x5 = Convolution2D(96,5,5,border_mode='same',activation='relu',name='inception_3b/5x5' )(inception_3b_5x5_reduce)
inception_3b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_3b/pool')(inception_3a_output)
inception_3b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_3b/pool_proj' )(inception_3b_pool)
inception_3b_output = merge([inception_3b_1x1,inception_3b_3x3,inception_3b_5x5,inception_3b_pool_proj],mode='concat',concat_axis=1,name='inception_3b/output')
inception_3b_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_3b_output)
pool3_helper = PoolHelper()(inception_3b_output_zero_pad)
pool3_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool3/3x3_s2')(pool3_helper)
# Third Inception module
inception_4a_1x1 = Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_4a/1x1')(pool3_3x3_s2)
inception_4a_3x3_reduce = Convolution2D(96,1,1,border_mode='same',activation='relu',name='inception_4a/3x3_reduce')(pool3_3x3_s2)
inception_4a_3x3 = Convolution2D(208,3,3,border_mode='same',activation='relu',name='inception_4a/3x3')(inception_4a_3x3_reduce)
inception_4a_5x5_reduce = Convolution2D(16,1,1,border_mode='same',activation='relu',name='inception_4a/5x5_reduce')(pool3_3x3_s2)
inception_4a_5x5 = Convolution2D(48,5,5,border_mode='same',activation='relu',name='inception_4a/5x5' )(inception_4a_5x5_reduce)
inception_4a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4a/pool')(pool3_3x3_s2)
inception_4a_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4a/pool_proj')(inception_4a_pool)
inception_4a_output = merge([inception_4a_1x1,inception_4a_3x3,inception_4a_5x5,inception_4a_pool_proj],mode='concat',concat_axis=1,name='inception_4a/output')
loss1_ave_pool = AveragePooling2D(pool_size=(5,5),strides=(3,3),name='loss1/ave_pool')(inception_4a_output)
loss1_conv = Convolution2D(128,1,1,border_mode='same',activation='relu',name='loss1/conv')(loss1_ave_pool)
loss1_flat = Flatten()(loss1_conv)
loss1_fc = Dense(1024,activation='relu',name='loss1/fc')(loss1_flat)
loss1_drop_fc = Dropout(0.7)(loss1_fc)
loss1_classifier = Dense(1000,name='loss1/classifier')(loss1_drop_fc)
loss1_classifier_act = Activation('softmax')(loss1_classifier)
# Fourth Inception module
inception_4b_1x1 = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4b/1x1')(inception_4a_output)
inception_4b_3x3_reduce = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4b/3x3_reduce')(inception_4a_output)
inception_4b_3x3 = Convolution2D(224,3,3,border_mode='same',activation='relu',name='inception_4b/3x3' )(inception_4b_3x3_reduce)
inception_4b_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4b/5x5_reduce')(inception_4a_output)
inception_4b_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4b/5x5')(inception_4b_5x5_reduce)
inception_4b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4b/pool')(inception_4a_output)
inception_4b_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4b/pool_proj')(inception_4b_pool)
inception_4b_output = merge([inception_4b_1x1,inception_4b_3x3,inception_4b_5x5,inception_4b_pool_proj],mode='concat',concat_axis=1,name='inception_4b_output')
# Fifth Inception module
inception_4c_1x1 = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/1x1')(inception_4b_output)
inception_4c_3x3_reduce = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4c/3x3_reduce')(inception_4b_output)
inception_4c_3x3 = Convolution2D(256,3,3,border_mode='same',activation='relu',name='inception_4c/3x3')(inception_4c_3x3_reduce)
inception_4c_5x5_reduce = Convolution2D(24,1,1,border_mode='same',activation='relu',name='inception_4c/5x5_reduce')(inception_4b_output)
inception_4c_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4c/5x5')(inception_4c_5x5_reduce)
inception_4c_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4c/pool')(inception_4b_output)
inception_4c_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4c/pool_proj')(inception_4c_pool)
inception_4c_output = merge([inception_4c_1x1,inception_4c_3x3,inception_4c_5x5,inception_4c_pool_proj],mode='concat',concat_axis=1,name='inception_4c/output')
# Sixth Inception module
inception_4d_1x1 = Convolution2D(112,1,1,border_mode='same',activation='relu',name='inception_4d/1x1')(inception_4c_output)
inception_4d_3x3_reduce = Convolution2D(144,1,1,border_mode='same',activation='relu',name='inception_4d/3x3_reduce')(inception_4c_output)
inception_4d_3x3 = Convolution2D(288,3,3,border_mode='same',activation='relu',name='inception_4d/3x3' )(inception_4d_3x3_reduce)
inception_4d_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4d/5x5_reduce')(inception_4c_output)
inception_4d_5x5 = Convolution2D(64,5,5,border_mode='same',activation='relu',name='inception_4d/5x5')(inception_4d_5x5_reduce)
inception_4d_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4d/pool')(inception_4c_output)
inception_4d_pool_proj = Convolution2D(64,1,1,border_mode='same',activation='relu',name='inception_4d/pool_proj')(inception_4d_pool)
inception_4d_output = merge([inception_4d_1x1,inception_4d_3x3,inception_4d_5x5,inception_4d_pool_proj],mode='concat',concat_axis=1,name='inception_4d/output')
loss2_ave_pool = AveragePooling2D(pool_size=(5,5),strides=(3,3),name='loss2/ave_pool')(inception_4d_output)
loss2_conv = Convolution2D(128,1,1,border_mode='same',activation='relu',name='loss2/conv')(loss2_ave_pool)
loss2_flat = Flatten()(loss2_conv)
loss2_fc = Dense(1024,activation='relu',name='loss2/fc' )(loss2_flat)
loss2_drop_fc = Dropout(0.7)(loss2_fc)
loss2_classifier = Dense(1000,name='loss2/classifier' )(loss2_drop_fc)
loss2_classifier_act = Activation('softmax')(loss2_classifier)
# Seventh Inception module
inception_4e_1x1 = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_4e/1x1' )(inception_4d_output)
inception_4e_3x3_reduce = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_4e/3x3_reduce' )(inception_4d_output)
inception_4e_3x3 = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_4e/3x3')(inception_4e_3x3_reduce)
inception_4e_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_4e/5x5_reduce' )(inception_4d_output)
inception_4e_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_4e/5x5' )(inception_4e_5x5_reduce)
inception_4e_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_4e/pool')(inception_4d_output)
inception_4e_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_4e/pool_proj')(inception_4e_pool)
inception_4e_output = merge([inception_4e_1x1,inception_4e_3x3,inception_4e_5x5,inception_4e_pool_proj],mode='concat',concat_axis=1,name='inception_4e/output')
inception_4e_output_zero_pad = ZeroPadding2D(padding=(1, 1))(inception_4e_output)
pool4_helper = PoolHelper()(inception_4e_output_zero_pad)
pool4_3x3_s2 = MaxPooling2D(pool_size=(3,3),strides=(2,2),border_mode='valid',name='pool4/3x3_s2')(pool4_helper)
# Eighth Inception module
inception_5a_1x1 = Convolution2D(256,1,1,border_mode='same',activation='relu',name='inception_5a/1x1' )(pool4_3x3_s2)
inception_5a_3x3_reduce = Convolution2D(160,1,1,border_mode='same',activation='relu',name='inception_5a/3x3_reduce' )(pool4_3x3_s2)
inception_5a_3x3 = Convolution2D(320,3,3,border_mode='same',activation='relu',name='inception_5a/3x3' )(inception_5a_3x3_reduce)
inception_5a_5x5_reduce = Convolution2D(32,1,1,border_mode='same',activation='relu',name='inception_5a/5x5_reduce' )(pool4_3x3_s2)
inception_5a_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_5a/5x5' )(inception_5a_5x5_reduce)
inception_5a_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_5a/pool')(pool4_3x3_s2)
inception_5a_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_5a/pool_proj' )(inception_5a_pool)
inception_5a_output = merge([inception_5a_1x1,inception_5a_3x3,inception_5a_5x5,inception_5a_pool_proj],mode='concat',concat_axis=1,name='inception_5a/output')
# Nineth Inception module
inception_5b_1x1 = Convolution2D(384,1,1,border_mode='same',activation='relu',name='inception_5b/1x1' )(inception_5a_output)
inception_5b_3x3_reduce = Convolution2D(192,1,1,border_mode='same',activation='relu',name='inception_5b/3x3_reduce' )(inception_5a_output)
inception_5b_3x3 = Convolution2D(384,3,3,border_mode='same',activation='relu',name='inception_5b/3x3' )(inception_5b_3x3_reduce)
inception_5b_5x5_reduce = Convolution2D(48,1,1,border_mode='same',activation='relu',name='inception_5b/5x5_reduce' )(inception_5a_output)
inception_5b_5x5 = Convolution2D(128,5,5,border_mode='same',activation='relu',name='inception_5b/5x5' )(inception_5b_5x5_reduce)
inception_5b_pool = MaxPooling2D(pool_size=(3,3),strides=(1,1),border_mode='same',name='inception_5b/pool')(inception_5a_output)
inception_5b_pool_proj = Convolution2D(128,1,1,border_mode='same',activation='relu',name='inception_5b/pool_proj' )(inception_5b_pool)
inception_5b_output = merge([inception_5b_1x1,inception_5b_3x3,inception_5b_5x5,inception_5b_pool_proj],mode='concat',concat_axis=1,name='inception_5b/output')
# DECONTRACTING PATH
# Unpool 1
score_pool4 = Convolution2D(nclasses, 1, 1, init, 'relu', border_mode='same',
name='score_pool4',
W_regularizer=l2(l2_reg))(x)
score2 = Deconvolution2D(nclasses, 4, 4, score_fr._keras_shape, init,
'linear', border_mode='valid', subsample=(2, 2),
name='score2', W_regularizer=l2(l2_reg))(score_fr)
score_pool4_crop = CropLayer2D(score2,
name='score_pool4_crop')(score_pool4)
score_fused = merge([score_pool4_crop, score2], mode=custom_sum,
output_shape=custom_sum_shape, name='score_fused')
# Unpool 2
score_pool3 = Convolution2D(nclasses, 1, 1, init, 'relu', border_mode='valid',
name='score_pool3',
W_regularizer=l2(l2_reg))(pool3)
score4 = Deconvolution2D(nclasses, 4, 4, score_fused._keras_shape, init,
'linear', border_mode='valid', subsample=(2, 2),
bias=True, # TODO: No bias??
name='score4', W_regularizer=l2(l2_reg))(score_fused)
score_pool3_crop = CropLayer2D(score4, name='score_pool3_crop')(score_pool3)
score_final = merge([score_pool3_crop, score4], mode=custom_sum,
output_shape=custom_sum_shape, name='score_final')
upsample = Deconvolution2D(nclasses, 16, 16, score_final._keras_shape, init,
'linear', border_mode='valid', subsample=(8, 8),
bias=False, # TODO: No bias??
name='upsample', W_regularizer=l2(l2_reg))(score_final)
score = CropLayer2D(inputs, name='score')(upsample)
# Softmax
predictions = NdSoftmax()(score)
## Decoder part ##
#First decoding block
#Final block
# pool5_7x7_s1 = AveragePooling2D(pool_size=(7,7),strides=(1,1),name='pool5/7x7_s2')(inception_5b_output)
# loss3_flat = Flatten()(pool5_7x7_s1)
# pool5_drop_7x7_s1 = Dropout(0.4)(loss3_flat)
# loss3_classifier = Dense(1000,name='loss3/classifier' )(pool5_drop_7x7_s1)
# predictions = Activation('softmax',name='prob')(loss3_classifier)
#googlenet = Model(input=input, output=[loss1_classifier_act,loss2_classifier_act,loss3_classifier_act])
return predictions
def DeeplabV2(input_shape, upsampling=8, apply_softmax=True,
weights='voc2012', input_tensor=None,
classes=21, weight_decay=0.0005):
"""Instantiate the DeeplabV2 architecture with VGG16 encoder,
optionally loading weights pre-trained on VOC2012 segmentation.
Note that pre-trained model is only available for Theano dim ordering.
The model and the weights should be compatible with both
TensorFlow and Theano backends.
# Arguments
input_shape: shape tuple. It should have exactly 3 inputs channels,
and the axis ordering should be coherent with what specified in
your keras.json (e.g. use (3, 512, 512) for 'th' and (512, 512, 3)
for 'tf').
upsampling: final front end upsampling (default is 8x).
apply_softmax: whether to apply softmax or return logits.
weights: one of `None` (random initialization)
or `voc2012` (pre-training on VOC2012 segmentation).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
classes: number of classes to classify images into
# Returns
A Keras model instance.
"""
if weights not in {'voc2012', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `voc2012` '
'(pre-training on VOC2012 segmentation).')
if weights == 'voc2012' and classes != 21:
raise ValueError('If using `weights` as voc2012 `classes` should be 21')
if input_shape is None:
raise ValueError('Please provide a valid input_shape to deeplab')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Block 1
h = ZeroPadding2D(padding=(1, 1))(img_input)
h = Convolution2D(64, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv1_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(64, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv1_2')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(h)
# Block 2
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(128, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv2_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(128, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv2_2')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(h)
# Block 3
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(256, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv3_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(256, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv3_2')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(256, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv3_3')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = MaxPooling2D(pool_size=(3, 3), strides=(2, 2))(h)
# Block 4
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(512, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv4_1')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(512, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv4_2')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = Convolution2D(512, 3, 3, activation='relu', W_regularizer=l2(weight_decay), name='conv4_3')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
h = MaxPooling2D(pool_size=(3, 3), strides=(1, 1))(h)
# Block 5
h = ZeroPadding2D(padding=(2, 2))(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_1')(h)
h = ZeroPadding2D(padding=(2, 2))(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_2')(h)
h = ZeroPadding2D(padding=(2, 2))(h)
h = AtrousConvolution2D(512, 3, 3, atrous_rate=(2, 2), activation='relu', name='conv5_3')(h)
h = ZeroPadding2D(padding=(1, 1))(h)
p5 = MaxPooling2D(pool_size=(3, 3), strides=(1, 1))(h)
#TODO: possible p5a = AveragePooling kernel_size=3, stride = 1 pad = 1
# branching for Atrous Spatial Pyramid Pooling
# hole = 6
b1 = ZeroPadding2D(padding=(6, 6))(p5)
b1 = AtrousConvolution2D(1024, 3, 3, atrous_rate=(6, 6), activation='relu', name='fc6_1')(b1)
b1 = Dropout(0.5)(b1)
b1 = Convolution2D(1024, 1, 1, activation='relu', name='fc7_1')(b1)
b1 = Dropout(0.5)(b1)
b1 = Convolution2D(classes, 1, 1, activation='relu', name='fc8_voc12_1')(b1)
# hole = 12
b2 = ZeroPadding2D(padding=(12, 12))(p5)
b2 = AtrousConvolution2D(1024, 3, 3, atrous_rate=(12, 12), activation='relu', name='fc6_2')(b2)
b2 = Dropout(0.5)(b2)
b2 = Convolution2D(1024, 1, 1, activation='relu', name='fc7_2')(b2)
b2 = Dropout(0.5)(b2)
b2 = Convolution2D(classes, 1, 1, activation='relu', name='fc8_voc12_2')(b2)
# hole = 18
b3 = ZeroPadding2D(padding=(18, 18))(p5)
b3 = AtrousConvolution2D(1024, 3, 3, atrous_rate=(18, 18), activation='relu', name='fc6_3')(b3)
b3 = Dropout(0.5)(b3)
b3 = Convolution2D(1024, 1, 1, activation='relu', name='fc7_3')(b3)
b3 = Dropout(0.5)(b3)
b3 = Convolution2D(classes, 1, 1, activation='relu', name='fc8_voc12_3')(b3)
# hole = 24
b4 = ZeroPadding2D(padding=(24, 24))(p5)
b4 = AtrousConvolution2D(1024, 3, 3, atrous_rate=(24, 24), activation='relu', name='fc6_4')(b4)
b4 = Dropout(0.5)(b4)
b4 = Convolution2D(1024, 1, 1, activation='relu', name='fc7_4')(b4)
b4 = Dropout(0.5)(b4)
b4 = Convolution2D(classes, 1, 1, activation='relu', name='fc8_voc12_4')(b4)
s = merge([b1, b2, b3, b4], mode='sum')
#logits = upsample_tf(factor=upsampling, input_img=s)
#logits = bilinear2D(ratio=upsampling)(s)
#upsampling=tf.div(s.get_shape(),(tf.TensorShape(input_shape)))
#logits = UpSampling2D(size=(upsampling, upsampling))(s)
logits = Deconvolution2D(classes, upsampling, upsampling, init=bilinear_init, subsample=(upsampling, upsampling), input_shape=s._keras_shape)(s)
if apply_softmax:
out = NdSoftmax()(logits)
else:
out = logits
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
#out = CRFLayer(classes)(logits)
# Create model.
model = Model(inputs, out, name='deeplabV2')
# load weights
if weights == 'voc2012':
if K.image_dim_ordering() == 'th':
weights_path = get_file('deeplabV2_weights_th.h5',
TH_WEIGHTS_PATH,
cache_subdir='models')
model.load_weights(weights_path)
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image dimension ordering convention '
'(`image_dim_ordering="th"`). '
'For best performance, set '
'`image_dim_ordering="tf"` in '
'your Keras config '
'at ~/.keras/keras.json.')
convert_all_kernels_in_model(model)
else:
raise NotImplementedError('Pretrained DeeplabV2 model is not available for'
'voc2012 dataset and tensorflow dim ordering')
return model