def build(self, use_cpu=False, print_summary=False):
vgg16 = VGG16(weights="imagenet",
include_top=False,
input_shape=(224, 224, 3))
if use_cpu:
device = '/cpu:0'
else:
device = '/gpu:0'
print('=' * 30 + 'Loading DeconvNet' + '=' * 30)
with tf.device(device):
inputs = Input(shape=(224, 224, 3), batch_size=self.batch_size)
print('=' * 40 + 'inputs' + '=' * 40 + '\n', inputs)
conv_block_1 = self.buildConv2DBlock(inputs, 64, 1, 2)
pool1, pool1_argmax = Lambda(self.max_pool_with_argmax,
name='pool1')(conv_block_1)
print('=' * 40 + 'pool1' + '=' * 40 + '\n', pool1)
conv_block_2 = self.buildConv2DBlock(pool1, 128, 2, 2)
pool2, pool2_argmax = Lambda(self.max_pool_with_argmax,
name='pool2')(conv_block_2)
print('=' * 40 + 'pool2' + '=' * 40 + '\n', pool2)
conv_block_3 = self.buildConv2DBlock(pool2, 256, 3, 3)
pool3, pool3_argmax = Lambda(self.max_pool_with_argmax,
name='pool3')(conv_block_3)
print('=' * 40 + 'pool3' + '=' * 40 + '\n', pool3)
conv_block_4 = self.buildConv2DBlock(pool3, 512, 4, 3)
pool4, pool4_argmax = Lambda(self.max_pool_with_argmax,
name='pool4')(conv_block_4)
print('=' * 40 + 'pool4' + '=' * 40 + '\n', pool4)
conv_block_5 = self.buildConv2DBlock(pool4, 512, 5, 3)
pool5, pool5_argmax = Lambda(self.max_pool_with_argmax,
name='pool5')(conv_block_5)
print('=' * 40 + 'pool5' + '=' * 40 + '\n', pool5)
fc6 = Conv2D(512, 7, use_bias=False, padding='valid',
name='fc6')(pool5) #4096
fc6 = BatchNormalization(name='batchnorm_fc6')(fc6)
fc6 = Activation('relu', name='relu_fc6')(fc6)
print('=' * 40 + 'fc6' + '=' * 40 + '\n', fc6)
fc7 = Conv2D(512, 1, use_bias=False, padding='valid',
name='fc7')(fc6) #4096
fc7 = BatchNormalization(name='batchnorm_fc7')(fc7)
fc7 = Activation('relu', name='relu_fc7')(fc7)
print('=' * 40 + 'fc7' + '=' * 40 + '\n', fc7)
x = Conv2DTranspose(512,
7,
use_bias=False,
padding='valid',
name='deconv-fc6')(fc7)
x = BatchNormalization(name='batchnorm_deconv-fc6')(x)
x = Activation('relu', name='relu_deconv-fc6')(x)
print('=' * 40 + 'relu_deconv-fc6' + '=' * 40 + '\n', x)
x = MaxUnpoolWithArgmax(pool5_argmax, name='unpool5')(x)
x.set_shape(conv_block_5.get_shape())
print('=' * 40 + 'unpool5' + '=' * 40 + '\n', x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv5-1')(x)
x = BatchNormalization(name='batchnorm_deconv5-1')(x)
x = Activation('relu', name='relu_deconv5-1')(x)
print('=' * 40 + 'relu_deconv5-1' + '=' * 40 + '\n', x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv5-2')(x)
x = BatchNormalization(name='batchnorm_deconv5-2')(x)
x = Activation('relu', name='relu_deconv5-2')(x)
print('=' * 40 + 'relu_deconv5-2' + '=' * 40 + '\n', x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv5-3')(x)
x = BatchNormalization(name='batchnorm_deconv5-3')(x)
x = Activation('relu', name='relu_deconv5-3')(x)
print('=' * 40 + 'relu_deconv5-3' + '=' * 40 + '\n', x)
x = MaxUnpoolWithArgmax(pool4_argmax, name='unpool4')(x)
x.set_shape(conv_block_4.get_shape())
print('=' * 40 + 'unpool4' + '=' * 40 + '\n', x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv4-1')(x)
x = BatchNormalization(name='batchnorm_deconv4-1')(x)
x = Activation('relu', name='relu_deconv4-1')(x)
print('=' * 40 + 'relu_deconv4-1' + '=' * 40 + '\n', x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv4-2')(x)
x = BatchNormalization(name='batchnorm_deconv4-2')(x)
x = Activation('relu', name='relu_deconv4-2')(x)
print('=' * 40 + 'relu_deconv4-2' + '=' * 40 + '\n', x)
x = Conv2DTranspose(256,
3,
use_bias=False,
padding='same',
name='deconv4-3')(x)
x = BatchNormalization(name='batchnorm_deconv4-3')(x)
x = Activation('relu', name='c3')(x)
print('=' * 40 + 'relu_deconv4-3' + '=' * 40 + '\n', x)
x = MaxUnpoolWithArgmax(pool3_argmax, name='unpool3')(x)
x.set_shape(conv_block_3.get_shape())
print('=' * 40 + 'unpool3' + '=' * 40 + '\n', x)
x = Conv2DTranspose(256,
3,
use_bias=False,
padding='same',
name='deconv3-1')(x)
x = BatchNormalization(name='batchnorm_deconv3-1')(x)
x = Activation('relu', name='relu_deconv3-1')(x)
print('=' * 40 + 'relu_deconv3-1' + '=' * 40 + '\n', x)
x = Conv2DTranspose(256,
3,
use_bias=False,
padding='same',
name='deconv3-2')(x)
x = BatchNormalization(name='batchnorm_deconv3-2')(x)
x = Activation('relu', name='relu_deconv3-2')(x)
print('=' * 40 + 'relu_deconv3-2' + '=' * 40 + '\n', x)
x = Conv2DTranspose(128,
3,
use_bias=False,
padding='same',
name='deconv3-3')(x)
x = BatchNormalization(name='batchnorm_deconv3-3')(x)
x = Activation('relu', name='relu_deconv3-3')(x)
print('=' * 40 + 'relu_deconv3-3' + '=' * 40 + '\n', x)
x = MaxUnpoolWithArgmax(pool2_argmax, name='unpool2')(x)
x.set_shape(conv_block_2.get_shape())
print('=' * 40 + 'unpool2' + '=' * 40 + '\n', x)
x = Conv2DTranspose(128,
3,
use_bias=False,
padding='same',
name='deconv2-1')(x)
x = BatchNormalization(name='batchnorm_deconv2-1')(x)
x = Activation('relu', name='relu_deconv2-1')(x)
print('=' * 40 + 'relu_deconv2-1' + '=' * 40 + '\n', x)
x = Conv2DTranspose(64,
3,
use_bias=False,
padding='same',
name='deconv2-2')(x)
x = BatchNormalization(name='batchnorm_deconv2-2')(x)
x = Activation('relu', name='relu_deconv2-2')(x)
print('=' * 40 + 'relu_deconv2-2' + '=' * 40 + '\n', x)
x = MaxUnpoolWithArgmax(pool1_argmax, name='unpool1')(x)
x.set_shape(conv_block_1.get_shape())
print('=' * 40 + 'unpool1' + '=' * 40 + '\n', x)
x = Conv2DTranspose(64,
3,
use_bias=False,
padding='same',
name='deconv1-1')(x)
x = BatchNormalization(name='batchnorm_deconv1-1')(x)
x = Activation('relu', name='relu_deconv1-1')(x)
print('=' * 40 + 'relu_deconv1-1' + '=' * 40 + '\n', x)
x = Conv2DTranspose(64,
3,
use_bias=False,
padding='same',
name='deconv1-2')(x)
x = BatchNormalization(name='batchnorm_deconv1-2')(x)
x = Activation('relu', name='relu_deconv1-2')(x)
print('=' * 40 + 'relu_deconv1-2' + '=' * 40 + '\n', x)
output = Conv2DTranspose(self.labels,
1,
activation='softmax',
padding='same',
name='output')(x)
print('=' * 40 + 'output' + '=' * 40 + '\n', output)
self.model = Model(inputs=inputs, outputs=output)
vgg16 = VGG16(weights="imagenet",
include_top=False,
input_shape=(224, 224, 3))
if print_summary:
print(self.model.summary())
for layer in self.model.layers:
if layer.name.startswith('conv'):
block = layer.name[4:].split('-')[0]
depth = layer.name[4:].split('-')[1]
# apply vgg16 weights without bias
layer.set_weights([
vgg16.get_layer('block{}_conv{}'.format(
block, depth)).get_weights()[0]
])
#sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
#rmsprop=RMSprop(lr=0.001, rho=0.9, epsilon=1e-06)
#adagrad=Adagrad(lr=0.01, epsilon=1e-06)
adadelta = Adadelta(lr=1.0, rho=0.95, epsilon=1e-06)
self.model.compile(optimizer=adadelta,
loss='categorical_crossentropy',
metrics=['accuracy', 'mse'])
def __init__(self,
num_classes=19,
output_stride=16,
backbonetype='mobilenetv2',
weights='imagenet',
dl_input_shape=(None, 483, 769, 3),
weight_decay=0.00004,
pooling='global',
residual_shortcut=False):
super(CMSNet, self).__init__(name='cmsnet')
"""
:param num_classes: (Default value = 19)
:param output_stride: (Default value = 16)
if strid count is 4 remove stride from block 13 and inser atrous in 14, 15 and 16
if strid count is 3 remove stride from block 6/13 and inser atrous rate 2 in 7-13/ and rate 4 14-16
:param backbonetype: (Default value = 'mobilenetv2')
:param weights: (Default value = 'imagenet')
:param input_shape: (Default value = (None, 483,769,3)
:param weight_decay: use 0.00004 for MobileNet-V2 or Xcpetion model backbonetype. Use 0.0001 for ResNet backbonetype.
"""
self.logger = logging.getLogger('perception.models.CMSNet')
self.logger.info('creating an instance of CMSNet with backbone ' +
backbonetype + ', OS' + str(output_stride) +
', nclass=' + str(num_classes) + ', input=' +
str(dl_input_shape) + ', pooling=' + pooling +
', residual=' + str(residual_shortcut))
self.num_classes = num_classes
self.output_stride = output_stride
self.dl_input_shape = dl_input_shape
self._createBackbone(backbonetype=backbonetype,
output_stride=output_stride)
# All with 256 filters and batch normalization.
# one 1×1 convolution and three 3×3 convolutions with rates = (6, 12, 18) when output stride = 16.
# Rates are doubled when output stride = 8.
#Create Spatial Pyramid Pooling
x = self.backbone.output
pooling_shape = self.backbone.compute_output_shape(self.dl_input_shape)
pooling_shape_float = tf.cast(pooling_shape[1:3], tf.float32)
assert pooling in [
'aspp', 'spp', 'global'
], "Only suported pooling= 'aspp', 'spp' or 'global'."
if pooling == 'aspp':
if output_stride == 16:
rates = (6, 12, 18)
elif output_stride == 8:
rates = (12, 24, 36)
#gride lavel: pooling
x0 = Conv2D(filters=256,
kernel_size=3,
name='aspp_0_expand',
padding="same",
dilation_rate=rates[0],
kernel_regularizer=l2(weight_decay))(x)
x0 = BatchNormalization(name='aspp_0_expand_BN')(x0) #epsilon=1e-5
x0 = ReLU(name='aspp_0_expand_relu')(x0)
x1 = Conv2D(filters=256,
kernel_size=3,
name='aspp_1_expand',
padding="same",
dilation_rate=rates[1],
kernel_regularizer=l2(weight_decay))(x)
x1 = BatchNormalization(name='aspp_1_expand_BN')(x1) #epsilon=1e-5
x1 = ReLU(name='aspp_1_expand_relu')(x1)
x2 = Conv2D(filters=256,
kernel_size=3,
name='aspp_2_expand',
padding="same",
dilation_rate=rates[2],
kernel_regularizer=l2(weight_decay))(x)
x2 = BatchNormalization(name='aspp_2_expand_BN')(x2) #epsilon=1e-5
x2 = ReLU(name='aspp_2_expand_relu')(x2)
#gride lavel: all
xn = Conv2D(filters=256,
kernel_size=1,
name='aspp_n_expand',
kernel_regularizer=l2(weight_decay))(x)
xn = BatchNormalization(name='aspp_n_expand_BN')(xn) #epsilon=1e-5
xn = ReLU(name='aspp_n_expand_relu')(xn)
#Concatenate spatial pyramid pooling
x0.set_shape(pooling_shape[0:3].concatenate(x0.get_shape()[-1]))
x1.set_shape(pooling_shape[0:3].concatenate(x1.get_shape()[-1]))
x2.set_shape(pooling_shape[0:3].concatenate(x2.get_shape()[-1]))
xn.set_shape(pooling_shape[0:3].concatenate(xn.get_shape()[-1]))
x = Concatenate(name='aspp_concatenate')([x0, x1, x2, xn])
elif pooling == 'spp':
rates = (1, 2, 3, 6)
#gride lavel: pooling
x0 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[0],
tf.int32),
padding="valid",
name='spp_0_average_pooling2d')(x)
x0 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_0_expand',
kernel_regularizer=l2(weight_decay))(x0)
x0 = BatchNormalization(name='spp_0_expand_BN')(x0) #epsilon=1e-5
x0 = ReLU(name='spp_0_expand_relu')(x0)
if tf.__version__.split('.')[0] == '1':
x0 = Lambda(lambda x0: tf.image.resize_bilinear(
x0, pooling_shape[1:3], align_corners=True),
name='spp_0_resize_bilinear')(x0)
else:
x0 = Lambda(lambda x0: tf.image.resize(x0,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_0_resize_bilinear')(x0)
x1 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[1],
tf.int32),
padding="valid",
name='spp_1_average_pooling2d')(x)
x1 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_1_expand',
kernel_regularizer=l2(weight_decay))(x1)
x1 = BatchNormalization(name='spp_1_expand_BN')(x1) #epsilon=1e-5
x1 = ReLU(name='spp_1_expand_relu')(x1)
if tf.__version__.split('.')[0] == '1':
x1 = Lambda(lambda x1: tf.image.resize_bilinear(
x1, pooling_shape[1:3], align_corners=True),
name='spp_1_resize_bilinear')(x1)
else:
x1 = Lambda(lambda x1: tf.image.resize(x1,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_1_resize_bilinear')(x1)
x2 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[2],
tf.int32),
padding="valid",
name='spp_2_average_pooling2d')(x)
x2 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_2_expand',
kernel_regularizer=l2(weight_decay))(x2)
x2 = BatchNormalization(name='spp_2_expand_BN')(x2) #epsilon=1e-5
x2 = ReLU(name='spp_2_expand_relu')(x2)
if tf.__version__.split('.')[0] == '1':
x2 = Lambda(lambda x2: tf.image.resize_bilinear(
x2, pooling_shape[1:3], align_corners=True),
name='spp_2_resize_bilinear')(x2)
else:
x2 = Lambda(lambda x2: tf.image.resize(x2,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_2_resize_bilinear')(x2)
x3 = AvgPool2D(pool_size=tf.cast(pooling_shape_float / rates[3],
tf.int32),
padding="valid",
name='spp_3_average_pooling2d')(x)
x3 = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_3_expand',
kernel_regularizer=l2(weight_decay))(x3)
x3 = BatchNormalization(name='spp_3_expand_BN')(x3) #epsilon=1e-5
x3 = ReLU(name='spp_3_expand_relu')(x3)
if tf.__version__.split('.')[0] == '1':
x3 = Lambda(lambda x3: tf.image.resize_bilinear(
x3, pooling_shape[1:3], align_corners=True),
name='spp_3_resize_bilinear')(x3)
else:
x3 = Lambda(lambda x3: tf.image.resize(x3,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_3_resize_bilinear')(x3)
#gride lavel: all
xn = Conv2D(filters=int(pooling_shape[-1] / len(rates)),
kernel_size=1,
name='spp_n_expand',
kernel_regularizer=l2(weight_decay))(x)
xn = BatchNormalization(name='spp_n_expand_BN')(xn) #epsilon=1e-5
xn = ReLU(name='spp_n_expand_relu')(xn)
#Concatenate spatial pyramid pooling
xn.set_shape(pooling_shape[0:3].concatenate(xn.get_shape()[-1]))
x = Concatenate(name='spp_concatenate')([x0, x1, x2, xn])
elif pooling == 'global':
#gride lavel: pooling
x0 = AvgPool2D(pool_size=pooling_shape[1:3],
padding="valid",
name='spp_0_average_pooling2d')(x)
x0 = Conv2D(filters=256,
kernel_size=1,
name='spp_0_expand',
kernel_regularizer=l2(weight_decay))(x0)
x0 = BatchNormalization(name='spp_0_expand_BN')(x0) #epsilon=1e-5
x0 = ReLU(name='spp_0_expand_relu')(x0)
# x0 = tf.image.resize(x0,
# size=pooling_shape[1:3],
# method=tf.image.ResizeMethod.BILINEAR, name='spp_0_resize_bilinear')
if tf.__version__.split('.')[0] == '1':
x0 = Lambda(lambda x0: tf.image.resize_bilinear(
x0, pooling_shape[1:3], align_corners=True),
name='spp_0_resize_bilinear')(x0)
else:
x0 = Lambda(lambda x0: tf.image.resize(x0,
pooling_shape[1:3],
method=tf.image.
ResizeMethod.BILINEAR),
name='spp_0_resize_bilinear')(x0)
#gride lavel: all
xn = Conv2D(filters=256,
kernel_size=1,
name='spp_1_expand',
kernel_regularizer=l2(weight_decay))(x)
xn = BatchNormalization(name='spp_1_expand_BN')(xn) #epsilon=1e-5
xn = ReLU(name='spp_1_expand_relu')(xn)
#Concatenate spatial pyramid pooling
xn.set_shape(pooling_shape[0:3].concatenate(xn.get_shape()[-1]))
x = Concatenate(name='spp_concatenate')([x0, xn])
#Concate Projection
x = Conv2D(filters=256,
kernel_size=1,
name='spp_concat_project',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name='spp_concat_project_BN')(x) #epsilon=1e-5
x = ReLU(name='spp_concat_project_relu')(x)
if residual_shortcut:
assert output_stride == 16, "For while residual shotcut is available for atous with os16."
#self.strideOutput8LayerName #block_6_project_BN (BatchNormal (None, 61, 97, 64)
os8_shape = self.backbone.get_layer(
self.strideOutput8LayerName).output_shape
os8_output = self.backbone.get_layer(
self.strideOutput8LayerName).output
x = Conv2D(filters=os8_shape[-1],
kernel_size=1,
name='shotcut_2x_conv',
kernel_regularizer=l2(weight_decay))(x)
x = BatchNormalization(name='shotcut_2x_BN')(x) #epsilon=1e-5
if tf.__version__.split('.')[0] == '1':
x = Lambda(lambda x: tf.image.resize_bilinear(
x, os8_shape[1:3], align_corners=True),
name='shotcut_2x_bilinear')(x)
else:
x = Lambda(lambda x: tf.image.resize(
x, os8_shape[1:3], method=tf.image.ResizeMethod.BILINEAR),
name='shotcut_2x_bilinear')(x)
x = ReLU(name='shotcut_2x_relu')(x)
x = Add(name='shotcut_2x_add')([x, os8_output])
x = Dropout(rate=0.1, name='dropout')(x)
#Semantic Segmentation
x = Conv2D(filters=num_classes,
kernel_size=1,
name='segmentation',
kernel_regularizer=l2(weight_decay))(x)
#x = BatchNormalization(name='segmentation_BN')(x)
# x = tf.image.resize(x, size=self.dl_input_shape[1:3],
# method=tf.image.ResizeMethod.BILINEAR, name='segmentation_bilinear')
if tf.__version__.split('.')[0] == '1':
x = Lambda(lambda x: tf.image.resize_bilinear(
x, self.dl_input_shape[1:3], align_corners=True),
name='segmentation_bilinear')(x)
else:
x = Lambda(lambda x: tf.image.resize(x,
self.dl_input_shape[1:3],
method=tf.image.ResizeMethod.
BILINEAR),
name='segmentation_bilinear')(x)
x = Softmax(name='logistic_softmax')(x)
#logist to training
#argmax
super(CMSNet, self).__init__(inputs=self.backbone.input,
outputs=x,
name='cmsnet')
def build(self, use_cpu=False, print_summary=False):
vgg16 = VGG16(weights="imagenet",
include_top=False,
input_shape=(224, 224, 3))
if use_cpu:
device = '/cpu:0'
else:
device = '/gpu:0'
with tf.device(device):
inputs = Input(shape=(224, 224, 3))
conv_block_1 = self.buildConv2DBlock(inputs, 64, 1, 2)
pool1, pool1_argmax = Lambda(self.max_pool_with_argmax,
name='pool1')(conv_block_1)
conv_block_2 = self.buildConv2DBlock(pool1, 128, 2, 2)
pool2, pool2_argmax = Lambda(self.max_pool_with_argmax,
name='pool2')(conv_block_2)
conv_block_3 = self.buildConv2DBlock(pool2, 256, 3, 3)
pool3, pool3_argmax = Lambda(self.max_pool_with_argmax,
name='pool3')(conv_block_3)
conv_block_4 = self.buildConv2DBlock(pool3, 512, 4, 3)
pool4, pool4_argmax = Lambda(self.max_pool_with_argmax,
name='pool4')(conv_block_4)
conv_block_5 = self.buildConv2DBlock(pool4, 512, 5, 3)
pool5, pool5_argmax = Lambda(self.max_pool_with_argmax,
name='pool5')(conv_block_5)
fc6 = Conv2D(512, 7, use_bias=False, padding='valid',
name='fc6')(pool5) #4096
fc6 = BatchNormalization(name='batchnorm_fc6')(fc6)
fc6 = Activation('relu', name='relu_fc6')(fc6)
fc7 = Conv2D(512, 1, use_bias=False, padding='valid',
name='fc7')(fc6) #4096
fc7 = BatchNormalization(name='batchnorm_fc7')(fc7)
fc7 = Activation('relu', name='relu_fc7')(fc7)
x = Conv2DTranspose(512,
7,
use_bias=False,
padding='valid',
name='deconv-fc6')(fc7)
x = BatchNormalization(name='batchnorm_deconv-fc6')(x)
x = Activation('relu', name='relu_deconv-fc6')(x)
x = MaxUnpoolWithArgmax(pool5_argmax, name='unpool5')(x)
x.set_shape(conv_block_5.get_shape())
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv5-1')(x)
x = BatchNormalization(name='batchnorm_deconv5-1')(x)
x = Activation('relu', name='relu_deconv5-1')(x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv5-2')(x)
x = BatchNormalization(name='batchnorm_deconv5-2')(x)
x = Activation('relu', name='relu_deconv5-2')(x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv5-3')(x)
x = BatchNormalization(name='batchnorm_deconv5-3')(x)
x = Activation('relu', name='relu_deconv5-3')(x)
x = MaxUnpoolWithArgmax(pool4_argmax, name='unpool4')(x)
x.set_shape(conv_block_4.get_shape())
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv4-1')(x)
x = BatchNormalization(name='batchnorm_deconv4-1')(x)
x = Activation('relu', name='relu_deconv4-1')(x)
x = Conv2DTranspose(512,
3,
use_bias=False,
padding='same',
name='deconv4-2')(x)
x = BatchNormalization(name='batchnorm_deconv4-2')(x)
x = Activation('relu', name='relu_deconv4-2')(x)
x = Conv2DTranspose(256,
3,
use_bias=False,
padding='same',
name='deconv4-3')(x)
x = BatchNormalization(name='batchnorm_deconv4-3')(x)
x = Activation('relu', name='relu_deconv4-3')(x)
x = MaxUnpoolWithArgmax(pool3_argmax, name='unpool3')(x)
x.set_shape(conv_block_3.get_shape())
x = Conv2DTranspose(256,
3,
use_bias=False,
padding='same',
name='deconv3-1')(x)
x = BatchNormalization(name='batchnorm_deconv3-1')(x)
x = Activation('relu', name='relu_deconv3-1')(x)
x = Conv2DTranspose(256,
3,
use_bias=False,
padding='same',
name='deconv3-2')(x)
x = BatchNormalization(name='batchnorm_deconv3-2')(x)
x = Activation('relu', name='relu_deconv3-2')(x)
x = Conv2DTranspose(128,
3,
use_bias=False,
padding='same',
name='deconv3-3')(x)
x = BatchNormalization(name='batchnorm_deconv3-3')(x)
x = Activation('relu', name='relu_deconv3-3')(x)
x = MaxUnpoolWithArgmax(pool2_argmax, name='unpool2')(x)
x.set_shape(conv_block_2.get_shape())
x = Conv2DTranspose(128,
3,
use_bias=False,
padding='same',
name='deconv2-1')(x)
x = BatchNormalization(name='batchnorm_deconv2-1')(x)
x = Activation('relu', name='relu_deconv2-1')(x)
x = Conv2DTranspose(64,
3,
use_bias=False,
padding='same',
name='deconv2-2')(x)
x = BatchNormalization(name='batchnorm_deconv2-2')(x)
x = Activation('relu', name='relu_deconv2-2')(x)
x = MaxUnpoolWithArgmax(pool1_argmax, name='unpool1')(x)
x.set_shape(conv_block_1.get_shape())
x = Conv2DTranspose(64,
3,
use_bias=False,
padding='same',
name='deconv1-1')(x)
x = BatchNormalization(name='batchnorm_deconv1-1')(x)
x = Activation('relu', name='relu_deconv1-1')(x)
x = Conv2DTranspose(64,
3,
use_bias=False,
padding='same',
name='deconv1-2')(x)
x = BatchNormalization(name='batchnorm_deconv1-2')(x)
x = Activation('relu', name='relu_deconv1-2')(x)
output = Conv2DTranspose(21,
1,
activation='softmax',
padding='same',
name='output')(x)
self.model = Model(inputs=inputs, outputs=output)
vgg16 = VGG16(weights="imagenet",
include_top=False,
input_shape=(224, 224, 3))
if print_summary:
print(self.model.summary())
for layer in self.model.layers:
if layer.name.startswith('conv'):
block = layer.name[4:].split('-')[0]
depth = layer.name[4:].split('-')[1]
# apply vgg16 weights without bias
layer.set_weights([
vgg16.get_layer('block{}_conv{}'.format(
block, depth)).get_weights()[0]
])
self.model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy', 'mse'])
def u_net(input_dim,
target_dim,
pool,
strides=1,
skips=True,
skip_type='concat',
batch_norm=False,
dropout=0.5,
dropout_allLayers=False,
layer_activation='relu',
out_activation='softmax',
filterdims=[],
filterdims_out=3,
latentdims=[64, 3],
prefilterdims=[],
separable_convs=False,
l2_pen=0,
per_image_standardization=False):
"""U-net model
Args:
input_dim : dimensions of the input data (x,x)
target_dim: dimensions of the target data (x,x)
strides: stride of the first convolution per conv block in encoder;
& stride of the last convolution per conv block in decoder;
pool: maxpool dimension
skips: True or false; use skip connections or not
skip_type: 'sum' or 'concat'
batch_norm: True or False; apply batch normalization
dropout: dropout on the fully connected layers
dropout_allLayers: use dropout for each layer block or only the latent dimension
layer_activation: type of activation between conv and batch_norm (e.g. 'relu', 'LeakyReLU')
out_activation: type of activation at the output (e.g. 'sigmoid', or None)
filterdims: filter dimensionality matrix
(e.g. for 3 layers: [[Nfeat1,dim1],[Nfeat2,dim2],[Nfeat3,dim3]])
filterdims_out: kernel dimensions of the filter at the output layer
latent_dims: latent filter dimensions
prefilterdims: optional set of filters before the encoder
separable_convs: use depthwise separable convolutions rather than regular convs [Xception: Deep Learning with Depthwise Separable Convolutions]
l2_pen: l2 penalty on the convolutional weights
per_image_standardization: normalize input images to zero mean unity variance
Returns:
keras model
"""
inputs = Input(shape=input_dim)
input_layer = inputs
# make input dimensions compatible with the network; i.e. add a channel dim if neccesary
if len(np.shape(input_layer)) < 4:
input_layer = Lambda(lambda x: K.expand_dims(x))(input_layer)
if per_image_standardization == True:
input_layer = Lambda(
standardization,
output_shape=standardization_output_shape)(input_layer)
if filterdims == []:
filterdims = [[16, 3], [32, 5], [64, 5]]
if len(target_dim) < 3:
n_classes = 1
else:
n_classes = target_dim[-1]
last_layer = input_layer
"""
=============================================================================
PREFILTER LAYER
=============================================================================
"""
for i in range(0, np.size(prefilterdims, 0)):
if separable_convs:
conv_pre = SeparableConv2D(
prefilterdims[i][0],
prefilterdims[i][1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='preconv{}'.format(i))
else:
conv_pre = Conv2D(prefilterdims[i][0],
prefilterdims[i][1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='preconv{}'.format(i))
conv = last_layer
if batch_norm == True:
conv = BatchNormalization()(conv)
conv = conv_pre(conv)
print("conv shape :", conv.shape)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
if dropout_allLayers and dropout > 0:
conv = Dropout(dropout)(conv)
print("dropout layer")
last_layer = conv
"""
=============================================================================
ENCODER
=============================================================================
"""
convs = []
convs_a = []
convs_b = []
pools = []
for i in range(0, np.size(filterdims, 0)):
if separable_convs:
conv_a = SeparableConv2D(
filterdims[i][0],
filterdims[i][1],
strides=strides,
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='conv{}a'.format(i))
conv_b = SeparableConv2D(
filterdims[i][0],
filterdims[i][1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='conv{}b'.format(i))
else:
conv_a = Conv2D(filterdims[i][0],
filterdims[i][1],
strides=strides,
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='conv{}a'.format(i))
conv_b = Conv2D(filterdims[i][0],
filterdims[i][1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='conv{}b'.format(i))
conv = last_layer
if batch_norm == True:
conv = BatchNormalization()(conv)
conv = conv_a(conv)
print("conv shape :", conv.shape)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
if batch_norm == True:
conv = BatchNormalization()(conv)
conv = conv_b(conv)
print("conv shape :", conv.shape)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
convs.append(conv)
pools.append(
MaxPooling2D(pool_size=pool[i], name='maxpool{}'.format(i))(conv))
print("pool shape :", pools[i].shape)
last_layer = pools[-1]
if dropout_allLayers and dropout > 0:
last_layer = Dropout(dropout)(last_layer)
print("dropout layer")
"""
=============================================================================
LATENT LAYER
=============================================================================
"""
if len(latentdims) == 2:
if separable_convs:
conv_latent = SeparableConv2D(
latentdims[0],
latentdims[1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='Conv1latent_space')(last_layer)
else:
conv_latent = Conv2D(latentdims[0],
latentdims[1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='Conv1latent_space')(last_layer)
print("conv shape :", conv_latent.shape)
if layer_activation == 'LeakyReLU':
conv_latent = LeakyReLU(alpha=0.1)(conv_latent)
else:
conv_latent = Activation(layer_activation)(conv_latent)
if dropout > 0:
conv_latent = Dropout(dropout)(conv_latent)
print("dropout layer")
if separable_convs:
conv_latent = SeparableConv2D(
latentdims[0],
latentdims[1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='Conv2latent_space')(conv_latent)
else:
conv_latent = Conv2D(latentdims[0],
latentdims[1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='Conv2latent_space')(conv_latent)
print("conv shape :", conv_latent.shape)
if layer_activation == 'LeakyReLU':
conv_latent = LeakyReLU(alpha=0.1)(conv_latent)
else:
conv_latent = Activation(layer_activation)(conv_latent)
else:
conv_latent = last_layer
print("skipping latent layer..")
if dropout > 0:
conv_latent = Dropout(dropout)(conv_latent)
print("dropout layer")
last_layer = conv_latent
"""
=============================================================================
DECODER
=============================================================================
"""
filterdims = filterdims[::-1]
for i in range(0, np.size(filterdims, 0)):
# 'learned' upsampling (nearest neighbor interpolation with 2x2, followed by conv of 2x2 )
# up = Conv2DTranspose(filterdims[i][0], 2, activation = None, padding = 'same', kernel_initializer = 'he_normal')(UpSampling2D(size = (2,2))(last_layer))
up = UpSampling2D(name='upsample{}'.format(i),
size=pool[-i - 1])(last_layer)
print("up shape :", up.shape)
if skips == True:
# Skip connections
if skip_type == 'concat':
merged = concatenate([convs[-1 - i], up], 3)
else:
merged = Add(name='Skip-connection{}'.format(i))(
[convs[-1 - i], up])
else:
merged = up
print("merge shape:", merged.shape)
if batch_norm == True:
conv = BatchNormalization()(conv)
shape_in = merged.shape.as_list()
layer = Conv2DTranspose(filterdims[i][0],
filterdims[i][1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='deconv{}a'.format(i))
conv = layer(merged)
shape_out = layer.compute_output_shape(shape_in)
conv.set_shape(shape_out)
print("conv shape :", conv.shape)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
if batch_norm == True:
conv = BatchNormalization()(conv)
if i < np.size(filterdims, 0) - 1:
shape_in = merged.shape.as_list()
layer = Conv2DTranspose(filterdims[i + 1][0],
filterdims[i + 1][1],
strides=strides,
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='deconv{}b'.format(i))
conv = layer(conv)
shape_out = layer.compute_output_shape(shape_in)
conv.set_shape(shape_out)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
print("conv shape :", conv.shape)
last_layer = conv
if dropout_allLayers and dropout > 0:
last_layer = Dropout(dropout)(last_layer)
print("dropout layer")
else: # last layer:
conv = Conv2DTranspose(filterdims[i][0],
filterdims[i][1],
activation=None,
strides=strides,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='deconv{}b'.format(i))(conv)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
conv = Conv2DTranspose(filterdims[i][0],
filterdims[i][1],
activation=None,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='deconv{}c'.format(i))(conv)
if layer_activation == 'LeakyReLU':
conv = LeakyReLU(alpha=0.1)(conv)
else:
conv = Activation(layer_activation)(conv)
final_layer = Conv2D(n_classes,
filterdims_out,
activation=out_activation,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(l2_pen),
name='Final_conv')(conv)
print("conv shape :", conv.shape)
final_layer = Reshape(target_dim)(final_layer)
model = Model(inputs=inputs, outputs=final_layer)
return model
