def create_decoder(self, backbone, skips):
stage5 = backbone.output # resolution 1/32
stage4 = skips[0] # resolution 1/16
stage3 = skips[1] # resolution 1/8
stage2 = skips[2] # resolution 1/4
# Pyramid pooling module for stage 5 tensor
stage5 = ppm_block(stage5, (1, 2, 3, 6), 128, 512)
# channel controllers
skip4 = conv2d(stage4, 256, 1, 1, kernel_size=1, use_relu=True)
skip3 = conv2d(stage3, 128, 1, 1, kernel_size=1, use_relu=True)
skip2 = conv2d(stage2, 64, 1, 1, kernel_size=1, use_relu=True)
# fusion nodes
fusion4 = fusion_node(stage5, skip4)
fusion3 = fusion_node(fusion4, skip3)
fusion2 = fusion_node(fusion3, skip2)
print(fusion2.shape)
# fusion nodes merging
merge = merge_block(fusion4, fusion3, fusion2,
out_channels=self.config.model.classes)
print(merge.shape)
# upsample to the right dimensions
upsampled = resize_img(
merge, self.config.model.height, self.config.model.width)
prediction = Activation('softmax', dtype='float32')(upsampled)
return prediction
def create_decoder(self, backbone, skips):
stage5 = backbone.output # resolution 1/32
stage4 = skips[0] # resolution 1/16
stage3 = skips[1] # resolution 1/8
stage2 = skips[2] # resolution 1/4
stage1 = skips[3] # resolution 1/2
# Pyramid pooling module for stage 5 tensor
stage5 = ppm_block(stage5, (1, 2, 3, 6), 128, 512)
# channel controllers
skip4 = conv2d(stage4, 254, 1, 1, kernel_size=1, use_relu=True)
skip3 = conv2d(stage3, 128, 1, 1, kernel_size=1, use_relu=True)
skip2 = conv2d(stage2, 64, 1, 1, kernel_size=1, use_relu=True)
skip1 = conv2d(stage1, 1, 1, 1, kernel_size=1, use_relu=True)
# fusion nodes
fusion = fusion_node(stage5, skip4)
fusion = fusion_node(fusion, skip3)
fusion = fusion_node(fusion, skip2)
fusion = fusion_node(fusion, skip1)
print(fusion.shape)
# upsample to the right dimensions
upsampled = resize_img(fusion, self.config.model.height,
self.config.model.width)
return upsampled
class FastScnnNyuv2(BaseModel):
def __init__(self, config):
super(FastScnnNyuv2, self).__init__(config, datagen)
def build_model(self):
# channels in beginning
input_channels = 1 if self.config.model.img_type == 'grayscale' else 3
# input layer
input_layer = Input(shape=(self.config.model.height,
self.config.model.width, input_channels), name="input_layer")
print(input_layer.shape)
# Learning to down-sample module
print("LEARNING TO DOWNSAMPLE")
ltd1 = conv2d(input_layer, 32, 2, 1)
print(ltd1.shape)
ltd2 = ds_conv2d(ltd1, 48, 2, 1)
print(ltd2.shape)
ltd3 = ds_conv2d(ltd2, 64, 2, 1)
print(ltd3.shape)
print("GLOBAL FEATURE EXTRACTOR")
# Adding the res blocks
gfe1 = bottleneck(ltd3, 3, 6, 64, 2)
print(gfe1.shape)
gfe2 = bottleneck(gfe1, 3, 6, 96, 2)
print(gfe2.shape)
gfe3 = bottleneck(gfe2, 3, 6, 128, 1)
print(gfe3.shape)
# adding the PPM module into layers
gfe4 = ppm_block(gfe3, [2, 4, 6, 8], 20, 15, 128)
print(gfe4.shape)
ffm = ffm_block(gfe4, ltd3, 128)
print(ffm.shape)
print("CLASSIFIER")
# classifier block method
class1 = ds_conv2d(ffm, 128, 1, 2, kernel_size=3)
print(class1.shape)
class2 = conv2d(class1, self.config.model.classes,
1, 1, kernel_size=3, use_relu=True)
print(class2.shape)
class3 = resize_img(class2, H=480, W=640)
print(class3.shape)
prediction = keras.activations.softmax(class3)
print(prediction.shape)
fast_scnn = keras.Model(
inputs=input_layer, outputs=prediction, name="FAST_SCNN")
def segmentation_head(self, features):
features = conv2d(features, 64, 1, 1, kernel_size=3,
name="segmentation_head_3x3")
features = conv2d(features, self.config.model.classes,
1, 1, kernel_size=1, name="segmentation_head_1x1")
upsampled = resize_img(
features, self.config.model.height, self.config.model.width)
segmentation_mask = Activation(
'softmax', name='seg_out')(upsampled)
return segmentation_mask
def depth_head(self, P2, P3, P4, P5, name="dep_head_"):
features = merge_block(P5, P4, P3, P2, 64, name=name + "merge_block_")
features = Conv2D(1, (3, 3),
strides=(1, 1),
padding="same",
name=name + "conv")(features)
upsampled = resize_img(features,
self.config.model.height,
self.config.model.width,
name=name + "out")
return upsampled
def segmentation_head(self, P2, P3, P4, P5, name="seg_head_"):
features = merge_block(P5, P4, P3, P2, 64, name=name + "merge_block_")
features = conv2d(features,
self.config.model.classes,
1,
1,
kernel_size=1,
use_bn=False,
name=name + "conv1x1")
upsampled = resize_img(features, self.config.model.height,
self.config.model.width)
segmentation_mask = Activation('softmax', name=name + "out")(upsampled)
return segmentation_mask
def create_decoder(self, backbone):
# reshape stage 5 to have convol filters
stage5 = backbone.output # resolution 1/32
print(stage5.shape)
stage5 = Reshape((
self.config.model.height//4,
self.config.model.width//4,
20
))(stage5)
print(stage5.shape)
stage5 = conv2d(stage5, 256, 1, 1)
stage5 = conv2d(stage5, 128, 1, 1)
stage5 = conv2d(stage5, 64, 1, 1)
# stage5 = ppm_block(stage5, (1, 2, 3, 6), 128, 1024)
# filters = Reshape((3, 3, 32, -1), dtype=tf.float32)(stage5)
# input = backbone.input # resolution 1/1
# input = conv2d(input, 32, 2, 1, kernel_size=1, use_relu=True)
# convolception = tf.map_fn(
# lambda x: tf.nn.conv2d(x[0], x[1], 1, padding="SAME"),
# (input, filters),
# dtype=(tf.float32, tf.float32)
# )
# convolception = tf.map_fn(
# lambda x: x,
# filters,
# dtype=tf.float32
# )
# print(convolception.shape)
# exit()
# convolception = tf.nn.conv2d(
# input, filters, strides=1, padding="SAME", name="convolception")
prediction = conv2d(
stage5, self.config.model.classes, 1, 1, kernel_size=1, use_relu=True)
# upsample to the right dimensions
prediction = resize_img(
prediction, self.config.model.height, self.config.model.width)
prediction = Activation('softmax', dtype='float32')(prediction)
return prediction
def depth_head(self, features):
features = conv2d(features,
64,
1,
1,
kernel_size=3,
name="depth_head_3x3")
features = conv2d(features,
1,
1,
1,
kernel_size=1,
name="depth_head_1x1")
upsampled = resize_img(features,
self.config.model.height,
self.config.model.width,
name="dep_out")
return upsampled
def merge_mini_decoders(self, mini_decoders):
"""
Merge all mini-decoders' outputs
"""
merge = concatenate(mini_decoders, name="merge_block_concatenate")
merge = conv2d(merge,
64,
1,
1,
kernel_size=3,
name="merge_block_conv3x3")
merge = conv2d(merge,
self.config.model.classes,
1,
1,
kernel_size=3,
name="merge_block_conv3x3_final")
merge = Activation('softmax', name="Main_Softmax")(merge)
merge = resize_img(merge,
self.config.model.height,
self.config.model.width,
name="Main_mask")
return merge
def mini_decoder(self, P2, P3, P4, P5, class_name, P2_filters, P3_filters,
P4_filters):
P5 = conv2d(P5,
P4_filters,
1,
1,
kernel_size=1,
name=class_name + "_decoder_control_backbone_")
decoder = fusion_node(P5,
P4,
filters=P4_filters,
next_filters=P3_filters,
name="decoder_" + class_name + "_P5_P4_fusion_")
decoder = fusion_node(decoder,
P3,
filters=P3_filters,
next_filters=P2_filters,
name="decoder_" + class_name + "_P4_P3_fusion_")
decoder = fusion_node(decoder,
P2,
filters=P2_filters,
next_filters=P2_filters,
name="decoder_" + class_name + "_P3_P2_fusion_")
# segmentation head
segmentation = Conv2D(1, (1, 1),
strides=(1, 1),
padding="same",
name=class_name + "_segout_conv")(decoder)
segmentation = Activation(lambda a: tf.nn.sigmoid(a),
name=class_name + "_sigmoid")(segmentation)
segmentation = resize_img(segmentation,
self.config.model.height,
self.config.model.width,
name=class_name + "_mask")
return segmentation, decoder
