def init_network(self):
"""
Building the Network here
:return:
"""
# Init MobileNet as an encoder
self.encoder = MobileNet(x_input=self.x_pl,
num_classes=self.params.num_classes,
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
width_multipler=1.0,
weight_decay=self.args.weight_decay)
# Build Encoding part
self.encoder.build()
# Build Decoding part
with tf.name_scope('dilation_2'):
self.conv4_2 = atrous_conv2d('conv_ds_7_dil',
self.encoder.conv4_1,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
activation=tf.nn.relu,
dilation_rate=2,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv4_2)
self.conv5_1 = depthwise_separable_conv2d(
'conv_ds_8_dil',
self.conv4_2,
width_multiplier=self.encoder.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv5_1)
self.conv5_2 = depthwise_separable_conv2d(
'conv_ds_9_dil',
self.conv5_1,
width_multiplier=self.encoder.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv5_2)
self.conv5_3 = depthwise_separable_conv2d(
'conv_ds_10_dil',
self.conv5_2,
width_multiplier=self.encoder.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv5_3)
self.conv5_4 = depthwise_separable_conv2d(
'conv_ds_11_dil',
self.conv5_3,
width_multiplier=self.encoder.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv5_4)
self.conv5_5 = depthwise_separable_conv2d(
'conv_ds_12_dil',
self.conv5_4,
width_multiplier=self.encoder.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv5_5)
self.conv5_6 = atrous_conv2d('conv_ds_13_dil',
self.conv5_5,
num_filters=1024,
kernel_size=(3, 3),
padding='SAME',
activation=tf.nn.relu,
dilation_rate=4,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv5_6)
self.conv6_1 = depthwise_separable_conv2d(
'conv_ds_14_dil',
self.conv5_6,
width_multiplier=self.encoder.width_multiplier,
num_filters=1024,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.wd)
_debug(self.conv6_1)
# Pooling is removed.
self.score_fr = conv2d('conv_1c_1x1_dil',
self.conv6_1,
num_filters=self.params.num_classes,
l2_strength=self.wd,
kernel_size=(1, 1))
_debug(self.score_fr)
if self.targets_resize < 8:
self.targets_resize = 8 // self.targets_resize
self.upscore8 = conv2d_transpose(
'upscore8',
x=self.score_fr,
output_shape=self.y_pl.shape.as_list()[0:3] +
[self.params.num_classes],
kernel_size=(self.targets_resize * 2,
self.targets_resize * 2),
stride=(self.targets_resize, self.targets_resize),
l2_strength=self.encoder.wd,
is_training=self.is_training)
_debug(self.upscore8)
self.logits = self.upscore8
else:
self.logits = self.score_fr
def encoder_build(self):
print("Building the MobileNet..")
with tf.variable_scope('mobilenet_encoder'):
with tf.name_scope('Pre_Processing'):
red, green, blue = tf.split(self.x_input, num_or_size_splits=3, axis=3)
preprocessed_input = tf.concat([
(blue - MobileNet.MEAN[0]) / 255.0,
(green - MobileNet.MEAN[1]) / 255.0,
(red - MobileNet.MEAN[2]) / 255.0,
], 3)
self.conv1_1 = conv2d('conv_1', preprocessed_input, num_filters=int(round(32 * self.width_multiplier)),
kernel_size=(3, 3),
padding='SAME', stride=(2, 2), activation=tf.nn.relu6, batchnorm_enabled=True,
is_training=self.train_flag, l2_strength=self.wd)
self._debug(self.conv1_1)
self.conv2_1 = depthwise_separable_conv2d('conv_ds_2', self.conv1_1, width_multiplier=self.width_multiplier,
num_filters=64, kernel_size=(3, 3), padding='SAME', stride=(1, 1),
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd, activation=tf.nn.relu6)
self._debug(self.conv2_1)
self.conv2_2 = depthwise_separable_conv2d('conv_ds_3', self.conv2_1, width_multiplier=self.width_multiplier,
num_filters=128, kernel_size=(3, 3), padding='SAME',
stride=(2, 2), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv2_2)
self.conv3_1 = depthwise_separable_conv2d('conv_ds_4', self.conv2_2, width_multiplier=self.width_multiplier,
num_filters=128, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv3_1)
self.conv3_2 = depthwise_separable_conv2d('conv_ds_5', self.conv3_1, width_multiplier=self.width_multiplier,
num_filters=256, kernel_size=(3, 3), padding='SAME',
stride=(2, 2), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv3_2)
self.conv4_1 = depthwise_separable_conv2d('conv_ds_6', self.conv3_2, width_multiplier=self.width_multiplier,
num_filters=256, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv4_1)
self.conv4_2 = depthwise_separable_conv2d('conv_ds_7', self.conv4_1, width_multiplier=self.width_multiplier,
num_filters=512, kernel_size=(3, 3), padding='SAME',
stride=(2, 2), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv4_2)
self.conv5_1 = depthwise_separable_conv2d('conv_ds_8', self.conv4_2, width_multiplier=self.width_multiplier,
num_filters=512, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv5_1)
self.conv5_2 = depthwise_separable_conv2d('conv_ds_9', self.conv5_1, width_multiplier=self.width_multiplier,
num_filters=512, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv5_2)
self.conv5_3 = depthwise_separable_conv2d('conv_ds_10', self.conv5_2,
width_multiplier=self.width_multiplier,
num_filters=512, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv5_3)
self.conv5_4 = depthwise_separable_conv2d('conv_ds_11', self.conv5_3,
width_multiplier=self.width_multiplier,
num_filters=512, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv5_4)
self.conv5_5 = depthwise_separable_conv2d('conv_ds_12', self.conv5_4,
width_multiplier=self.width_multiplier,
num_filters=512, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv5_5)
self.conv5_6 = depthwise_separable_conv2d('conv_ds_13', self.conv5_5,
width_multiplier=self.width_multiplier,
num_filters=1024, kernel_size=(3, 3), padding='SAME',
stride=(2, 2), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv5_6)
self.conv6_1 = depthwise_separable_conv2d('conv_ds_14', self.conv5_6,
width_multiplier=self.width_multiplier,
num_filters=1024, kernel_size=(3, 3), padding='SAME',
stride=(1, 1), activation=tf.nn.relu6,
batchnorm_enabled=True, is_training=self.train_flag,
l2_strength=self.wd)
self._debug(self.conv6_1)
# Pooling is removed.
self.score_fr = conv2d('conv_1c_1x1', self.conv6_1, num_filters=self.num_classes, l2_strength=self.wd,
kernel_size=(1, 1))
self._debug(self.score_fr)
self.feed1 = self.conv4_2
self.feed2 = self.conv3_2
print("\nEncoder MobileNet is built successfully\n\n")
def init_network(self):
"""
Building the Network here
:return:
"""
# Init MobileNet as an encoder
self.app_encoder = MobileNet(x_input=self.x_pl,
num_classes=self.params.num_classes,
prefix='app_',
pretrained_path=self.args.pretrained_path,
mean_path=self.args.data_dir + 'mean.npy',
train_flag=self.is_training,
width_multipler=1.0,
weight_decay=self.args.weight_decay)
self.motion_encoder = MobileNet(
x_input=self.flo_pl,
num_classes=self.params.num_classes,
prefix='mot_',
pretrained_path=self.args.pretrained_path,
mean_path=self.args.data_dir + 'flo_mean.npy',
train_flag=self.is_training,
width_multipler=1.0,
weight_decay=self.args.weight_decay)
# Build Encoding part
self.app_encoder.build()
self.motion_encoder.build()
self.feed2 = tf.multiply(self.app_encoder.conv3_2,
self.motion_encoder.conv3_2)
self.width_multiplier = 1.0
self.conv4_1 = depthwise_separable_conv2d(
'conv_ds_6_1',
self.feed2,
width_multiplier=self.width_multiplier,
num_filters=256,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv4_1)
self.conv4_2 = depthwise_separable_conv2d(
'conv_ds_7_1',
self.conv4_1,
width_multiplier=self.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(2, 2),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv4_2)
self.conv5_1 = depthwise_separable_conv2d(
'conv_ds_8_1',
self.conv4_2,
width_multiplier=self.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv5_1)
self.conv5_2 = depthwise_separable_conv2d(
'conv_ds_9_1',
self.conv5_1,
width_multiplier=self.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv5_2)
self.conv5_3 = depthwise_separable_conv2d(
'conv_ds_10_1',
self.conv5_2,
width_multiplier=self.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv5_3)
self.conv5_4 = depthwise_separable_conv2d(
'conv_ds_11_1',
self.conv5_3,
width_multiplier=self.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv5_4)
self.conv5_5 = depthwise_separable_conv2d(
'conv_ds_12_1',
self.conv5_4,
width_multiplier=self.width_multiplier,
num_filters=512,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv5_5)
self.conv5_6 = depthwise_separable_conv2d(
'conv_ds_13_1',
self.conv5_5,
width_multiplier=self.width_multiplier,
num_filters=1024,
kernel_size=(3, 3),
padding='SAME',
stride=(2, 2),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv5_6)
self.conv6_1 = depthwise_separable_conv2d(
'conv_ds_14_1',
self.conv5_6,
width_multiplier=self.width_multiplier,
num_filters=1024,
kernel_size=(3, 3),
padding='SAME',
stride=(1, 1),
activation=tf.nn.relu6,
batchnorm_enabled=True,
is_training=self.is_training,
l2_strength=self.args.weight_decay)
_debug(self.conv6_1)
# Pooling is removed.
self.score_fr = conv2d('conv_1c_1x1_1',
self.conv6_1,
num_filters=self.params.num_classes,
l2_strength=self.args.weight_decay,
kernel_size=(1, 1))
self.feed1 = self.conv4_2
# Build Decoding part
with tf.name_scope('upscore_2s'):
self.upscore2 = conv2d_transpose(
'upscore2',
x=self.score_fr,
output_shape=self.feed1.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.args.weight_decay,
bias=self.args.bias)
_debug(self.upscore2)
self.score_feed1 = conv2d(
'score_feed1',
x=self.feed1,
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.args.weight_decay)
_debug(self.score_feed1)
self.fuse_feed1 = tf.add(self.score_feed1, self.upscore2)
with tf.name_scope('upscore_4s'):
self.upscore4 = conv2d_transpose(
'upscore4',
x=self.fuse_feed1,
output_shape=self.feed2.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.args.weight_decay,
bias=self.args.bias)
_debug(self.upscore4)
self.score_feed2 = conv2d(
'score_feed2',
x=self.feed2,
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.args.weight_decay)
_debug(self.score_feed2)
self.fuse_feed2 = tf.add(self.score_feed2, self.upscore4)
with tf.name_scope('upscore_8s'):
self.upscore8 = conv2d_transpose(
'upscore8',
x=self.fuse_feed2,
output_shape=self.x_pl.shape.as_list()[0:3] +
[self.params.num_classes],
is_training=self.is_training,
kernel_size=(16, 16),
stride=(8, 8),
l2_strength=self.args.weight_decay,
bias=self.args.bias)
_debug(self.upscore8)
self.logits = self.upscore8