def init_network(self):
"""
Building the Network here
:return:
"""
# Init ShuffleNet as an encoder
self.encoder = ShuffleNet(
x_input=self.x_pl,
num_classes=self.params.num_classes,
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias)
# Build Encoding part
self.encoder.build()
with tf.name_scope('dilation_2'):
self.stage3 = self.encoder.stage(self.encoder.stage2,
stage=3,
repeat=7,
dilation=2)
_debug(self.stage3)
self.stage4 = self.encoder.stage(self.stage3,
stage=4,
repeat=3,
dilation=4)
_debug(self.stage4)
self.score_fr = conv2d('score_fr_dil',
x=self.stage4,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
l2_strength=self.encoder.wd,
is_training=self.is_training)
_debug(self.score_fr)
if self.targets_resize < 8:
self.targets_resize = 8 // self.targets_resize
self.upscore8 = conv2d_transpose(
'upscore8',
x=self.score_fr,
output_shape=self.y_pl.shape.as_list()[0:3] +
[self.params.num_classes],
kernel_size=(self.targets_resize * 2, self.targets_resize * 2),
stride=(self.targets_resize, self.targets_resize),
l2_strength=self.encoder.wd,
is_training=self.is_training)
_debug(self.upscore8)
self.logits = self.upscore8
else:
self.logits = self.score_fr
def init_network(self):
"""
Building the Network here
:return:
"""
# Init ShuffleNet as an encoder
self.encoder = ShuffleNet(x_input=self.x_pl, num_classes=self.params.num_classes,
pretrained_path=self.args.pretrained_path, train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled, num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay, bias=self.args.bias)
# Build Encoding part
self.encoder.build()
# Build Decoding part
with tf.name_scope('upscore_2s'):
shape = self.encoder.score_fr.shape.as_list()[1:3]
upscore2_upsample = tf.image.resize_images(self.encoder.score_fr,(2 * shape[0], 2 * shape[1]))
self.upscore2 = conv2d('upscore2', x=upscore2_upsample, num_filters=self.params.num_classes,
l2_strength=self.encoder.wd)
self.score_feed1 = conv2d('score_feed1', x=self.encoder.feed1,
num_filters=self.params.num_classes, kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self.fuse_feed1 = tf.add(self.score_feed1, self.upscore2)
with tf.name_scope('upscore_4s'):
shape = self.fuse_feed1.shape.as_list()[1:3]
upscore4_upsample = tf.image.resize_images(self.fuse_feed1,(2 * shape[0], 2 * shape[1]))
self.upscore4 = conv2d('upscore4', x=upscore4_upsample, num_filters=self.params.num_classes,
l2_strength=self.encoder.wd)
self.score_feed2 = conv2d('score_feed2', x=self.encoder.feed2,
num_filters=self.params.num_classes, kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self.fuse_feed2 = tf.add(self.score_feed2, self.upscore4)
with tf.name_scope('upscore_8s'):
shape = self.fuse_feed2.shape.as_list()[1:3]
upscore8_upsample = tf.image.resize_images(self.fuse_feed2,(8 * shape[0], 8 * shape[1]))
self.upscore8 = conv2d('upscore8', x=upscore8_upsample, num_filters=self.params.num_classes,
l2_strength=self.encoder.wd)
self.logits = self.upscore8
def init_network(self):
"""
Building the Network here
:return:
"""
# Init ShuffleNet as an encoder
self.app_encoder = ShuffleNet(
x_input=self.x_pl,
num_classes=self.params.num_classes,
prefix='app_',
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias,
mean_path=self.args.data_dir + 'mean.npy')
self.motion_encoder = ShuffleNet(
x_input=self.flo_pl,
num_classes=self.params.num_classes,
prefix='mot_',
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias,
mean_path=self.args.data_dir + 'flo_mean.npy')
# Build Encoding part
self.app_encoder.build()
self.motion_encoder.build()
self.combined_score = tf.multiply(self.app_encoder.score_fr,
self.motion_encoder.score_fr)
self.combined_feed1 = tf.multiply(self.app_encoder.feed1,
self.motion_encoder.feed1)
self.combined_feed2 = tf.multiply(self.app_encoder.feed2,
self.motion_encoder.feed2)
# Build Decoding part
with tf.name_scope('upscore_2s'):
self.upscore2 = conv2d_transpose(
'upscore2',
x=self.combined_score,
output_shape=self.combined_feed1.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.app_encoder.wd,
bias=self.args.bias)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'upscore2')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.score_feed1 = conv2d(
'score_feed1',
x=self.combined_feed1,
batchnorm_enabled=self.args.batchnorm_enabled,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.app_encoder.wd)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'score_feed1')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.fuse_feed1 = tf.add(self.score_feed1, self.upscore2)
with tf.name_scope('upscore_4s'):
self.upscore4 = conv2d_transpose(
'upscore4',
x=self.fuse_feed1,
output_shape=self.combined_feed2.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.app_encoder.wd,
bias=self.args.bias)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'upscore4')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.score_feed2 = conv2d(
'score_feed2',
x=self.combined_feed2,
batchnorm_enabled=self.args.batchnorm_enabled,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.app_encoder.wd)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'score_feed2')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.fuse_feed2 = tf.add(self.score_feed2, self.upscore4)
with tf.name_scope('upscore_8s'):
self.upscore8 = conv2d_transpose(
'upscore8',
x=self.fuse_feed2,
output_shape=self.x_pl.shape.as_list()[0:3] +
[self.params.num_classes],
kernel_size=(16, 16),
stride=(8, 8),
l2_strength=self.app_encoder.wd,
bias=self.args.bias)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'upscore8')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.logits = self.upscore8
def init_network(self):
"""
Building the Network here
:return:
"""
# Init ShuffleNet as an encoder
self.encoder = ShuffleNet(x_input=self.x_pl, num_classes=self.params.num_classes,
pretrained_path=self.args.pretrained_path, train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled, num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay, bias=self.args.bias)
# Build Encoding part
self.encoder.build()
# Build Decoding part
with tf.name_scope('upscale_1'):
self.expand11 = conv2d('expand1_1', x=self.encoder.stage4, batchnorm_enabled=True,
is_training=self.is_training,
num_filters=self.encoder.stage3.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand11)
self.upscale1 = conv2d_transpose('upscale1', x=self.expand11, is_training=self.is_training,
output_shape=self.encoder.stage3.shape.as_list(), batchnorm_enabled=True,
kernel_size=(4, 4), stride=(2, 2), l2_strength=self.encoder.wd)
self._debug(self.upscale1)
self.add1 = tf.add(self.upscale1, self.encoder.stage3)
self._debug(self.add1)
self.expand12 = conv2d('expand1_2', x=self.add1, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.stage3.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand12)
with tf.name_scope('upscale_2'):
self.expand21 = conv2d('expand2_1', x=self.expand12, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.stage2.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand21)
self.upscale2 = conv2d_transpose('upscale2', x=self.expand21, is_training=self.is_training,
output_shape=self.encoder.stage2.shape.as_list(), batchnorm_enabled=True,
kernel_size=(4, 4), stride=(2, 2), l2_strength=self.encoder.wd)
self._debug(self.upscale2)
self.add2 = tf.add(self.upscale2, self.encoder.stage2)
self._debug(self.add2)
self.expand22 = conv2d('expand2_2', x=self.add2, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.stage2.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand22)
with tf.name_scope('upscale_3'):
self.expand31 = conv2d('expand3_1', x=self.expand22, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.max_pool.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand31)
self.upscale3 = conv2d_transpose('upscale3', x=self.expand31, batchnorm_enabled=True,
is_training=self.is_training,
output_shape=[self.encoder.max_pool.shape[0],
self.encoder.max_pool.shape.as_list()[1] + 1,
self.encoder.max_pool.shape.as_list()[2] + 1,
self.encoder.max_pool.shape.as_list()[3]],
kernel_size=(4, 4), stride=(2, 2), l2_strength=self.encoder.wd)
self._debug(self.upscale3)
padded = tf.pad(self.encoder.max_pool, [[0, 0], [0, 1], [0, 1], [0, 0]], "CONSTANT")
self.add3 = tf.add(self.upscale3, padded)
self._debug(self.add3)
self.expand32 = conv2d('expand3_2', x=self.add3, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.max_pool.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand32)
with tf.name_scope('upscale_4'):
self.expand41 = conv2d('expand4_1', x=self.expand32, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.conv1.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand41)
self.upscale4 = conv2d_transpose('upscale4', x=self.expand41, batchnorm_enabled=True,
is_training=self.is_training,
output_shape=[self.encoder.conv1.shape[0],
self.encoder.conv1.shape.as_list()[1] + 1,
self.encoder.conv1.shape.as_list()[2] + 1,
self.encoder.conv1.shape.as_list()[3]],
kernel_size=(4, 4), stride=(2, 2), l2_strength=self.encoder.wd)
self._debug(self.upscale4)
padded2 = tf.pad(self.encoder.conv1, [[0, 0], [0, 1], [0, 1], [0, 0]], "CONSTANT")
self.add4 = tf.add(self.upscale4, padded2)
self._debug(self.add4)
self.expand42 = conv2d('expand4_2', x=self.add4, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.conv1.shape.as_list()[3], kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.expand42)
with tf.name_scope('upscale_5'):
self.upscale5 = conv2d_transpose('upscale5', x=self.expand42, batchnorm_enabled=True,
is_training=self.is_training,
output_shape=self.x_pl.shape.as_list()[0:3] + [
self.encoder.conv1.shape.as_list()[3]],
kernel_size=(4, 4), stride=(2, 2), l2_strength=self.encoder.wd)
self._debug(self.upscale5)
self.expand5 = conv2d('expand5', x=self.upscale5, batchnorm_enabled=True, is_training=self.is_training,
num_filters=self.encoder.conv1.shape.as_list()[3], kernel_size=(1, 1),
dropout_keep_prob=0.5,
l2_strength=self.encoder.wd)
self._debug(self.expand5)
with tf.name_scope('final_score'):
self.fscore = conv2d('fscore', x=self.expand5,
num_filters=self.params.num_classes, kernel_size=(1, 1),
l2_strength=self.encoder.wd)
self._debug(self.fscore)
self.logits = self.fscore
def init_network(self):
"""
Building the Network here
:return:
"""
# Init ShuffleNet as an encoder
self.app_encoder = ShuffleNet(
x_input=self.x_pl,
num_classes=self.params.num_classes,
prefix='app_',
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias,
mean_path=self.args.data_dir + 'mean.npy')
self.motion_encoder = ShuffleNet(
x_input=self.flo_pl,
num_classes=self.params.num_classes,
prefix='mot_',
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias,
mean_path=self.args.data_dir + 'flo_mean.npy')
# Build Encoding part
self.app_encoder.build()
self.motion_encoder.build()
self.combined_score = tf.multiply(self.app_encoder.stage2,
self.motion_encoder.stage2)
# self.combined_score= tf.concat((self.app_encoder.stage2, self.motion_encoder.stage2), axis=3)
# _debug(self.combined_score)
# self.combined_score = conv2d('combined_score', self.combined_score, num_filters= 240, l2_strength=self.args.weight_decay,
# kernel_size=(1, 1))
self.stage3 = self.app_encoder.stage(self.combined_score,
stage=3,
repeat=7)
_debug(self.stage3)
self.stage4 = self.app_encoder.stage(self.stage3, stage=4, repeat=3)
_debug(self.stage4)
self.feed1 = self.stage3
self.feed2 = self.combined_score
# First Experiment is to use the regular conv2d
self.score_fr = conv2d('combined_conv_1c_1x1',
self.stage4,
num_filters=self.params.num_classes,
l2_strength=self.args.weight_decay,
kernel_size=(1, 1))
_debug(self.score_fr)
# Build Decoding part
with tf.name_scope('upscore_2s'):
self.upscore2 = conv2d_transpose(
'upscore2',
x=self.score_fr,
output_shape=self.feed1.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.args.weight_decay,
bias=self.args.bias)
_debug(self.upscore2)
self.score_feed1 = conv2d(
'score_feed1',
x=self.feed1,
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.args.weight_decay)
_debug(self.score_feed1)
self.fuse_feed1 = tf.add(self.score_feed1, self.upscore2)
with tf.name_scope('upscore_4s'):
self.upscore4 = conv2d_transpose(
'upscore4',
x=self.fuse_feed1,
output_shape=self.feed2.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.args.weight_decay,
bias=self.args.bias)
_debug(self.upscore4)
self.score_feed2 = conv2d(
'score_feed2',
x=self.feed2,
batchnorm_enabled=self.args.batchnorm_enabled,
is_training=self.is_training,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.args.weight_decay)
_debug(self.score_feed2)
self.fuse_feed2 = tf.add(self.score_feed2, self.upscore4)
with tf.name_scope('upscore_8s'):
self.upscore8 = conv2d_transpose(
'upscore8',
x=self.fuse_feed2,
output_shape=self.x_pl.shape.as_list()[0:3] +
[self.params.num_classes],
is_training=self.is_training,
kernel_size=(16, 16),
stride=(8, 8),
l2_strength=self.args.weight_decay,
bias=self.args.bias)
_debug(self.upscore8)
self.logits = self.upscore8