def _conv_1x1_block(self, name, x, filters):
with tf.variable_scope(name):
with tf.variable_scope('conv2d'):
initializer = tf.contrib.layers.xavier_initializer()
kernel_shape = [1, 1, x.shape[-1], filters]
w = variable_with_weight_decay2(kernel_shape, initializer,
self.args.weight_decay)
out = tf.nn.conv2d(x, w, [1, 1, 1, 1], padding='VALID')
if self.args.use_bias:
bias = tf.get_variable(
'bias',
filters,
initializer=tf.constant_initializer(0.0))
out = tf.nn.bias_add(out, bias)
tf.add_to_collection('debug_layers', out)
out = tf.layers.batch_normalization(out,
training=self.is_training,
fused=True)
tf.add_to_collection('debug_layers', out)
tf.add_to_collection('decoding_trainable_vars', w)
tf.add_to_collection('decoding_trainable_vars', bias)
bnvars = get_vars_underscope(tf.get_variable_scope().name,
'batch_normalization')
for v in bnvars:
tf.add_to_collection('decoding_trainable_vars', v)
out = tf.nn.relu(out)
return out
def _full_conv_3x3_block(self, name, x, out_channels, stride=2):
with tf.variable_scope(name):
out = self._deconv('deconv', x, out_channels, kernel_size=(3, 3), stride=stride)
out = tf.layers.batch_normalization(out, training=self.is_training, fused=True)
tf.add_to_collection('debug_layers', out)
bnvars= get_vars_underscope(tf.get_variable_scope().name, 'batch_normalization')
for v in bnvars:
tf.add_to_collection('decoding_trainable_vars', v)
out = tf.nn.relu(out)
return out
def init_network(self):
"""
Building the Network here
:return:
"""
# Init ShuffleNet as an encoder
self.app_encoder = ShuffleNet(
x_input=self.x_pl,
num_classes=self.params.num_classes,
prefix='app_',
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias,
mean_path=self.args.data_dir + 'mean.npy')
self.motion_encoder = ShuffleNet(
x_input=self.flo_pl,
num_classes=self.params.num_classes,
prefix='mot_',
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
batchnorm_enabled=self.args.batchnorm_enabled,
num_groups=self.args.num_groups,
weight_decay=self.args.weight_decay,
bias=self.args.bias,
mean_path=self.args.data_dir + 'flo_mean.npy')
# Build Encoding part
self.app_encoder.build()
self.motion_encoder.build()
self.combined_score = tf.multiply(self.app_encoder.score_fr,
self.motion_encoder.score_fr)
self.combined_feed1 = tf.multiply(self.app_encoder.feed1,
self.motion_encoder.feed1)
self.combined_feed2 = tf.multiply(self.app_encoder.feed2,
self.motion_encoder.feed2)
# Build Decoding part
with tf.name_scope('upscore_2s'):
self.upscore2 = conv2d_transpose(
'upscore2',
x=self.combined_score,
output_shape=self.combined_feed1.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.app_encoder.wd,
bias=self.args.bias)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'upscore2')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.score_feed1 = conv2d(
'score_feed1',
x=self.combined_feed1,
batchnorm_enabled=self.args.batchnorm_enabled,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.app_encoder.wd)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'score_feed1')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.fuse_feed1 = tf.add(self.score_feed1, self.upscore2)
with tf.name_scope('upscore_4s'):
self.upscore4 = conv2d_transpose(
'upscore4',
x=self.fuse_feed1,
output_shape=self.combined_feed2.shape.as_list()[0:3] +
[self.params.num_classes],
batchnorm_enabled=self.args.batchnorm_enabled,
kernel_size=(4, 4),
stride=(2, 2),
l2_strength=self.app_encoder.wd,
bias=self.args.bias)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'upscore4')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.score_feed2 = conv2d(
'score_feed2',
x=self.combined_feed2,
batchnorm_enabled=self.args.batchnorm_enabled,
num_filters=self.params.num_classes,
kernel_size=(1, 1),
bias=self.args.bias,
l2_strength=self.app_encoder.wd)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'score_feed2')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.fuse_feed2 = tf.add(self.score_feed2, self.upscore4)
with tf.name_scope('upscore_8s'):
self.upscore8 = conv2d_transpose(
'upscore8',
x=self.fuse_feed2,
output_shape=self.x_pl.shape.as_list()[0:3] +
[self.params.num_classes],
kernel_size=(16, 16),
stride=(8, 8),
l2_strength=self.app_encoder.wd,
bias=self.args.bias)
currvars = get_vars_underscope(tf.get_variable_scope().name,
'upscore8')
for v in currvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.logits = self.upscore8
def init_network(self):
"""
Building the Network here
:return:
"""
# Init a RESNET as an encoder
self.encoder = RESNET18(x_input=self.x_pl,
num_classes=self.params.num_classes,
pretrained_path=self.args.pretrained_path,
train_flag=self.is_training,
bias=self.args.bias,
weight_decay=self.args.weight_decay)
# Build Encoding part
self.encoder.build()
# Build Decoding part
self.out_decoder_block_4 = self._decoder_block('decoder_block_4', self.encoder.encoder_4, 256, stride=2)
self.out_decoder_block_4 = tf.add(self.out_decoder_block_4, self.encoder.encoder_3, 'add_features_4')
self.out_decoder_block_3 = self._decoder_block('decoder_block_3', self.out_decoder_block_4, 128, stride=2)
self.out_decoder_block_3 = tf.add(self.out_decoder_block_3, self.encoder.encoder_2, 'add_features_3')
self.out_decoder_block_2 = self._decoder_block('decoder_block_2', self.out_decoder_block_3, 64, stride=2)
self.out_decoder_block_2 = tf.add(self.out_decoder_block_2, self.encoder.encoder_1, 'add_features_2')
self.out_decoder_block_1 = self._decoder_block('decoder_block_1', self.out_decoder_block_2, 64, stride=1)
with tf.variable_scope('output_block'):
self.out_full_conv1 = self._deconv('deconv_out_1', self.out_decoder_block_1, 32, (3, 3), stride=2)
self.out_full_conv1 = tf.layers.batch_normalization(self.out_full_conv1, training=self.is_training,
fused=True)
tf.add_to_collection('debug_layers', self.out_full_conv1)
bnvars= get_vars_underscope(tf.get_variable_scope().name, 'batch_normalization')
for v in bnvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.out_full_conv1 = tf.nn.relu(self.out_full_conv1)
print("output_block_full_conv1: %s" % (str(self.out_full_conv1.shape.as_list())))
self.out_conv1 = tf.pad(self.out_full_conv1, [[0,0],[1,1],[1,1],[0,0]], "CONSTANT")
self.out_conv1 = tf.layers.conv2d(self.out_conv1, filters=32, kernel_size=(3, 3), padding="valid",
use_bias=self.args.use_bias,
kernel_initializer=tf.contrib.layers.xavier_initializer(),
kernel_regularizer=tf.contrib.layers.l2_regularizer(
self.args.weight_decay))
tf.add_to_collection('debug_layers', self.out_conv1)
convvars= get_vars_underscope(tf.get_variable_scope().name, 'conv2d')
for v in convvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.out_conv1 = tf.layers.batch_normalization(self.out_conv1, training=self.is_training, fused=True)
tf.add_to_collection('debug_layers', self.out_conv1)
bnvars= get_vars_underscope(tf.get_variable_scope().name, 'batch_normalization')
for v in bnvars:
tf.add_to_collection('decoding_trainable_vars', v)
self.out_conv1 = tf.nn.relu(self.out_conv1)
print("output_block_conv1: %s" % (str(self.out_conv1.shape.as_list())))
self.fscore = self._deconv('deconv_out_2', self.out_conv1, self.params.num_classes, (2, 2), stride=2)
print("logits: %s" % (str(self.fscore.shape.as_list())))
self.logits = self.fscore