def warp(self, net, input_images):
net_copy = net
flow = net[:, :, :, 0:2]
# mask = tf.expand_dims(net[:, :, :, 2], 3)
grid_x, grid_y = meshgrid(net.shape[1], net.shape[2])
grid_x = tf.tile(grid_x, [FLAGS.batch_size, 1, 1])
grid_y = tf.tile(grid_y, [FLAGS.batch_size, 1, 1])
# flow = 0.5 * flow
coor_x_1 = grid_x + flow[:, :, :, 0]
coor_y_1 = grid_y + flow[:, :, :, 1]
# coor_x_2 = grid_x - flow[:, :, :, 0]
# coor_y_2 = grid_y - flow[:, :, :, 1]
output_1 = bilinear_interp(input_images[:, :, :, 0:3], coor_x_1, coor_y_1, 'interpolate')
# output_2 = bilinear_interp(input_images[:, :, :, 3:6], coor_x_2, coor_y_2, 'interpolate')
# mask = (1.0 + mask)
# mask = tf.tile(mask, [1, 1, 1, 3])
# net = tf.multiply(mask, output_1) \
# + tf.multiply(1.0 - mask, output_2)
# for the correct loss function
# flow_motion = net_copy[:, :, :, 0:2]
# flow_mask = tf.expand_dims(net_copy[:, :, :, 2], 3)
return(output_1)
def _build_model(self, input_images):
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.leaky_relu,
weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),
weights_regularizer=slim.l2_regularizer(0.0001)):
# Define network
batch_norm_params = {
'decay': 0.9997,
'epsilon': 0.001,
'is_training': self.is_train,
}
with slim.arg_scope([slim.batch_norm],
is_training=self.is_train,
updates_collections=None):
with slim.arg_scope([slim.conv2d],
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
x0 = slim.conv2d(input_images,
32, [7, 7],
stride=1,
scope='conv1')
x0_1 = slim.conv2d(x0,
32, [7, 7],
stride=1,
scope='conv1_1')
net = slim.avg_pool2d(x0_1, [2, 2], scope='pool1')
x1 = slim.conv2d(net, 64, [5, 5], stride=1, scope='conv2')
x1_1 = slim.conv2d(x1,
64, [5, 5],
stride=1,
scope='conv2_1')
net = slim.avg_pool2d(x1_1, [2, 2], scope='pool2')
x2 = slim.conv2d(net, 128, [3, 3], stride=1, scope='conv3')
x2_1 = slim.conv2d(x2,
128, [3, 3],
stride=1,
scope='conv3_1')
net = slim.avg_pool2d(x2_1, [2, 2], scope='pool3')
x3 = slim.conv2d(net, 256, [3, 3], stride=1, scope='conv4')
x3_1 = slim.conv2d(x3,
256, [3, 3],
stride=1,
scope='conv4_1')
net = slim.avg_pool2d(x3_1, [2, 2], scope='pool4')
x4 = slim.conv2d(net, 512, [3, 3], stride=1, scope='conv5')
x4_1 = slim.conv2d(x4,
512, [3, 3],
stride=1,
scope='conv5_1')
net = slim.avg_pool2d(x4_1, [2, 2], scope='pool5')
net = slim.conv2d(net,
512, [3, 3],
stride=1,
scope='conv6')
net = slim.conv2d(net,
512, [3, 3],
stride=1,
scope='conv6_1')
net = tf.image.resize_bilinear(net, [
x4.get_shape().as_list()[1],
x4.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x4_1], -1),
512, [3, 3],
stride=1,
scope='conv7')
net = slim.conv2d(net,
512, [3, 3],
stride=1,
scope='conv7_1')
net = tf.image.resize_bilinear(net, [
x3.get_shape().as_list()[1],
x3.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x3_1], -1),
256, [3, 3],
stride=1,
scope='conv8')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv8_1')
net = tf.image.resize_bilinear(net, [
x2.get_shape().as_list()[1],
x2.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x2_1], -1),
128, [3, 3],
stride=1,
scope='conv9')
net = slim.conv2d(net,
128, [3, 3],
stride=1,
scope='conv9_1')
net = tf.image.resize_bilinear(net, [
x1.get_shape().as_list()[1],
x1.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x1_1], -1),
64, [3, 3],
stride=1,
scope='conv10')
net = slim.conv2d(net,
64, [3, 3],
stride=1,
scope='conv10_1')
net = tf.image.resize_bilinear(net, [
x0.get_shape().as_list()[1],
x0.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x0_1], -1),
32, [3, 3],
stride=1,
scope='conv11')
y0 = slim.conv2d(net,
32, [3, 3],
stride=1,
scope='conv11_1')
net = slim.conv2d(y0,
3, [5, 5],
stride=1,
activation_fn=tf.tanh,
normalizer_fn=None,
scope='conv12')
net_copy = net
flow = net[:, :, :, 0:2]
mask = tf.expand_dims(net[:, :, :, 2], 3)
self.flow = flow
grid_x, grid_y = meshgrid(x0.get_shape().as_list()[1],
x0.get_shape().as_list()[2])
grid_x = tf.tile(grid_x, [FLAGS.batch_size, 1, 1])
grid_y = tf.tile(grid_y, [FLAGS.batch_size, 1, 1])
flow = 0.5 * flow
flow_ratio = tf.constant([
255.0 / (x0.get_shape().as_list()[2] - 1),
255.0 / (x0.get_shape().as_list()[1] - 1)
])
flow = flow * tf.expand_dims(
tf.expand_dims(tf.expand_dims(flow_ratio, 0), 0), 0)
coor_x_1 = grid_x + flow[:, :, :, 0]
coor_y_1 = grid_y + flow[:, :, :, 1]
coor_x_2 = grid_x - flow[:, :, :, 0]
coor_y_2 = grid_y - flow[:, :, :, 1]
output_1 = bilinear_interp(input_images[:, :, :, 0:3], coor_x_1,
coor_y_1, 'interpolate')
output_2 = bilinear_interp(input_images[:, :, :, 3:6], coor_x_2,
coor_y_2, 'interpolate')
self.warped_img1 = output_1
self.warped_img2 = output_2
self.warped_flow1 = bilinear_interp(-flow[:, :, :, 0:3] * 0.5,
coor_x_1, coor_y_1, 'interpolate')
self.warped_flow2 = bilinear_interp(flow[:, :, :, 0:3] * 0.5, coor_x_2,
coor_y_2, 'interpolate')
mask = 0.5 * (1.0 + mask)
self.mask = mask
mask = tf.tile(mask, [1, 1, 1, 3])
net = tf.multiply(mask, output_1) + tf.multiply(1.0 - mask, output_2)
return [net, net_copy]
def _build_model(self, input_images):
"""Build a VAE model.
Args:
"""
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),
weights_regularizer=slim.l2_regularizer(0.0001)):
# Define network
batch_norm_params = {
'decay': 0.9997,
'epsilon': 0.001,
'is_training': self.is_train,
}
with slim.arg_scope([slim.batch_norm],
is_training=self.is_train,
updates_collections=None):
with slim.arg_scope([slim.conv2d],
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
net = slim.conv2d(input_images,
64, [5, 5],
stride=1,
scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.conv2d(net,
128, [5, 5],
stride=1,
scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = tf.image.resize_bilinear(net, [64, 64])
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv4')
net = tf.image.resize_bilinear(net, [128, 128])
net = slim.conv2d(net,
128, [3, 3],
stride=1,
scope='conv5')
net = tf.image.resize_bilinear(net, [256, 256])
net = slim.conv2d(net, 64, [5, 5], stride=1, scope='conv6')
net = slim.conv2d(net,
3, [5, 5],
stride=1,
activation_fn=tf.tanh,
normalizer_fn=None,
scope='conv7')
flow = net[:, :, :, 0:2]
mask = tf.expand_dims(net[:, :, :, 2], 3)
grid_x, grid_y = meshgrid(256, 256)
grid_x = tf.tile(grid_x, [32, 1, 1]) # batch_size = 32
grid_y = tf.tile(grid_y, [32, 1, 1]) # batch_size = 32
flow = 0.5 * flow
coor_x_1 = grid_x + flow[:, :, :, 0]
coor_y_1 = grid_y + flow[:, :, :, 1]
coor_x_2 = grid_x - flow[:, :, :, 0]
coor_y_2 = grid_y - flow[:, :, :, 1]
output_1 = bilinear_interp(input_images[:, :, :, 0:3], coor_x_1,
coor_y_1, 'interpolate')
output_2 = bilinear_interp(input_images[:, :, :, 3:6], coor_x_2,
coor_y_2, 'interpolate')
mask = 0.5 * (1.0 + mask)
mask = tf.tile(mask, [1, 1, 1, 3])
net = tf.mul(mask, output_1) + tf.mul(1.0 - mask, output_2)
return net
def FAB_inference(self,
input_images_boundary,
input_images_blur,
F,
H,
batch_size,
net_channel=64,
num_classes=136,
num_blocks=[2, 2, 2, 2],
use_bias=False,
bottleneck=True,
dropout_ratio=1.0):
####structure_predictor_model####
with tf.variable_scope('structure_predictor_model_'):
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(
0.0, 0.01),
weights_regularizer=slim.l2_regularizer(0.0001)):
batch_norm_params = {
'decay': 0.9997,
'epsilon': 0.001,
'is_training': self.structure_predictor_is_train,
}
with slim.arg_scope(
[slim.batch_norm],
is_training=self.structure_predictor_is_train,
updates_collections=None):
with slim.arg_scope([slim.conv2d],
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
net = slim.conv2d(input_images_boundary,
64, [5, 5],
stride=1,
scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.conv2d(net,
128, [5, 5],
stride=1,
scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = tf.image.resize_bilinear(net, [64, 64])
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv4')
net = tf.image.resize_bilinear(net, [128, 128])
net = slim.conv2d(net,
128, [3, 3],
stride=1,
scope='conv5')
net = tf.image.resize_bilinear(net, [256, 256])
net = slim.conv2d(net,
64, [5, 5],
stride=1,
scope='conv6')
net = slim.conv2d(net,
3, [5, 5],
stride=1,
activation_fn=tf.tanh,
normalizer_fn=None,
scope='conv7')
flow = net[:, :, :, 0:2]
mask = tf.expand_dims(net[:, :, :, 2], 3)
grid_x, grid_y = meshgrid(256, 256)
grid_x = tf.tile(grid_x, [batch_size, 1, 1])
grid_y = tf.tile(grid_y, [batch_size, 1, 1])
coor_x_1 = grid_x + flow[:, :, :, 0] * 2
coor_y_1 = grid_y + flow[:, :, :, 1] * 2
coor_x_2 = grid_x + flow[:, :, :, 0]
coor_y_2 = grid_y + flow[:, :, :, 1]
output_1 = bilinear_interp(input_images_boundary[:, :, :, 0:1],
coor_x_1, coor_y_1, 'extrapolate')
output_2 = bilinear_interp(input_images_boundary[:, :, :, 1:2],
coor_x_2, coor_y_2, 'extrapolate')
mask = 0.5 * (1.0 + mask)
mask = tf.tile(mask, [1, 1, 1, 3])
self.next_frame = tf.multiply(mask, output_1) + tf.multiply(
1.0 - mask, output_2)
self.structure_predictor_output = tf.concat(
[self.next_frame, input_images_blur], 3)
####video_deblur_model####
with tf.variable_scope('video_deblur_model_'):
H_curr = []
with tf.variable_scope("encoding"):
with tf.variable_scope("conv1"):
filter_size = 5
net_X = self.conv2d(
self.structure_predictor_output,
self.weight_variable([
filter_size, filter_size,
self.get_shape(self.structure_predictor_output, 3),
net_channel
]))
net_X = tf.nn.relu(net_X)
with tf.variable_scope("conv2"):
filter_size = 3
net_X = self.conv2d(net_X,
self.weight_variable([
filter_size, filter_size,
self.get_shape(net_X, 3),
net_channel // 2
]),
stride=2)
net_X = tf.nn.relu(net_X)
net = tf.concat([net_X, F], 3)
f0 = net
filter_size = 3
num_resnet_layers = 8
for i in range(num_resnet_layers):
with tf.variable_scope('resnet_block%d' % (i + 1)):
net = self.resnet_block(net, net_channel)
if i == 3:
(net, alpha) = self.dynamic_fusion(net, H[0])
h = tf.expand_dims(net, axis=0)
H_curr = h
with tf.variable_scope("feat_out"):
F = self.conv2d(
net,
self.weight_variable([
filter_size, filter_size,
self.get_shape(net, 3), net_channel // 2
],
name='conv_F'))
F = tf.nn.relu(F)
with tf.variable_scope("img_out"):
filter_size = 4
shape = [
self.get_shape(self.structure_predictor_output, 0),
self.get_shape(self.structure_predictor_output, 1),
self.get_shape(self.structure_predictor_output, 2),
net_channel
]
Y = self.conv2d_transpose(
net,
self.weight_variable(
[filter_size, filter_size, net_channel, net_channel],
name="deconv"),
shape,
stride=2)
Y = tf.nn.relu(Y)
filter_size = 3
self.video_deblur_output = self.conv2d(
Y,
self.weight_variable(
[filter_size, filter_size,
self.get_shape(Y, 3), 3],
name='conv'))
####resnet_model####
with tf.variable_scope('resnet_model_'):
c = Config()
c['bottleneck'] = bottleneck
c['is_training'] = tf.convert_to_tensor(self.resnet_is_train,
dtype='bool',
name='is_training')
c['ksize'] = 3
c['stride'] = 1
c['use_bias'] = use_bias
c['fc_units_out'] = num_classes
c['num_blocks'] = num_blocks
c['stack_stride'] = 2
with tf.variable_scope('scale1'):
c['conv_filters_out'] = 16
c['ksize'] = 7
c['stride'] = 2
x = self.conv(self.video_deblur_output, c)
x = self.resnet_bn(x, c)
x = self.activation(x)
with tf.variable_scope('scale1_pool'):
x = self._max_pool(x, ksize=3, stride=2)
x = self.resnet_bn(x, c)
x = self.activation(x)
with tf.variable_scope('scale2'):
x = self._max_pool(x, ksize=3, stride=2)
c['num_blocks'] = num_blocks[0]
c['stack_stride'] = 1
c['block_filters_internal'] = 8
x = self.stack(x, c)
with tf.variable_scope('scale3'):
c['num_blocks'] = num_blocks[1]
c['block_filters_internal'] = 16
assert c['stack_stride'] == 2
x = self.stack(x, c)
with tf.variable_scope('scale4'):
c['num_blocks'] = num_blocks[2]
c['block_filters_internal'] = 32
x = self.stack(x, c)
with tf.variable_scope('scale5'):
c['num_blocks'] = num_blocks[3]
c['block_filters_internal'] = 64
x = self.stack(x, c)
x = tf.reduce_mean(x, reduction_indices=[1, 2], name="avg_pool")
if num_classes != None:
with tf.variable_scope('fc1'):
c['fc_units_out'] = 256
x = self.fc(x, c)
with tf.variable_scope('dropout1'):
x = tf.nn.dropout(x, dropout_ratio)
with tf.variable_scope('fc2'):
c['fc_units_out'] = 256
x = self.fc(x, c)
with tf.variable_scope('dropout2'):
x = tf.nn.dropout(x, dropout_ratio)
with tf.variable_scope('fc3'):
c['fc_units_out'] = 136
self.logits = self.fc(x, c)
return F, H_curr
def structure_predictor_inference(self, input_images_boundary, batch_size):
with tf.variable_scope('structure_predictor_model_'):
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(
0.0, 0.01),
weights_regularizer=slim.l2_regularizer(0.0001)):
batch_norm_params = {
'decay': 0.9997,
'epsilon': 0.0001,
'is_training': self.structure_predictor_is_train
}
with slim.arg_scope(
[slim.batch_norm],
is_training=self.structure_predictor_is_train,
updates_collections=None):
with slim.arg_scope([slim.conv2d],
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
net = slim.conv2d(input_images_boundary,
64, [5, 5],
stride=1,
scope='conv1')
net = slim.max_pool2d(net, [2, 2], scope='pool1')
net = slim.conv2d(net,
128, [5, 5],
stride=1,
scope='conv2')
net = slim.max_pool2d(net, [2, 2], scope='pool2')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv3')
net = slim.max_pool2d(net, [2, 2], scope='pool3')
net = tf.image.resize_bilinear(net, [64, 64])
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv4')
net = tf.image.resize_bilinear(net, [128, 128])
net = slim.conv2d(net,
128, [3, 3],
stride=1,
scope='conv5')
net = tf.image.resize_bilinear(net, [256, 256])
net = slim.conv2d(net,
64, [5, 5],
stride=1,
scope='conv6')
net = slim.conv2d(net,
3, [5, 5],
stride=1,
activation_fn=tf.tanh,
normalizer_fn=None,
scope='conv7')
flow = net[:, :, :, 0:2]
mask = tf.expand_dims(net[:, :, :, 2], 3)
grid_x, grid_y = meshgrid(256, 256)
grid_x = tf.tile(grid_x, [batch_size, 1, 1])
grid_y = tf.tile(grid_y, [batch_size, 1, 1])
coor_x_1 = grid_x + flow[:, :, :, 0] * 2
coor_y_1 = grid_y + flow[:, :, :, 1] * 2
coor_x_2 = grid_x + flow[:, :, :, 0]
coor_y_2 = grid_y + flow[:, :, :, 1]
output_1 = bilinear_interp(input_images_boundary[:, :, :, 0:1],
coor_x_1, coor_y_1, 'extrapolate')
output_2 = bilinear_interp(input_images_boundary[:, :, :, 1:2],
coor_x_2, coor_y_2, 'extrapolate')
mask = 0.33 * (1.0 + mask)
mask = tf.tile(mask, [1, 1, 1, 3])
next_frame = tf.multiply(mask, output_1) + tf.multiply(
1.0 - mask, output_2)
return next_frame
def _build_model(self, input_images):
"""Build a VAE model.
Args:
"""
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(0.0,
0.01)):
# Define network
batch_norm_params = {
'decay': 0.9997,
'epsilon': 0.001,
'is_training': self.is_train,
}
with slim.arg_scope([slim.batch_norm],
is_training=self.is_train,
updates_collections=None):
with slim.arg_scope([slim.conv2d],
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
# encoders
conv_1 = slim.conv2d(input_images,
16, [5, 5],
stride=1,
scope='conv1')
pool_1 = slim.max_pool2d(conv_1, [2, 2], scope='pool1')
conv_2 = slim.conv2d(pool_1,
32, [5, 5],
stride=1,
scope='conv2')
pool_2 = slim.max_pool2d(conv_2, [2, 2], scope='pool2')
conv_3 = slim.conv2d(pool_2,
64, [3, 3],
stride=1,
scope='conv3')
pool_3 = slim.max_pool2d(conv_3, [2, 2], scope='pool3')
bottleneck = slim.conv2d(pool_3,
128, [3, 3],
stride=1,
scope='bottleneck')
# decoders
upsamp_1 = tf.image.resize_bilinear(bottleneck, [64, 64])
deconv_1 = slim.conv2d(tf.concat([upsamp_1, conv_3],
axis=3),
64, [3, 3],
stride=1,
scope='deconv4')
upsamp_2 = tf.image.resize_bilinear(deconv_1, [128, 128])
deconv_2 = slim.conv2d(tf.concat([upsamp_2, conv_2],
axis=3),
32, [3, 3],
stride=1,
scope='deconv5')
# upsampled to input dimensions
upsamp_A = tf.image.resize_bilinear(deconv_2, [256, 256])
deconv_A = slim.conv2d(tf.concat([upsamp_A, conv_1],
axis=3),
32, [5, 5],
stride=1,
scope='deconvA')
upsamp_B = tf.image.resize_bilinear(deconv_1, [256, 256])
deconv_B = slim.conv2d(upsamp_B,
32, [5, 5],
stride=1,
scope='deconvB')
upsamp_C = tf.image.resize_bilinear(bottleneck, [256, 256])
deconv_C = slim.conv2d(upsamp_C,
32, [5, 5],
stride=1,
scope='deconvC')
# concatenated
conv_4 = slim.conv2d(tf.concat(
[deconv_A, deconv_B, deconv_C], axis=3),
64, [5, 5],
scope='conv_4')
net = slim.conv2d(conv_4,
3, [5, 5],
stride=1,
activation_fn=tf.tanh,
normalizer_fn=None,
scope='conv7')
net_copy = net
flow = net[:, :, :, 0:2]
mask = tf.expand_dims(net[:, :, :, 2], 3)
grid_x, grid_y = meshgrid(256, 256)
grid_x = tf.tile(grid_x, [FLAGS.batch_size, 1, 1])
grid_y = tf.tile(grid_y, [FLAGS.batch_size, 1, 1])
flow = 0.5 * flow
coor_x_1 = grid_x + flow[:, :, :, 0]
coor_y_1 = grid_y + flow[:, :, :, 1]
coor_x_2 = grid_x - flow[:, :, :, 0]
coor_y_2 = grid_y - flow[:, :, :, 1]
output_1 = bilinear_interp(input_images[:, :, :, 0:3], coor_x_1,
coor_y_1, 'interpolate')
output_2 = bilinear_interp(input_images[:, :, :, 3:6], coor_x_2,
coor_y_2, 'interpolate')
mask = 0.5 * (1.0 + mask)
mask = tf.tile(mask, [1, 1, 1, 3])
net = tf.multiply(mask, output_1) + tf.multiply(1.0 - mask, output_2)
# for the correct loss function
flow_motion = net_copy[:, :, :, 0:2]
flow_mask = tf.expand_dims(net[:, :, :, 2], 3)
return (net, flow_motion, flow_mask)
def _build_model(self, input_images):
with slim.arg_scope(
[slim.conv2d],
activation_fn=tf.nn.relu,
weights_initializer=tf.truncated_normal_initializer(0.0, 0.01),
weights_regularizer=slim.l2_regularizer(0.0001)):
# Define network
batch_norm_params = {
'decay': 0.9997,
'epsilon': 0.001,
'is_training': self.is_train,
}
with slim.arg_scope([slim.batch_norm],
is_training=self.is_train,
updates_collections=None):
with slim.arg_scope([slim.conv2d],
normalizer_fn=slim.batch_norm,
normalizer_params=batch_norm_params):
x0 = slim.conv2d(input_images,
64, [5, 5],
stride=1,
scope='conv1')
# with tf.name_scope('conv1') as scope:
# kernel = tf.Variable(tf.truncated_normal([5, 5, 8, 64], dtype=tf.float32, stddev=1e-1), name='weights')
# conv = tf.nn.atrous_conv2d(input_images, kernel, 2, padding='SAME')
# x0 = tf.nn.relu(conv, name=scope)
net = slim.max_pool2d(x0, [2, 2], scope='pool1')
x1 = slim.conv2d(net, 128, [5, 5], stride=1, scope='conv2')
net = slim.max_pool2d(x1, [2, 2], scope='pool2')
x2 = slim.conv2d(net, 256, [3, 3], stride=1, scope='conv3')
net = slim.max_pool2d(x2, [2, 2], scope='pool3')
net = slim.conv2d(net,
256, [3, 3],
stride=1,
scope='conv4')
net = tf.image.resize_bilinear(net, [
x2.get_shape().as_list()[1],
x2.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x2], -1),
256, [3, 3],
stride=1,
scope='conv5')
net = tf.image.resize_bilinear(net, [
x1.get_shape().as_list()[1],
x1.get_shape().as_list()[2]
])
net = slim.conv2d(tf.concat([net, x1], -1),
128, [3, 3],
stride=1,
scope='conv6')
net = tf.image.resize_bilinear(net, [
x0.get_shape().as_list()[1],
x0.get_shape().as_list()[2]
])
y0 = slim.conv2d(tf.concat([net, x0], -1),
64, [5, 5],
stride=1,
scope='conv7')
net = slim.conv2d(y0,
3, [5, 5],
stride=1,
activation_fn=tf.tanh,
normalizer_fn=None,
scope='conv8')
net_copy = net
flow = net[:, :, :, 0:2]
mask = tf.expand_dims(net[:, :, :, 2], 3)
self.flow = flow
grid_x, grid_y = meshgrid(x0.get_shape().as_list()[1],
x0.get_shape().as_list()[2])
grid_x = tf.tile(grid_x, [FLAGS.batch_size, 1, 1])
grid_y = tf.tile(grid_y, [FLAGS.batch_size, 1, 1])
flow = 0.5 * flow
flow_ratio = tf.constant([
255.0 / (x0.get_shape().as_list()[2] - 1),
255.0 / (x0.get_shape().as_list()[1] - 1)
])
flow = flow * tf.expand_dims(
tf.expand_dims(tf.expand_dims(flow_ratio, 0), 0), 0)
if self.is_extrapolation:
coor_x_1 = grid_x + flow[:, :, :, 0] * 2
coor_y_1 = grid_y + flow[:, :, :, 1] * 2
coor_x_2 = grid_x + flow[:, :, :, 0]
coor_y_2 = grid_y + flow[:, :, :, 1]
else:
coor_x_1 = grid_x + flow[:, :, :, 0]
coor_y_1 = grid_y + flow[:, :, :, 1]
coor_x_2 = grid_x - flow[:, :, :, 0]
coor_y_2 = grid_y - flow[:, :, :, 1]
output_1 = bilinear_interp(input_images[:, :, :, 0:3], coor_x_1,
coor_y_1, 'interpolate')
output_2 = bilinear_interp(input_images[:, :, :, 3:6], coor_x_2,
coor_y_2, 'interpolate')
self.warped_img1 = output_1
self.warped_img2 = output_2
self.warped_flow1 = bilinear_interp(-flow[:, :, :, 0:3] * 0.5,
coor_x_1, coor_y_1, 'interpolate')
self.warped_flow2 = bilinear_interp(flow[:, :, :, 0:3] * 0.5, coor_x_2,
coor_y_2, 'interpolate')
mask = 0.5 * (1.0 + mask)
self.mask = mask
mask = tf.tile(mask, [1, 1, 1, 3])
net = tf.multiply(mask, output_1) + tf.multiply(1.0 - mask, output_2)
return [net, net_copy]
