def build_flow_model(self, scope=None, reuse_scope=False):
print("[Info] Building flow network ...")
print("[Info] img_height:", self.img_height, "img_width",
self.img_width)
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
feature1_flow = feature_pyramid_flow(self.tgt_image, reuse=False)
feature0_flow = feature_pyramid_flow(self.src_image_stack[:, :, :,
0:3],
reuse=True)
feature2_flow = feature_pyramid_flow(self.src_image_stack[:, :, :,
3:6],
reuse=True)
flow_fw0 = construct_model_pwc_full(
self.src_image_stack[:, :, :, 0:3], self.tgt_image,
feature0_flow, feature1_flow)
with tf.variable_scope(scope, reuse=True):
flow_fw1 = construct_model_pwc_full(
self.tgt_image, self.src_image_stack[:, :, :, 3:6],
feature1_flow, feature2_flow)
flow_fw2 = construct_model_pwc_full(
self.src_image_stack[:, :, :,
0:3], self.src_image_stack[:, :, :, 3:6],
feature0_flow, feature2_flow)
flow_bw0 = construct_model_pwc_full(
self.tgt_image, self.src_image_stack[:, :, :, 0:3],
feature1_flow, feature0_flow)
flow_bw1 = construct_model_pwc_full(
self.src_image_stack[:, :, :, 3:6], self.tgt_image,
feature2_flow, feature1_flow)
flow_bw2 = construct_model_pwc_full(
self.src_image_stack[:, :, :,
3:6], self.src_image_stack[:, :, :, 0:3],
feature2_flow, feature0_flow)
self.pred_fw_flows = [flow_fw0, flow_fw1, flow_fw2]
self.pred_bw_flows = [flow_bw0, flow_bw1, flow_bw2]
def build_dp_model(self,
scale_normalize=True,
scope=None,
reuse_scope=False):
print("[Info] Building depth and pose network ...")
print("[Info] img_height:", self.img_height, "img_width",
self.img_width)
self.disp = {}
self.depth = {}
self.depth_upsampled = {}
self.disp_bottleneck = {}
def spatial_normalize(self, disp):
# Credit: https://github.com/yzcjtr/GeoNet/blob/master/geonet_model.py
_, curr_h, curr_w, curr_c = disp.get_shape().as_list()
disp_mean = tf.reduce_mean(disp, axis=[1, 2, 3], keepdims=True)
disp_mean = tf.tile(disp_mean, [1, curr_h, curr_w, curr_c])
return disp / disp_mean
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
tgt_pred_disp, tgt_disp_bottlenecks = D_Net(
self.tgt_image_norm,
weight_reg=self.weight_reg,
is_training=True,
reuse=False)
if scale_normalize:
# As proposed in https://arxiv.org/abs/1712.00175, this can
# bring improvement in depth estimation, but not included in our paper.
tgt_pred_disp = [
self.spatial_normalize(disp) for disp in tgt_pred_disp
]
tgt_pred_depth = [1. / d for d in tgt_pred_disp]
self.disp['tgt'] = tgt_pred_disp
self.depth['tgt'] = tgt_pred_depth
self.disp_bottleneck['tgt'] = tgt_disp_bottlenecks
if self.is_depth_upsampling:
self.depth_upsampled['tgt'] = []
for s in range(len(tgt_pred_depth)):
self.depth_upsampled['tgt'].append(
tf.image.resize_bilinear(
tgt_pred_depth[s],
[self.img_height, self.img_width],
align_corners=True))
for i in range(self.num_source):
temp_disp, temp_disp_bottlenecks = D_Net(
self.src_image_stack_norm[:, :, :, 3 * i:3 * (i + 1)],
weight_reg=self.weight_reg,
is_training=True,
reuse=True)
if scale_normalize:
temp_disp = [
self.spatial_normalize(disp) for disp in temp_disp
]
temp_depth = [1. / d for d in temp_disp]
name = 'src{}'.format(i)
self.disp[name] = temp_disp
self.depth[name] = temp_depth
self.disp_bottleneck[name] = temp_disp_bottlenecks
self.depth_upsampled[name] = []
for s in range(len(temp_depth)):
self.depth_upsampled[name].append(
tf.image.resize_bilinear(
temp_depth[s], [self.img_height, self.img_width],
align_corners=True))
if self.joint_encoder:
disp_bottleneck_stack = tf.concat([
self.disp_bottleneck['src0'], self.disp_bottleneck['tgt'],
self.disp_bottleneck['src1']
],
axis=3)
else:
disp_bottleneck_stack = None
pose_inputs = tf.concat([
self.src_image_stack_norm[:, :, :, 0:3], self.tgt_image_norm,
self.src_image_stack_norm[:, :, :, 3:6]
],
axis=3)
self.pred_poses = P_Net3(pose_inputs, disp_bottleneck_stack,
self.joint_encoder, self.weight_reg)
feature_tgt_flow = feature_pyramid_flow(self.tgt_image_norm,
reuse=False)
feature_src0_flow = feature_pyramid_flow(
self.src_image_stack_norm[:, :, :, 0:3], reuse=True)
feature_src1_flow = feature_pyramid_flow(
self.src_image_stack_norm[:, :, :, 3:6], reuse=True)
flow_fw0 = construct_model_pwc_full(
self.src_image_stack_norm[:, :, :, 0:3], self.tgt_image_norm,
feature_src0_flow, feature_tgt_flow)
flow_fw1 = construct_model_pwc_full(
self.tgt_image_norm, self.src_image_stack_norm[:, :, :, 3:6],
feature_tgt_flow, feature_src1_flow)
flow_fw2 = construct_model_pwc_full(
self.src_image_stack_norm[:, :, :, 0:3],
self.src_image_stack_norm[:, :, :, 3:6], feature_src0_flow,
feature_src1_flow)
flow_bw0 = construct_model_pwc_full(
self.tgt_image_norm, self.src_image_stack_norm[:, :, :, 0:3],
feature_tgt_flow, feature_src0_flow)
flow_bw1 = construct_model_pwc_full(
self.src_image_stack_norm[:, :, :, 3:6], self.tgt_image_norm,
feature_src1_flow, feature_tgt_flow)
flow_bw2 = construct_model_pwc_full(
self.src_image_stack_norm[:, :, :, 3:6],
self.src_image_stack_norm[:, :, :, 0:3], feature_src1_flow,
feature_src0_flow)
self.pred_fw_flows = [flow_fw0, flow_fw1,
flow_fw2] #0->1 0->2 1->2
self.pred_bw_flows = [flow_bw0, flow_bw1,
flow_bw2] #1->0 2->1 2->0
def build_dpflow_model2(self,
scale_normalize=True,
fix_pose=False,
scope=None,
reuse_scope=None):
print("[Info] Building depth & pose & flow network ...")
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
self.tgt_pred_disp, self.tgt_disp_bottlenecks = D_Net(
self.tgt_image, is_training=True, reuse=False)
if scale_normalize:
# As proposed in https://arxiv.org/abs/1712.00175, this can
# bring improvement in depth estimation, but not included in our paper.
self.tgt_pred_disp = [
self.spatial_normalize(disp) for disp in self.tgt_pred_disp
]
self.tgt_pred_depth = [1. / d for d in self.tgt_pred_disp]
self.src_pred_disps = []
self.src_pred_depths = []
self.src_disp_bottlenecks = []
for i in range(self.num_source):
temp_disp, temp_disp_bottlenecks = D_Net(
self.src_image_stack[:, :, :, 3 * i:3 * (i + 1)],
is_training=True,
reuse=True)
if scale_normalize:
temp_disp = [
self.spatial_normalize(disp) for disp in temp_disp
]
self.src_pred_disps.append(temp_disp)
self.src_pred_depths.append([1. / d for d in temp_disp])
self.src_disp_bottlenecks.append(temp_disp_bottlenecks)
self.pred_poses = P_Net(self.tgt_image,
self.src_image_stack,
is_training=True)
# pose_inputs = tf.concat([self.tgt_feature, self.src_feature[0], self.src_feature[1]], axis=3)
# self.pred_poses, _ = pose_net_2(pose_inputs, is_training=True, reuse=False)
if fix_pose:
self.pred_poses = tf.stop_gradient(self.pred_poses)
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
feature_tgt_flow = feature_pyramid_flow(self.tgt_image,
reuse=False)
feature_src0_flow = feature_pyramid_flow(
self.src_image_stack[:, :, :, 0:3], reuse=True)
feature_src1_flow = feature_pyramid_flow(
self.src_image_stack[:, :, :, 3:6], reuse=True)
flow_fw0 = construct_model_pwc_full(
self.src_image_stack[:, :, :, 0:3], self.tgt_image,
feature_src0_flow, feature_tgt_flow)
flow_fw1 = construct_model_pwc_full(
self.tgt_image, self.src_image_stack[:, :, :, 3:6],
feature_tgt_flow, feature_src1_flow)
flow_fw2 = construct_model_pwc_full(
self.src_image_stack[:, :, :, 0:3],
self.src_image_stack[:, :, :, 3:6], feature_src0_flow,
feature_src1_flow)
flow_bw0 = construct_model_pwc_full(
self.tgt_image, self.src_image_stack[:, :, :, 0:3],
feature_tgt_flow, feature_src0_flow)
flow_bw1 = construct_model_pwc_full(
self.src_image_stack[:, :, :, 3:6], self.tgt_image,
feature_src1_flow, feature_tgt_flow)
flow_bw2 = construct_model_pwc_full(
self.src_image_stack[:, :, :, 3:6],
self.src_image_stack[:, :, :, 0:3], feature_src1_flow,
feature_src0_flow)
self.pred_flows = [
flow_fw0, flow_bw0, flow_bw1, flow_fw1, flow_fw2, flow_bw2
]