def forward(self, x, yaw, pose, Tz=0):
z_conv = self.enc(x)
if self.use_vae:
mu = self.fc(z_conv.view(x.size(0), -1))
logvar = self.fc_var(z_conv.view(x.size(0), -1))
std = logvar.mul(0.5).exp_()
eps = get_z_random(std.size(0), std.size(1), 'gauss')
z_fc = eps.mul(std).add_(mu).view(x.size(0), self.nz, 3)
T = np.array([1, 0, 0])
T = Variable(torch.from_numpy(T.astype(np.float32))).cuda().expand(
x.size(0), self.nz, 3)
z_fc += T
self.mu = mu
self.logvar = logvar
else:
z_fc = self.fc(z_conv.view(x.size(0),
-1)).view(x.size(0), self.nz, 3)
# z_fc = F.tanh(z_fc)
R = rotation_tensor(yaw, x.size(0), self.gpu_ids[0])
z_rot = z_fc.bmm(R) # + T
z_rot_fc = self.fc2(z_rot.view(x.size(0), self.nz * 3))
depths = []
y_list = []
output = self.deconv_layers[0](z_rot_fc.view(z_conv.size(0),
z_conv.size(1),
z_conv.size(2),
z_conv.size(3)))
scale = 2
b = x.size(0)
dist = 2
for i, layers in enumerate(self.deconv_layers[1:]):
output = layers(output)
depth = F.tanh(output[:, :1, :, :])
# depths.append( F.tanh(output[:,:1,:,:]) )
scale *= 2
if i < 4:
continue
x_downsampled = F.avg_pool2d(x, int(x.size(2) / scale))
intrinsics = np.array(
[scale / 32. * 60, 0., scale / 2., \
0., scale / 32. * 60, scale / 2., \
0., 0., 1.]).reshape((3, 3))
intrinsics_inv = np.linalg.inv(intrinsics)
intrinsics = Variable(
torch.from_numpy(intrinsics.astype(
np.float32)).cuda()).unsqueeze(0).expand(b, 3, 3)
intrinsics_inv = Variable(
torch.from_numpy(intrinsics_inv.astype(
np.float32)).cuda()).unsqueeze(0).expand(b, 3, 3)
flow_converted = inverse_warp(x_downsampled, depth + dist, pose,
dist, intrinsics, intrinsics_inv)
y_downsampled = F.grid_sample(x_downsampled, flow_converted)
y_list.append(y_downsampled)
return y_list, depth, flow_converted
def forward(self):
add_grid = self.opt.add_grid
rectified = self.opt.rectified
self.real_A = Variable(self.input_A)
self.real_B = Variable(self.input_B)
self.real_Yaw = Variable(self.input_Yaw)
b = self.real_A.size(0)
zeros = Variable(torch.zeros((b, 1)).cuda())
ones = Variable(torch.ones((b, 1)).cuda())
pose_rel = torch.cat(
[zeros, zeros, zeros, zeros, -self.real_Yaw, zeros], dim=1)
# pose_abs = torch.cat( [zeros,zeros, 2*ones, zeros, zeros, zeros], dim=1) #
pose_abs = torch.cat([
zeros, -0.9 * ones, 1.7 * ones, -0.15 * np.pi * ones, zeros, zeros
],
dim=1)
dist = 1.7 / np.cos(
0.15 * np.pi
) # this is to ease the generation of depth, so that depth can be zero meaned
elevated = 1
if elevated:
self.depth = self.netG(real_A_grid, self.real_Yaw,
self.grid) + dist
self.fake_B_flow = self.depth
self.fake_B_flow_converted = projection_layer2.inverse_warp(
self.real_A, self.depth, pose_rel, pose_abs,
self.intrinsics[:b, :, :], self.intrinsics_inv[:b, :, :])
self.fake_B = F.grid_sample(self.real_A,
self.fake_B_flow_converted)
return
if not self.opt.use_pyramid:
self.depth = self.netG(real_A_grid, self.real_Yaw,
self.grid) + dist
self.fake_B_flow = self.depth
self.fake_B_flow_converted = inverse_warp(
self.real_A, self.depth, pose_rel, dist,
self.intrinsics[:b, :, :], self.intrinsics_inv[:b, :, :])
self.fake_B = F.grid_sample(self.real_A,
self.fake_B_flow_converted)
else:
self.fake_B_pyramid, self.depth, self.fake_B_flow_converted = self.netG(
real_A_grid, self.real_Yaw, pose_rel)
self.fake_B = self.fake_B_pyramid[-1]
if self.opt.lambda_kl > 0:
self.mu, self.logvar = self.netG.get_mu_var()
def forward(self):
self.real_A = Variable(self.input_A)
self.real_B = Variable(self.input_B)
self.real_C = Variable(self.input_C)
pose = np.array([0, 0, 0, 0, -np.pi / 4., 0, ]).reshape((1, 6))
pose = Variable(torch.from_numpy(pose.astype(np.float32)).cuda()).expand(self.opt.batchSize,6)
self.forward_map = inverse_warp(self.real_A,self.real_C, pose, self.intrinsics, self.intrinsics_inv)
self.backward_map = self.flow_remapper(self.forward_map, self.forward_map)
self.backward_map_refined = self.netG(self.backward_map.permute(0,3,1,2)).permute(0,2,3,1)
self.fake_B = F.grid_sample(self.real_A, self.backward_map_refined)
def test(self):
self.real_A = Variable(self.input_A, volatile=True)
self.real_B = Variable(self.input_B, volatile=True)
self.fake_B_list = []
NV = 320
b = self.real_A.size(0)
zeros = Variable(torch.zeros((b, 1)).cuda())
ones = Variable(torch.ones((b, 1)).cuda())
pose_abs = torch.cat([
zeros, -0.9 * ones, 1.7 * ones, -0.15 * np.pi * ones, zeros, zeros
],
dim=1)
elevated = 1
dist = 1.7 / np.cos(0.15 * np.pi)
for i in range(NV):
self.real_Yaw = Variable(-torch.Tensor(
[-4 / 9. * np.pi + 8 / 9. * np.pi * i /
(NV - 1)]).cuda(self.gpu_ids[0], async=True)).unsqueeze(0)
self.depth = self.netG(self.real_A, self.real_Yaw,
self.grid) + dist
self.fake_B_flow = self.depth
pose_rel = torch.cat(
[zeros, zeros, zeros, zeros, -self.real_Yaw, zeros], dim=1)
if elevated:
self.fake_B_flow_converted = projection_layer2.inverse_warp(
self.real_A, self.depth, pose_rel, pose_abs,
self.intrinsics[:b, :, :], self.intrinsics_inv[:b, :, :])
self.fake_B = F.grid_sample(self.real_A,
self.fake_B_flow_converted)
else:
self.fake_B_flow_converted = inverse_warp(
self.real_A, self.depth, pose_rel, dist,
self.intrinsics[:b, :, :], self.intrinsics_inv[:b, :, :])
self.fake_B = F.grid_sample(self.real_A,
self.fake_B_flow_converted)
self.fake_B_list.append(self.fake_B)
def forward(self):
self.real_A = Variable(self.input_A)
self.real_B = Variable(self.input_B)
self.real_C = Variable(self.input_C)
self.mask = self.real_C != 0
self.depth = self.netG(self.real_A) + 4
pose = np.array([
0,
0,
0,
0,
-np.pi / 4.,
0,
]).reshape((1, 6))
pose = Variable(torch.from_numpy(pose.astype(
np.float32)).cuda()).expand(self.opt.batchSize, 6)
self.forward_map = inverse_warp(self.real_A, self.depth, pose,
self.intrinsics, self.intrinsics_inv)
self.fake_A = F.grid_sample(self.real_B, self.forward_map)