def forward(ctx, trusted_depth_center, color_center, camera_center,
trusted_depths_neighbours, colors_neighbours,
cameras_neighbours):
B = colors_neighbours.shape[0]
N = colors_neighbours.shape[1]
C = colors_neighbours.shape[2]
H_out = color_center.shape[2]
W_out = color_center.shape[3]
sentinel = 1e9
output_depth = trusted_depth_center.new_full((B, N, 1, H_out, W_out),
fill_value=sentinel)
output_color = color_center.new_full((B, N, C, H_out, W_out),
fill_value=0.0)
invKRs_neighbours = cameras_neighbours.new_empty(B, N, 3, 3)
camlocs_neighbours = cameras_neighbours.new_empty(B, N, 3, 1)
MYTH.InvertCams_gpu(cameras_neighbours.reshape(-1, 3, 4),
invKRs_neighbours.reshape(-1, 3, 3),
camlocs_neighbours.reshape(-1, 3, 1))
MYTH.PatchedReprojectionNeighbours_updateOutput_gpu(
trusted_depths_neighbours, colors_neighbours, output_depth,
output_color, camera_center, invKRs_neighbours, camlocs_neighbours)
output_depth[output_depth > sentinel / 10] = 0
ctx.save_for_backward(trusted_depths_neighbours, output_depth,
camera_center.clone(), invKRs_neighbours.clone(),
camlocs_neighbours.clone(),
colors_neighbours.clone())
return output_depth, output_color
def forward(ctx, depths, cameras, scale):
B = depths.shape[0]
N = depths.shape[1]
H = depths.shape[3]
W = depths.shape[4]
sentinel = 1e9
output_depth = depths.new_full((B, N, 1, H, W), fill_value=sentinel)
camlocs = depths.new_empty(B, N, 3, 1)
invKRs = depths.new_empty(B, N, 3, 3)
if scale != 1.0:
for b in range(B):
for n in range(N):
cameras[b][n] = cameras[b][n].clone()
cameras[b][n][:2, :] = cameras[b][n][:2, :] * scale
invKRs[b][n] = torch.inverse(
cameras[b][n][:3, :3]).contiguous()
camlocs[b][n] = -torch.mm(invKRs[b][n], cameras[b][n][:3,
3:4])
else:
MYTH.InvertCams_gpu(cameras.reshape(-1, 3, 4),
invKRs.reshape(-1, 3, 3),
camlocs.reshape(-1, 3, 1))
MYTH.DepthReprojectionNeighbours_updateOutput_gpu(
depths, output_depth, cameras, invKRs, camlocs)
output_depth[output_depth > sentinel / 10] = 0
return output_depth
def forward(ctx, depths, reprojected, cameras, scale, dmin, dmax, dstep):
B = depths.shape[0]
N = depths.shape[1]
H = depths.shape[3]
W = depths.shape[4]
camlocs = depths.new_empty(B, N, 3, 1)
invKRs = depths.new_empty(B, N, 3, 3)
if scale != 1.0:
for b in range(B):
for n in range(N):
cameras[b][n] = cameras[b][n].clone()
cameras[b][n][:2, :] = cameras[b][n][:2, :] * scale
invKRs[b][n] = torch.inverse(
cameras[b][n][:3, :3]).contiguous()
camlocs[b][n] = -torch.mm(invKRs[b][n], cameras[b][n][:3,
3:4])
else:
MYTH.InvertCams_gpu(cameras.reshape(-1, 3, 4),
invKRs.reshape(-1, 3, 3),
camlocs.reshape(-1, 3, 1))
ctx.save_for_backward(depths, cameras, camlocs, invKRs)
MYTH.DepthReprojectionNonzeroCompleteBound_updateOutput_gpu(
depths, reprojected, cameras, invKRs, camlocs, dmin, dmax, dstep)
return reprojected
def get_view_neighbours(self, cameras, center_view, nr_neighbours):
if nr_neighbours == 0:
return []
cameras = cameras.cuda()
B = cameras.shape[0]
camlocs = cameras.new_empty(B, 3, 1)
invKRs = cameras.new_empty(B, 3, 3)
MYTH.InvertCams_gpu(cameras, invKRs, camlocs)
distances = (camlocs -
camlocs[center_view:center_view + 1, :, :]).pow(2).sum(
dim=1).sum(dim=1)
distances = [d.item() for d in distances]
orders = sorted(range(len(distances)), key=distances.__getitem__)
if nr_neighbours >= len(distances):
return orders
if self._neighbour_selection == "closest":
return orders[1:1 + nr_neighbours]
elif self._neighbour_selection == "furthest":
return orders[-nr_neighbours:]
elif self._neighbour_selection == "mixed":
return orders[1:1 + nr_neighbours //
2] + orders[-(nr_neighbours - nr_neighbours // 2):]
else:
raise ValueError(
"Unsupported neighbourhood selection approach '%s'" %
self._neighbour_selection)
def sparsity_count_gpu(points, sparsity1, sparsity2):
N = points.shape[1]
counts1 = np.zeros((N, ))
counts2 = np.zeros((N, ))
if N == 0:
return counts1, counts2
points = torch.Tensor(points).cuda()
counts1 = torch.Tensor(counts1).cuda()
counts2 = torch.Tensor(counts2).cuda()
MYTH.bruteforce_sparsity_count_gpu(points, counts1, counts2, N,
sparsity1, sparsity2)
torch.cuda.synchronize()
counts1 = counts1.cpu().numpy()
counts2 = counts2.cpu().numpy()
return counts1, counts2
def forward(ctx, depth_center, camera_center, depths_neighbours,
cameras_neighbours, dmin, dmax, dstep):
B = depths_neighbours.shape[0]
N = depths_neighbours.shape[1]
H_center = depth_center.shape[2]
W_center = depth_center.shape[3]
invKR_center = camera_center.new_empty(B, 3, 3)
camloc_center = camera_center.new_empty(B, 3, 1)
MYTH.InvertCams_gpu(camera_center, invKR_center, camloc_center)
bounds_neighbours = depth_center.new_zeros(
(B, N, 1, H_center, W_center))
MYTH.PatchedReprojectionCompleteBound_updateOutput_gpu(
depth_center, depths_neighbours, bounds_neighbours,
cameras_neighbours, invKR_center, camloc_center, dmin, dmax, dstep)
return bounds_neighbours
def backward(ctx, grad_output_depth, grad_output_color, grad_output_angle):
# note: backprop currently only implemented for the color
depths, output_depth, cameras, camlocs, invKRs = ctx.saved_variables
grad_input_color = grad_output_color.new_zeros(grad_output_color.shape)
MYTH.DepthColorAngleReprojectionNeighbours_updateGradInput_gpu(
depths, output_depth, grad_input_color, grad_output_color, cameras,
invKRs, camlocs)
return None, grad_input_color, None, None
def distance_gpu(points_from, points_to):
N = points_from.shape[1]
M = points_to.shape[1]
dists = np.zeros((N, ))
if N == 0:
return dists
if M == 0:
dists.fill(np.inf)
return dists
points_from = torch.Tensor(points_from).cuda()
points_to = torch.Tensor(points_to).cuda()
dists = torch.Tensor(dists).cuda()
MYTH.bruteforce_distance_gpu(points_from, points_to, dists, N, M)
torch.cuda.synchronize()
dists = np.sqrt(dists.cpu().numpy())
return dists
def backward(ctx, dloss_doutput_depth, dloss_doutput_color):
# note: backprop currently only implemented for the color
trusted_depths_neighbours, output_depth, camera_center, invKRs_neighbours, camlocs_neighbours, colors_neighbours = ctx.saved_variables
dloss_dinput_color = dloss_doutput_color.new_zeros(
colors_neighbours.shape)
MYTH.PatchedReprojectionNeighbours_updateGradInput_gpu(
trusted_depths_neighbours, output_depth, dloss_dinput_color,
dloss_doutput_color, camera_center, invKRs_neighbours,
camlocs_neighbours)
return None, None, None, None, dloss_dinput_color, None
def forward(ctx, bounds, depths, avg_bounds, avg_depths, cull_mask):
B = bounds.shape[0]
N = bounds.shape[1]
H = bounds.shape[3]
W = bounds.shape[4]
var_bounds = bounds.new_zeros((B, 1, H, W))
var_depths = bounds.new_zeros((B, 1, H, W))
MYTH.VariationReprojectionPooling_updateOutput_gpu(
avg_bounds, avg_depths, bounds, depths, var_bounds, var_depths,
cull_mask)
ctx.save_for_backward(bounds, depths, avg_bounds, avg_depths,
cull_mask)
return var_bounds, var_depths
def forward(ctx, bounds, depths, features):
B = features.shape[0]
N = features.shape[1]
F = features.shape[2]
H = features.shape[3]
W = features.shape[4]
avg_bounds = bounds.new_zeros((B, 1, H, W))
avg_depths = bounds.new_zeros((B, 1, H, W))
avg_features = bounds.new_zeros((B, F, H, W))
MYTH.PatchedAverageReprojectionPooling_updateOutput_gpu(
bounds, depths, features, avg_bounds, avg_depths, avg_features)
ctx.save_for_backward(bounds, depths)
return avg_bounds, avg_depths, avg_features
def backward(ctx, dloss_max_bounds, dloss_max_depths, dloss_max_features):
bounds, depths, cull_mask = ctx.saved_variables
B = dloss_max_features.shape[0]
F = dloss_max_features.shape[1]
H = dloss_max_features.shape[2]
W = dloss_max_features.shape[3]
N = cull_mask.shape[1]
dloss_bounds = dloss_max_bounds.new_zeros((B, N, 1, H, W))
dloss_depths = dloss_max_bounds.new_zeros((B, N, 1, H, W))
dloss_features = dloss_max_bounds.new_zeros((B, N, F, H, W))
MYTH.PatchedMinimumAbsReprojectionPooling_updateGradInput_gpu(
bounds, depths, dloss_bounds, dloss_depths, dloss_features,
dloss_max_bounds, dloss_max_depths, dloss_max_features, cull_mask)
return dloss_bounds, dloss_depths, dloss_features, None
def forward(ctx, bounds, depths, features, cull_mask):
B = features.shape[0]
N = features.shape[1]
F = features.shape[2]
H = features.shape[3]
W = features.shape[4]
max_bounds = bounds.new_zeros((B, 1, H, W))
max_depths = bounds.new_zeros((B, 1, H, W))
max_features = bounds.new_zeros((B, F, H, W))
MYTH.PatchedMinimumAbsReprojectionPooling_updateOutput_gpu(
bounds, depths, features, max_bounds, max_depths, max_features,
cull_mask)
ctx.save_for_backward(bounds, depths, cull_mask)
return max_bounds, max_depths, max_features
def backward(ctx, dloss_avg_bounds, dloss_avg_depths, dloss_avg_features):
bounds, depths = ctx.saved_variables
B = dloss_avg_features.shape[0]
F = dloss_avg_features.shape[1]
H = dloss_avg_features.shape[2]
W = dloss_avg_features.shape[3]
N = bounds.shape[1]
dloss_bounds = dloss_avg_bounds.new_zeros((B, N, 1, H, W))
dloss_depths = dloss_avg_bounds.new_zeros((B, N, 1, H, W))
dloss_features = dloss_avg_bounds.new_zeros((B, N, F, H, W))
MYTH.PatchedAverageReprojectionPooling_updateGradInput_gpu(
bounds, depths, dloss_bounds, dloss_depths, dloss_features,
dloss_avg_bounds, dloss_avg_depths, dloss_avg_features)
return dloss_bounds, dloss_depths, dloss_features
def backward(ctx, dloss_var_bounds, dloss_var_depths):
bounds, depths, avg_bounds, avg_depths, cull_mask = ctx.saved_variables
B = dloss_var_bounds.shape[0]
F = dloss_var_bounds.shape[1]
H = dloss_var_bounds.shape[2]
W = dloss_var_bounds.shape[3]
N = cull_mask.shape[1]
dloss_bounds = dloss_var_bounds.new_zeros((B, N, 1, H, W))
dloss_depths = dloss_var_bounds.new_zeros((B, N, 1, H, W))
dloss_avg_bounds = dloss_var_bounds.new_zeros((B, 1, H, W))
dloss_avg_depths = dloss_var_bounds.new_zeros((B, 1, H, W))
MYTH.VariationReprojectionPooling_updateGradInput_gpu(
avg_bounds, avg_depths, bounds, depths, dloss_bounds, dloss_depths,
dloss_avg_bounds, dloss_avg_depths, dloss_var_bounds,
dloss_var_depths, cull_mask)
return dloss_bounds, dloss_depths, dloss_avg_bounds, dloss_avg_depths, None
