def forward(self, pts3D, src):
bs = src.size(0)
if len(src.size()) > 3:
bs, c, w, _ = src.size()
image_size = w
pts3D = pts3D.permute(0, 2, 1)
src = src.unsqueeze(2).repeat(1, 1, w, 1, 1).view(bs, c, -1)
else:
bs = src.size(0)
image_size = self.size
# Make sure these have been arranged in the same way
assert pts3D.size(2) == 3
assert pts3D.size(1) == src.size(2)
pts3D[:, :, 1] = -pts3D[:, :, 1]
pts3D[:, :, 0] = -pts3D[:, :, 0]
# Add on the default feature to the end of the src
# src = torch.cat((src, self.default_feature.repeat(bs, 1, 1)), 2)
radius = float(self.radius) / float(image_size) * 2.0
pts3D = Pointclouds(points=pts3D, features=src.permute(0, 2, 1))
points_idx, _, dist = rasterize_points(pts3D, image_size, radius,
self.points_per_pixel)
if os.environ["DEBUG"]:
print("Max dist: ", dist.max(), pow(radius, self.opts.rad_pow))
dist = dist / pow(radius, self.opts.rad_pow)
if os.environ["DEBUG"]:
print("Max dist: ", dist.max())
alphas = ((1 - dist.clamp(max=1, min=1e-3).pow(0.5)).pow(
self.opts.tau).permute(0, 3, 1, 2))
if self.opts.accumulation == 'alphacomposite':
transformed_src_alphas = compositing.alpha_composite(
points_idx.permute(0, 3, 1, 2).long(),
alphas,
pts3D.features_packed().permute(1, 0),
)
elif self.opts.accumulation == 'wsum':
transformed_src_alphas = compositing.weighted_sum(
points_idx.permute(0, 3, 1, 2).long(),
alphas,
pts3D.features_packed().permute(1, 0),
)
elif self.opts.accumulation == 'wsumnorm':
transformed_src_alphas = compositing.weighted_sum_norm(
points_idx.permute(0, 3, 1, 2).long(),
alphas,
pts3D.features_packed().permute(1, 0),
)
return transformed_src_alphas
def forward(self, point_clouds, **kwargs) -> torch.Tensor:
"""
point_clouds_filter: used to get activation mask and update visibility mask
cutoff_threshold
"""
if point_clouds.isempty():
return None
# rasterize
fragments = kwargs.get('fragments', None)
if fragments is None:
if kwargs.get('verbose', False):
fragments, point_clouds, per_point_info = self.rasterizer(
point_clouds, **kwargs)
else:
fragments, point_clouds = self.rasterizer(
point_clouds, **kwargs)
# compute weight: scalar*exp(-0.5Q)
weights = torch.exp(-0.5 * fragments.qvalue) * fragments.scaler
weights = weights.permute(0, 3, 1, 2)
# from fragments to rgba
pts_rgb = point_clouds.features_packed()[:, :3]
if self.compositor is None:
# NOTE: weight _splat_points_weights_backward, weighted sum will return
# zero gradient for the weights.
images = weighted_sum(fragments.idx.long().permute(0, 3, 1, 2),
weights, pts_rgb.permute(1, 0), **kwargs)
else:
images = self.compositor(fragments.idx.long().permute(0, 3, 1, 2),
weights, pts_rgb.permute(1, 0), **kwargs)
# permute so image comes at the end
images = images.permute(0, 2, 3, 1)
mask = fragments.occupancy
images = torch.cat([images, mask.unsqueeze(-1)], dim=-1)
if kwargs.get('verbose', False):
return images, fragments
return images
def forward(self, pts3D, src):
bs = src.size(0)
if len(src.size()) > 3:
bs, c, w, _ = src.size()
image_size = w
pts3D = pts3D.permute(0, 2, 1)
src = src.unsqueeze(2).repeat(1, 1, w, 1, 1).view(bs, c, -1)
else:
bs = src.size(0)
image_size = self.size
# Make sure these have been arranged in the same way
assert pts3D.size(2) == 3
assert pts3D.size(1) == src.size(2)
# pts3D.shape = (bs, w*h, 3) --> (x,y,z) coordinate for ever element in the image raster
# Because we have done re-projection, the i-th coordinate in the image raster must no longer be identical to (x,y)!
# src.shape = (bs, c, w*h) --> c features for every element in the image raster (w*h)
#print("Features: {}".format(src.shape))
#print("3D Pointcloud: {}".format(pts3D.shape))
# flips the x and y coordinate
pts3D[:, :, 1] = -pts3D[:, :, 1]
pts3D[:, :, 0] = -pts3D[:, :, 0]
# Add on the default feature to the end of the src
#src = torch.cat((src, self.default_feature.repeat(bs, 1, 1)), 2)
radius = float(self.radius) / float(
image_size
) * 2.0 # convert radius to fit the [-1,1] NDC ?? Or is this just arbitrary scaling s.t. radius as meaningful size?
params = compositing.CompositeParams(radius=radius)
#print("Radius - before: {}, converted: {}".format(self.radius, radius))
pts3D = Pointclouds(points=pts3D, features=src.permute(0, 2, 1))
points_idx, _, dist = rasterize_points(
pts3D, image_size, radius, self.points_per_pixel
) # see method signature for meaning of these output values
#print("points_idx: {}".format(points_idx.shape))
#print("dist: {}".format(points_idx.shape))
#print("Max dist: ", dist.max(), pow(radius, self.rad_pow))
dist = dist / pow(
radius, self.rad_pow
) # equation 1 from the paper (3.2): this calculates N(p_i, l_mn) from the d2 dist
#print("Max dist: ", dist.max())
alphas = (
(1 - dist.clamp(max=1, min=1e-3).pow(0.5)).pow(
self.accumulation_tau).permute(0, 3, 1, 2)
) # equation 2 from the paper (3.2): prepares alpha values for composition of the feature vectors
#print("alphas: ", alphas.shape)
#print("pointclouds object: {}".format(pts3D.features_packed().shape))
#print("alphas: ", alphas)
if self.accumulation == 'alphacomposite':
transformed_src_alphas = compositing.alpha_composite(
points_idx.permute(0, 3, 1, 2).long(),
alphas,
pts3D.features_packed().permute(
1, 0
), # pts3D also contains features here, because this is now a Pointclouds object
params,
)
elif self.accumulation == 'wsum':
transformed_src_alphas = compositing.weighted_sum(
points_idx.permute(0, 3, 1, 2).long(),
alphas,
pts3D.features_packed().permute(1, 0),
params,
)
elif self.accumulation == 'wsumnorm':
transformed_src_alphas = compositing.weighted_sum_norm(
points_idx.permute(0, 3, 1, 2).long(),
alphas,
pts3D.features_packed().permute(1, 0),
params,
)
else:
raise NotImplementedError('Unsupported accumulation type: ' +
self.accumulation)
return transformed_src_alphas