def calculate_rhos(self, Ls, Vs, normals, parameters):
Hs, NdotHs = BrdfParametrization._calculate_NdotHs(Ls, Vs, normals)
NdotLs = inner_product(normals, Ls)
NdotVs = inner_product(normals, Vs)
VdotHs = inner_product(Vs, Hs)
p_diffuse = parameters['diffuse'].view(1, -1, 3)
p_specular = parameters['specular']['albedo'].view(1, -1, 3)
# somewhat non-standard, we parametrize roughness as its square root
# this yields better resolution around zero
p_roughness = parameters['specular']['roughness'].view(1, -1, 1)**2
# fresnel term -- optional
if 'eta' in parameters['specular']:
p_eta = parameters['specular']['eta'].view(1, -1, 1)
Fs = Fresnel(VdotHs, p_eta)
else:
Fs = 1.
# microfacet distribution
Ds = GTR(NdotHs, p_roughness)
# Smith's shadow-masking function
Gs = SmithG1(NdotLs, p_roughness) * SmithG1(NdotVs, p_roughness)
denominator = 4 * np.pi * NdotLs * NdotVs
CTs = p_specular * (Fs * Ds * Gs) / (denominator +
(denominator == 0).float())
rhos = p_diffuse / np.pi + CTs
return rhos, NdotHs
def implied_normal_image(self):
"""
Returns the normal image implied by the current state of the LocationParametrization.
Outputs:
normals HxWx3 torch.tensor containing the depth values
"""
location_image = self.create_image(self.location_vector()) # HxWx3
if self.mask[-1, :].sum() + self.mask[:, -1].sum() > 0:
error(
"The mask should not reach the bottom and right image edges.")
down_vectors = normalize(location_image[:-1, :-1] -
location_image[1:, :-1])
right_vectors = normalize(location_image[:-1, :-1] -
location_image[:-1, 1:])
normal_image = normalize(cross_product(down_vectors, right_vectors))
camloc = self.invRt[:3, 3:]
# make sure the normal is the one that points towards the camera
reorientations = inner_product(
normal_image,
camloc.view(1, 1, 3) - location_image[:-1, :-1]).sign()
normal_image = normal_image * reorientations # H-1 x W-1 x 3
# extend the normals to the edges of the mask
local_normal_mean = normalize(
torch.nn.functional.conv2d(normal_image.permute(2, 0, 1)[:, None],
torch.ones((1, 1, 5, 5),
device=normal_image.device),
padding=2)[:, 0].permute(1, 2, 0))
invalid_normals = norm(normal_image, keepdim=False) == 0
replacement_mask = (
self.mask[:-1, :-1] *
(invalid_normals + ~self.mask[1:, :-1] + ~self.mask[:-1, 1:] > 0))
normal_image[replacement_mask] = local_normal_mean[replacement_mask]
# for badly connected masks, some of the pixels are currently still not filled
# fill them just with a normal pointing roughly in the right direction
invalid_normals = norm(normal_image) == 0
replacement_mask = invalid_normals[:, :, 0] * self.mask[:-1, :-1]
mean_normal = normalize(
local_normal_mean.view(-1, 3).sum(dim=0, keepdim=True))
normal_image[replacement_mask] = mean_normal
normal_image = torch.cat((normal_image,
torch.zeros(1,
normal_image.shape[1],
3,
dtype=normal_image.dtype,
device=normal_image.device)),
dim=0)
normal_image = torch.cat((normal_image,
torch.zeros(normal_image.shape[0],
1,
3,
dtype=normal_image.dtype,
device=normal_image.device)),
dim=1)
return normal_image
def _calculate_NdotHs(Ls, Vs, normals):
"""
Internal function for calculation of half-vectors and their inner products
with the surface normals.
Inputs:
Ls NxLx3 torch.tensor with the directions between
the points and the scene lights
Vs Nx3 torch.tensor with the directions between
the points and the camera
normals Nx3 torch.tensor with the surface normals
Outputs:
Hs NxLx3 torch.tensor with the normalized half-vectors between
viewing and light directions.
NdotHs NxLx1 torch.tensor containing the inner products between
the surface normals and the view-light half vectors
"""
Hs = (Ls + Vs)
Hs = normalize(Hs)
NdotHs = inner_product(normals, Hs)
return Hs, NdotHs
def evaluate(self, simulations, observations, experiment_state,
data_adapter):
normals_from_depth = experiment_state.locations.implied_normal_vector()
normals_estimated = experiment_state.normals.normals()
inner_products = inner_product(normals_from_depth, normals_estimated)
return 1 - inner_products
def closed_form_lambertian_solution(experiment_state,
data_adapter,
sample_radius=7,
shadows_occlusions=True,
verbose=True):
"""
Calculate the closed form solution of the materials and normals, given a lambertian assumption.
Inputs:
experiment_state
data_adapter
[sample_radius] Observations can optionally be box-blurred before extraction. Defaults to 7.
[shadows_occlusions] Whether to simulate obstructions in the rendering. Defaults to True.
[verbose] Whether to be verbose about the progress. Defaults to True.
Outputs:
normals Nx3 torch.tensor containing the normal estimates
albedos Nx3 torch.tensor containing the diffuse albedo estimates
inliers NxC containing, for each point and each observation, whether it is an inlier in the RANSAC.
residuals NxCx3 containing, for each point and each observation, the residual modelling error.
"""
device = torch.device(general_settings.device_name)
# calculates the closed-form solution given the current state and the measurements
with torch.no_grad():
light_intensities = []
light_directions = []
shadows = []
observations = []
occlusions = []
training_indices_batches, training_light_infos_batches = data_adapter.get_training_info(
)
for batch_indices, batch_light_infos in zip(
training_indices_batches, training_light_infos_batches):
batch_light_intensities, batch_light_directions, batch_shadows = experiment_state.light_parametrization.get_light_intensities(
experiment_state.locations,
experiment_state.observation_poses.Rts(batch_indices),
light_infos=batch_light_infos,
calculate_shadowing=shadows_occlusions)
light_intensities.append(batch_light_intensities.transpose(0, 1))
light_directions.append(batch_light_directions.transpose(0, 1))
shadows.append(batch_shadows.transpose(0, 1))
batch_observations, batch_occlusions = experiment_state.extract_observations(
data_adapter,
batch_indices,
smoothing_radius=sample_radius,
calculate_occlusions=shadows_occlusions)
batch_occlusions = (
batch_occlusions +
(batch_observations[..., 1] == OBSERVATION_OUT_OF_BOUNDS)) > 0
observations.append(batch_observations.transpose(0, 1))
occlusions.append(batch_occlusions.transpose(0, 1))
light_intensities = torch.cat(light_intensities, dim=1)
light_directions = torch.cat(light_directions, dim=1)
shadows = torch.cat(shadows, dim=1)
observations = torch.cat(observations, dim=1)
occlusions = torch.cat(occlusions, dim=1)
incident_light = ( # points x views x color channels x direction_ray
light_intensities[:, :, :, None] * light_directions[:, :, None, :])
shadowing_mask = (shadows == 0).float()[:, :, None]
occlusion_mask = (occlusions == 0).float()[:, :, None]
valid_mask = (shadowing_mask * occlusion_mask)
invalids = (valid_mask.sum(dim=1) <= 2).float().view(-1, 1, 1)
eye = torch.eye(3, device=invalids.device).view(1, 3, 3) * 1e8
# normals are estimated from a gray-scale version. However, here we are aiming for maximum SNR rather than
# a visually pleasing grayscale. As such, no fancy color-dependent scaling
gray_light_intensities = incident_light.mean(
dim=-2, keepdim=False) * valid_mask
gray_observations = observations.mean(dim=-1,
keepdim=True) * valid_mask
# a RANSAC approach
# at every iteration, we take a small subset to calculate the solution from
# and count the number of inliers
subset_size = min(observations.shape[1], 6)
threshold = 1e4
# we aim for 10 succesfull RANSAC samples (i.e. without outliers)
# not taking into account the sample set size
estimated_outlier_ratio = 0.20
targeted_success_chance = 1 - 1e-4
nr_its = 10 * np.log(1 - targeted_success_chance) / np.log(
1 - (1 - estimated_outlier_ratio)**subset_size
) if subset_size < observations.shape[1] else 1
best_inlier_counts = torch.zeros(observations.shape[0], 1, 1).to(
observations.device) - 1
best_inliers = torch.zeros(*observations.shape[:2],
1).to(observations.device)
ransac_loop = tqdm(range(int(nr_its))) if verbose else range(
int(nr_its))
for iteration in ransac_loop:
subset = np.random.choice(observations.shape[1],
size=[subset_size],
replace=False)
subset = torch.tensor(subset,
dtype=torch.long,
device=observations.device)
intensities_subset = gray_light_intensities.index_select(
dim=1, index=subset)
observations_subset = gray_observations.index_select(dim=1,
index=subset)
shadowing_subset = shadowing_mask.index_select(dim=1, index=subset)
occlusion_subset = occlusion_mask.index_select(dim=1, index=subset)
valid_subset = shadowing_subset * occlusion_subset
invalids_subset = (valid_subset.sum(dim=1) <= 2).float().view(
-1, 1, 1)
intensities_subset_pinv = (
intensities_subset.transpose(1, 2) @ intensities_subset + eye *
invalids_subset).inverse() @ intensities_subset.transpose(
1, 2)
solution_subset = intensities_subset_pinv @ observations_subset
residuals = gray_light_intensities @ solution_subset - gray_observations
inliers = ((residuals.abs() < threshold) * valid_mask).float()
inlier_counts = inliers.sum(dim=1, keepdim=True)
better_subset = (inlier_counts > best_inlier_counts).float()
best_inlier_counts = best_inlier_counts * (
1 - better_subset) + inlier_counts * better_subset
best_inliers = best_inliers * (
1 - better_subset) + inliers * better_subset
if verbose:
ransac_loop.set_description(
desc=
"Lambertian closed form | RANSAC total energy | avg inliers %8.5f"
% best_inlier_counts.mean())
# now calculate the normal estimate for the best inlier group
inlier_intensities = gray_light_intensities * best_inliers
inlier_observations = gray_observations * best_inliers
inlier_invalids = ((valid_mask * best_inliers).sum(dim=1) <=
2).float().view(-1, 1, 1)
inlier_intensities_pinv = (
inlier_intensities.transpose(1, 2) @ inlier_intensities + eye *
inlier_invalids).inverse() @ inlier_intensities.transpose(1, 2)
inlier_normals = normalize(
(inlier_intensities_pinv @ inlier_observations).view(-1, 3))
albedos = []
residuals = []
inlier_mask = best_inliers * valid_mask
for c in range(3):
c_light_intensities = inner_product(inlier_normals.view(
-1, 1, 3), incident_light[:, :, c]) * inlier_mask
c_observations = observations[:, :, c, None] * inlier_mask
c_light_intensities_pinv = (
c_light_intensities.transpose(1, 2) @ c_light_intensities +
inlier_invalids).inverse() @ c_light_intensities.transpose(
1, 2)
c_albedo = (c_light_intensities_pinv @ c_observations)
albedos.append(np.pi * c_albedo.view(-1, 1))
c_residuals = (c_light_intensities @ c_albedo - c_observations)
residuals.append(c_residuals)
albedos = torch.cat(albedos, dim=1)
residuals = torch.cat(residuals, dim=2)
return inlier_normals, albedos, best_inliers, residuals
def simulate(self,
observation_indices,
light_infos,
shadow_cache=None,
override=None,
calculate_shadowing=True):
"""
Simulate the result of the current experiment state for a given set of observation indices and light infos.
Including the shadow mask if requested, True by default.
Shadows can be cached persistently by passing a cache dictionary to shadow_cache.
"""
if shadow_cache is None:
shadow_cache = {}
surface_points = self.locations.location_vector()
obs_Rt = self.observation_poses.Rts(observation_indices)
shadow_cache_miss = not light_infos in shadow_cache
light_intensities, light_directions, calculated_shadowing = self.light_parametrization.get_light_intensities(
self.locations,
obs_Rt,
light_infos=light_infos,
calculate_shadowing=calculate_shadowing and shadow_cache_miss)
if calculate_shadowing:
if shadow_cache_miss:
shadow_mask = calculated_shadowing
shadow_cache[light_infos] = shadow_mask
else:
shadow_mask = shadow_cache[light_infos]
else:
shadow_mask = None
obs_camloc = self.observation_poses.camlocs(observation_indices)
view_directions = normalize(
obs_camloc.view(-1, 1, 3) - surface_points[None])
if override is not None and "geometry" in override:
surface_normals = NormalParametrizationFactory("hard linked")(
self.locations).normals()
brdf = BrdfParametrizationFactory("cook torrance F1")()
N, device = surface_normals.shape[0], surface_normals.device
surface_materials = {
"diffuse": torch.empty(N, 3, device=device).fill_(0.8),
"specular": {
"albedo":
torch.empty(N, 3, device=device).fill_(0.2),
"roughness":
torch.empty(N, 1, device=device).fill_(0.15 - 0.10 *
("1" in override)),
}
}
else:
surface_materials = self.materials.get_brdf_parameters()
if override is not None:
surface_materials = dict(surface_materials)
if override == "diffuse":
surface_materials['specular'] = dict(
surface_materials['specular'])
surface_materials['specular']['albedo'] = torch.zeros_like(
surface_materials['specular']['albedo'])
elif override == "specular":
surface_materials['diffuse'] = torch.zeros_like(
surface_materials['diffuse'])
surface_normals = self.normals.normals().view(1, -1, 3)
brdf = self.brdf
surface_rhos, NdotHs = brdf.calculate_rhos(light_directions,
view_directions,
surface_normals,
surface_materials)
incident_light = light_intensities * inner_product(
light_directions, surface_normals)
incident_light.clamp_(min=0.)
if shadow_mask is not None:
incident_light[shadow_mask] = 0.
simulation = surface_rhos * incident_light
# TODO: this is where vignetting would normally come in
simulation[incident_light <= 0] = SIMULATION_SHADOWED
return simulation