def forward(self, input, target): # Split the SVBRDF into its individual components input_normals, input_diffuse, input_roughness, input_specular = utils.unpack_svbrdf(input) target_normals, target_diffuse, target_roughness, target_specular = utils.unpack_svbrdf(target) epsilon_l1 = 0.01 input_diffuse = torch.log(input_diffuse + epsilon_l1) input_specular = torch.log(input_specular + epsilon_l1) target_diffuse = torch.log(target_diffuse + epsilon_l1) target_specular = torch.log(target_specular + epsilon_l1) return nn.functional.l1_loss(input_normals, target_normals) + nn.functional.l1_loss(input_diffuse, target_diffuse) + nn.functional.l1_loss(input_roughness, target_roughness) + nn.functional.l1_loss(input_specular, target_specular)
def mix(self, svbrdf_0, svbrdf_1, alpha=None): if alpha is None: alpha = torch.Tensor(1).uniform_(0.1, 0.9) normals_0, diffuse_0, roughness_0, specular_0 = utils.unpack_svbrdf(svbrdf_0) normals_1, diffuse_1, roughness_1, specular_1 = utils.unpack_svbrdf(svbrdf_1) # Reference "Project the normals to use the X and Y derivative" normals_0_projected = normals_0 / torch.max(torch.Tensor([0.01]), normals_0[2:3,:,:]) normals_1_projected = normals_1 / torch.max(torch.Tensor([0.01]), normals_1[2:3,:,:]) normals_mixed = alpha * normals_0_projected + (1.0 - alpha) * normals_1_projected normals_mixed = normals_mixed / torch.sqrt(torch.sum(normals_mixed**2, axis=0,keepdim=True)) # Normalization diffuse_mixed = alpha * diffuse_0 + (1.0 - alpha) * diffuse_1 roughness_mixed = alpha * roughness_0 + (1.0 - alpha) * roughness_1 specular_mixed = alpha * specular_0 + (1.0 - alpha) * specular_1 return utils.pack_svbrdf(normals_mixed, diffuse_mixed, roughness_mixed, specular_mixed)
def render(self, scene, svbrdf): device = svbrdf.device # Generate surface coordinates for the material patch # The center point of the patch is located at (0, 0, 0) which is the center of the global coordinate system. # The patch itself spans from (-1, -1, 0) to (1, 1, 0). xcoords_row = torch.linspace(-1, 1, svbrdf.shape[-1], device=device) xcoords = xcoords_row.unsqueeze(0).expand( svbrdf.shape[-2], svbrdf.shape[-1]).unsqueeze(0) ycoords = -1 * torch.transpose(xcoords, dim0=1, dim1=2) coords = torch.cat((xcoords, ycoords, torch.zeros_like(xcoords)), dim=0) # [x,y,z] (shape = (3)) -> [[[x]], [[y]], [[z]]] (shape = (3, 1, 1)) camera_pos = torch.Tensor( scene.camera.pos).unsqueeze(-1).unsqueeze(-1).to(device) # We treat the center of the material patch as focal point of the camera relative_camera_pos = camera_pos - coords wo = normalize(relative_camera_pos) normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf) # Avoid zero roughness (i. e., potential division by zero) roughness = torch.clamp(roughness, min=0.001) # For each light do: # [x,y,z] (shape = (3)) -> [[[x]], [[y]], [[z]]] (shape = (3, 1, 1)) light_pos = torch.Tensor( scene.light.pos).unsqueeze(-1).unsqueeze(-1).to(device) relative_light_pos = light_pos - coords wi = normalize(relative_light_pos) f = self.evaluate_brdf(wi, wo, normals, diffuse, roughness, specular) LN = torch.clamp(dot_product(wi, normals), min=0.0) # Only consider the upper hemisphere light_color = torch.Tensor(scene.light.color).unsqueeze(-1).unsqueeze( -1).unsqueeze(0).to(device) falloff = 1.0 / torch.sqrt( dot_product( relative_light_pos, relative_light_pos))**2 # Radial light intensity falloff radiance = torch.mul(torch.mul(f, light_color * falloff), LN) # TODO: Add camera exposure return radiance
def forward(self, input): # Split the input of shape (B, N, C, H, W) into a list over the input images [(B, 1, C, H, W)_1, ..., (B, 1, C, H, W)_N] input_images = torch.split(input, 1, dim=1) # Invoke the generator for all the input images encoder_decoder_outputs = [] global_track_outputs = [] for input_image in input_images: encoder_decoder_output, global_track_output = self.generator( input_image.squeeze(1)) encoder_decoder_outputs.append(encoder_decoder_output.unsqueeze(1)) global_track_outputs.append(global_track_output.unsqueeze(1)) # Merge the outputs back into a tensors of shape (B, N, C, H, W) encoder_decoder_outputs = torch.cat(encoder_decoder_outputs, dim=1) global_track_outputs = torch.cat(global_track_outputs, dim=1) # Pool over the input image dimension pooled_encoder_decoder_outputs, _ = torch.max(encoder_decoder_outputs, dim=1) pooled_global_track_outputs, _ = torch.max(global_track_outputs, dim=1) x = self.merge(pooled_encoder_decoder_outputs, pooled_global_track_outputs) mean = torch.mean(pooled_encoder_decoder_outputs, dim=(2, 3), keepdim=False) global_track = self.gt1(mean, pooled_global_track_outputs) x, mean = self.conv1(x, global_track) global_track = self.gt2(mean, global_track) x, mean = self.conv2(x, global_track) global_track = self.gt3(mean, global_track) x, mean = self.conv3(x, global_track) svbrdf = self.activation(x) # 9 channel SVBRDF to 12 channels svbrdf = utils.decode_svbrdf(svbrdf) # Map ranges from [-1, 1] to [0, 1], except for the normals normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf) diffuse = utils.encode_as_unit_interval(diffuse) roughness = utils.encode_as_unit_interval(roughness) specular = utils.encode_as_unit_interval(specular) return utils.pack_svbrdf(normals, diffuse, roughness, specular)
def forward(self, input): if len(input.shape) == 5: # If we get multiple input images, we just ignore all but one input = input[:, 0, :, :, :] svbrdf, _ = self.generator(input) svbrdf = self.activation(svbrdf) # 9 channel SVBRDF to 12 channels svbrdf = utils.decode_svbrdf(svbrdf) # Map ranges from [-1, 1] to [0, 1], except for the normals normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf) diffuse = utils.encode_as_unit_interval(diffuse) roughness = utils.encode_as_unit_interval(roughness) specular = utils.encode_as_unit_interval(specular) return utils.pack_svbrdf(normals, diffuse, roughness, specular)
ax.imshow(img[0].detach().permute(1,2,0), aspect='auto') fig.savefig('/content/experiment1/figures/render3.png') print("size", batch_inputs[0][0].size()) img = batch_inputs[0][0] fig = plt.figure(frameon=False) # fig.set_size_inches(w,h) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.set_axis_off() fig.add_axes(ax) # print("shape", img.size()) ax.imshow(img.detach().permute(1,2,0), aspect='auto') fig.savefig('/content/experiment1/figures/render4.png') print("size", batch_inputs[0][0].size()) normals, diffuse, roughness, specular = utils.unpack_svbrdf(outputs[0]) img = normals fig = plt.figure(frameon=False) # fig.set_size_inches(w,h) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.set_axis_off() fig.add_axes(ax) # print("shape", img.size()) ax.imshow(img.detach().permute(1,2,0), aspect='auto') fig.savefig('/content/experiment1/figures/output_normal.png') img = diffuse fig = plt.figure(frameon=False) # fig.set_size_inches(w,h) ax = plt.Axes(fig, [0., 0., 1., 1.]) ax.set_axis_off()
def render(self, scene, svbrdf): imgs = [] svbrdf = svbrdf.unsqueeze(0) if len(svbrdf.shape) == 3 else svbrdf sensor_size = (svbrdf.shape[-1], svbrdf.shape[-2]) for svbrdf_single in torch.split(svbrdf, 1, dim=0): normals, diffuse, roughness, specular = utils.unpack_svbrdf( svbrdf_single.squeeze(0)) # Redner expects the normal map to be in range [0, 1] normals = utils.encode_as_unit_interval(normals) # Redner expects the roughness to have one channel only. # We also need to convert from GGX roughness to Blinn-Phong power. # See: https://github.com/iondune/csc473/blob/master/lectures/07-cook-torrance.md roughness = torch.mean(torch.clamp(roughness, min=0.001), dim=0, keepdim=True)**4 # Convert from [c,h,w] to [h,w,c] for redner normals = normals.permute(1, 2, 0) diffuse = diffuse.permute(1, 2, 0) roughness = roughness.permute(1, 2, 0) specular = specular.permute(1, 2, 0) material = pyredner.Material( diffuse_reflectance=pyredner.Texture( diffuse.to(self.redner_device)), specular_reflectance=pyredner.Texture( specular.to(self.redner_device)), roughness=pyredner.Texture(roughness.to(self.redner_device)), normal_map=pyredner.Texture(normals.to(self.redner_device))) material_patch = pyredner.Object(vertices=self.patch_vertices, uvs=self.patch_uvs, indices=self.patch_indices, material=material) # Define the camera parameters (focused at the middle of the patch) and make sure we always have a valid 'up' direction position = np.array(scene.camera.pos) lookat = np.array([0.0, 0.0, 0.0]) cz = lookat - position # Principal axis up = np.array([0.0, 0.0, 1.0]) if np.linalg.norm(np.cross(cz, up)) == 0.0: up = np.array([0.0, 1.0, 0.0]) camera = pyredner.Camera( position=torch.FloatTensor(position).to(self.redner_device), look_at=torch.FloatTensor(lookat).to(self.redner_device), up=torch.FloatTensor(up).to(self.redner_device), fov=torch.FloatTensor([90]), resolution=sensor_size, camera_type=self.camera_type) # # The deferred rendering path. # # It does not have a specular model and therefore is of limited usability for us # full_scene = pyredner.Scene(camera = camera, objects = [material_patch]) # light = pyredner.PointLight(position = torch.tensor(scene.light.pos).to(self.redner_device), # intensity = torch.tensor(scene.light.color).to(self.redner_device)) # img = pyredner.render_deferred(scene = full_scene, lights = [light]) light = pyredner.generate_quad_light( position=torch.Tensor(scene.light.pos).to(self.redner_device), look_at=torch.zeros(3).to(self.redner_device), size=torch.Tensor([0.6, 0.6]).to(self.redner_device), intensity=torch.Tensor(scene.light.color).to( self.redner_device)) full_scene = pyredner.Scene(camera=camera, objects=[material_patch, light]) img = pyredner.render_pathtracing(full_scene, num_samples=(16, 8)) # Transform the rendered image back to something torch can interprete imgs.append(img.permute(2, 0, 1).to(svbrdf.device)) return torch.stack(imgs)
perspective_mapping = OrthoToPerspectiveMapping(scene.camera, (600, 600)) fig = plt.figure(figsize=(8, 8)) row_count = 2 * len(data) col_count = 5 for i_row, batch in enumerate(loader): batch_inputs = batch["inputs"] batch_svbrdf = batch["svbrdf"] # We only have one image in the inputs batch_inputs.squeeze_(0) input = utils.gamma_encode(batch_inputs) svbrdf = batch_svbrdf normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf) fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 1) plt.imshow(input.squeeze(0).permute(1, 2, 0)) plt.axis('off') fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 2) plt.imshow( utils.encode_as_unit_interval(normals.squeeze(0).permute(1, 2, 0))) plt.axis('off') fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 3) plt.imshow(diffuse.squeeze(0).permute(1, 2, 0)) plt.axis('off') fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 4)