def forward(self, input, target):
# Split the SVBRDF into its individual components
input_normals, input_diffuse, input_roughness, input_specular = utils.unpack_svbrdf(input)
target_normals, target_diffuse, target_roughness, target_specular = utils.unpack_svbrdf(target)
epsilon_l1 = 0.01
input_diffuse = torch.log(input_diffuse + epsilon_l1)
input_specular = torch.log(input_specular + epsilon_l1)
target_diffuse = torch.log(target_diffuse + epsilon_l1)
target_specular = torch.log(target_specular + epsilon_l1)
return nn.functional.l1_loss(input_normals, target_normals) + nn.functional.l1_loss(input_diffuse, target_diffuse) + nn.functional.l1_loss(input_roughness, target_roughness) + nn.functional.l1_loss(input_specular, target_specular)
def mix(self, svbrdf_0, svbrdf_1, alpha=None):
if alpha is None:
alpha = torch.Tensor(1).uniform_(0.1, 0.9)
normals_0, diffuse_0, roughness_0, specular_0 = utils.unpack_svbrdf(svbrdf_0)
normals_1, diffuse_1, roughness_1, specular_1 = utils.unpack_svbrdf(svbrdf_1)
# Reference "Project the normals to use the X and Y derivative"
normals_0_projected = normals_0 / torch.max(torch.Tensor([0.01]), normals_0[2:3,:,:])
normals_1_projected = normals_1 / torch.max(torch.Tensor([0.01]), normals_1[2:3,:,:])
normals_mixed = alpha * normals_0_projected + (1.0 - alpha) * normals_1_projected
normals_mixed = normals_mixed / torch.sqrt(torch.sum(normals_mixed**2, axis=0,keepdim=True)) # Normalization
diffuse_mixed = alpha * diffuse_0 + (1.0 - alpha) * diffuse_1
roughness_mixed = alpha * roughness_0 + (1.0 - alpha) * roughness_1
specular_mixed = alpha * specular_0 + (1.0 - alpha) * specular_1
return utils.pack_svbrdf(normals_mixed, diffuse_mixed, roughness_mixed, specular_mixed)
def render(self, scene, svbrdf):
device = svbrdf.device
# Generate surface coordinates for the material patch
# The center point of the patch is located at (0, 0, 0) which is the center of the global coordinate system.
# The patch itself spans from (-1, -1, 0) to (1, 1, 0).
xcoords_row = torch.linspace(-1, 1, svbrdf.shape[-1], device=device)
xcoords = xcoords_row.unsqueeze(0).expand(
svbrdf.shape[-2], svbrdf.shape[-1]).unsqueeze(0)
ycoords = -1 * torch.transpose(xcoords, dim0=1, dim1=2)
coords = torch.cat((xcoords, ycoords, torch.zeros_like(xcoords)),
dim=0)
# [x,y,z] (shape = (3)) -> [[[x]], [[y]], [[z]]] (shape = (3, 1, 1))
camera_pos = torch.Tensor(
scene.camera.pos).unsqueeze(-1).unsqueeze(-1).to(device)
# We treat the center of the material patch as focal point of the camera
relative_camera_pos = camera_pos - coords
wo = normalize(relative_camera_pos)
normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf)
# Avoid zero roughness (i. e., potential division by zero)
roughness = torch.clamp(roughness, min=0.001)
# For each light do:
# [x,y,z] (shape = (3)) -> [[[x]], [[y]], [[z]]] (shape = (3, 1, 1))
light_pos = torch.Tensor(
scene.light.pos).unsqueeze(-1).unsqueeze(-1).to(device)
relative_light_pos = light_pos - coords
wi = normalize(relative_light_pos)
f = self.evaluate_brdf(wi, wo, normals, diffuse, roughness, specular)
LN = torch.clamp(dot_product(wi, normals),
min=0.0) # Only consider the upper hemisphere
light_color = torch.Tensor(scene.light.color).unsqueeze(-1).unsqueeze(
-1).unsqueeze(0).to(device)
falloff = 1.0 / torch.sqrt(
dot_product(
relative_light_pos,
relative_light_pos))**2 # Radial light intensity falloff
radiance = torch.mul(torch.mul(f, light_color * falloff), LN)
# TODO: Add camera exposure
return radiance
def forward(self, input):
# Split the input of shape (B, N, C, H, W) into a list over the input images [(B, 1, C, H, W)_1, ..., (B, 1, C, H, W)_N]
input_images = torch.split(input, 1, dim=1)
# Invoke the generator for all the input images
encoder_decoder_outputs = []
global_track_outputs = []
for input_image in input_images:
encoder_decoder_output, global_track_output = self.generator(
input_image.squeeze(1))
encoder_decoder_outputs.append(encoder_decoder_output.unsqueeze(1))
global_track_outputs.append(global_track_output.unsqueeze(1))
# Merge the outputs back into a tensors of shape (B, N, C, H, W)
encoder_decoder_outputs = torch.cat(encoder_decoder_outputs, dim=1)
global_track_outputs = torch.cat(global_track_outputs, dim=1)
# Pool over the input image dimension
pooled_encoder_decoder_outputs, _ = torch.max(encoder_decoder_outputs,
dim=1)
pooled_global_track_outputs, _ = torch.max(global_track_outputs, dim=1)
x = self.merge(pooled_encoder_decoder_outputs,
pooled_global_track_outputs)
mean = torch.mean(pooled_encoder_decoder_outputs,
dim=(2, 3),
keepdim=False)
global_track = self.gt1(mean, pooled_global_track_outputs)
x, mean = self.conv1(x, global_track)
global_track = self.gt2(mean, global_track)
x, mean = self.conv2(x, global_track)
global_track = self.gt3(mean, global_track)
x, mean = self.conv3(x, global_track)
svbrdf = self.activation(x)
# 9 channel SVBRDF to 12 channels
svbrdf = utils.decode_svbrdf(svbrdf)
# Map ranges from [-1, 1] to [0, 1], except for the normals
normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf)
diffuse = utils.encode_as_unit_interval(diffuse)
roughness = utils.encode_as_unit_interval(roughness)
specular = utils.encode_as_unit_interval(specular)
return utils.pack_svbrdf(normals, diffuse, roughness, specular)
def forward(self, input):
if len(input.shape) == 5:
# If we get multiple input images, we just ignore all but one
input = input[:, 0, :, :, :]
svbrdf, _ = self.generator(input)
svbrdf = self.activation(svbrdf)
# 9 channel SVBRDF to 12 channels
svbrdf = utils.decode_svbrdf(svbrdf)
# Map ranges from [-1, 1] to [0, 1], except for the normals
normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf)
diffuse = utils.encode_as_unit_interval(diffuse)
roughness = utils.encode_as_unit_interval(roughness)
specular = utils.encode_as_unit_interval(specular)
return utils.pack_svbrdf(normals, diffuse, roughness, specular)
ax.imshow(img[0].detach().permute(1,2,0), aspect='auto')
fig.savefig('/content/experiment1/figures/render3.png')
print("size", batch_inputs[0][0].size())
img = batch_inputs[0][0]
fig = plt.figure(frameon=False)
# fig.set_size_inches(w,h)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
# print("shape", img.size())
ax.imshow(img.detach().permute(1,2,0), aspect='auto')
fig.savefig('/content/experiment1/figures/render4.png')
print("size", batch_inputs[0][0].size())
normals, diffuse, roughness, specular = utils.unpack_svbrdf(outputs[0])
img = normals
fig = plt.figure(frameon=False)
# fig.set_size_inches(w,h)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
fig.add_axes(ax)
# print("shape", img.size())
ax.imshow(img.detach().permute(1,2,0), aspect='auto')
fig.savefig('/content/experiment1/figures/output_normal.png')
img = diffuse
fig = plt.figure(frameon=False)
# fig.set_size_inches(w,h)
ax = plt.Axes(fig, [0., 0., 1., 1.])
ax.set_axis_off()
def render(self, scene, svbrdf):
imgs = []
svbrdf = svbrdf.unsqueeze(0) if len(svbrdf.shape) == 3 else svbrdf
sensor_size = (svbrdf.shape[-1], svbrdf.shape[-2])
for svbrdf_single in torch.split(svbrdf, 1, dim=0):
normals, diffuse, roughness, specular = utils.unpack_svbrdf(
svbrdf_single.squeeze(0))
# Redner expects the normal map to be in range [0, 1]
normals = utils.encode_as_unit_interval(normals)
# Redner expects the roughness to have one channel only.
# We also need to convert from GGX roughness to Blinn-Phong power.
# See: https://github.com/iondune/csc473/blob/master/lectures/07-cook-torrance.md
roughness = torch.mean(torch.clamp(roughness, min=0.001),
dim=0,
keepdim=True)**4
# Convert from [c,h,w] to [h,w,c] for redner
normals = normals.permute(1, 2, 0)
diffuse = diffuse.permute(1, 2, 0)
roughness = roughness.permute(1, 2, 0)
specular = specular.permute(1, 2, 0)
material = pyredner.Material(
diffuse_reflectance=pyredner.Texture(
diffuse.to(self.redner_device)),
specular_reflectance=pyredner.Texture(
specular.to(self.redner_device)),
roughness=pyredner.Texture(roughness.to(self.redner_device)),
normal_map=pyredner.Texture(normals.to(self.redner_device)))
material_patch = pyredner.Object(vertices=self.patch_vertices,
uvs=self.patch_uvs,
indices=self.patch_indices,
material=material)
# Define the camera parameters (focused at the middle of the patch) and make sure we always have a valid 'up' direction
position = np.array(scene.camera.pos)
lookat = np.array([0.0, 0.0, 0.0])
cz = lookat - position # Principal axis
up = np.array([0.0, 0.0, 1.0])
if np.linalg.norm(np.cross(cz, up)) == 0.0:
up = np.array([0.0, 1.0, 0.0])
camera = pyredner.Camera(
position=torch.FloatTensor(position).to(self.redner_device),
look_at=torch.FloatTensor(lookat).to(self.redner_device),
up=torch.FloatTensor(up).to(self.redner_device),
fov=torch.FloatTensor([90]),
resolution=sensor_size,
camera_type=self.camera_type)
# # The deferred rendering path.
# # It does not have a specular model and therefore is of limited usability for us
# full_scene = pyredner.Scene(camera = camera, objects = [material_patch])
# light = pyredner.PointLight(position = torch.tensor(scene.light.pos).to(self.redner_device),
# intensity = torch.tensor(scene.light.color).to(self.redner_device))
# img = pyredner.render_deferred(scene = full_scene, lights = [light])
light = pyredner.generate_quad_light(
position=torch.Tensor(scene.light.pos).to(self.redner_device),
look_at=torch.zeros(3).to(self.redner_device),
size=torch.Tensor([0.6, 0.6]).to(self.redner_device),
intensity=torch.Tensor(scene.light.color).to(
self.redner_device))
full_scene = pyredner.Scene(camera=camera,
objects=[material_patch, light])
img = pyredner.render_pathtracing(full_scene, num_samples=(16, 8))
# Transform the rendered image back to something torch can interprete
imgs.append(img.permute(2, 0, 1).to(svbrdf.device))
return torch.stack(imgs)
perspective_mapping = OrthoToPerspectiveMapping(scene.camera, (600, 600))
fig = plt.figure(figsize=(8, 8))
row_count = 2 * len(data)
col_count = 5
for i_row, batch in enumerate(loader):
batch_inputs = batch["inputs"]
batch_svbrdf = batch["svbrdf"]
# We only have one image in the inputs
batch_inputs.squeeze_(0)
input = utils.gamma_encode(batch_inputs)
svbrdf = batch_svbrdf
normals, diffuse, roughness, specular = utils.unpack_svbrdf(svbrdf)
fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 1)
plt.imshow(input.squeeze(0).permute(1, 2, 0))
plt.axis('off')
fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 2)
plt.imshow(
utils.encode_as_unit_interval(normals.squeeze(0).permute(1, 2, 0)))
plt.axis('off')
fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 3)
plt.imshow(diffuse.squeeze(0).permute(1, 2, 0))
plt.axis('off')
fig.add_subplot(row_count, col_count, 2 * i_row * col_count + 4)
