def parse_material(node, two_sided=False):
node_id = None
if 'id' in node.attrib:
node_id = node.attrib['id']
if node.attrib['type'] == 'diffuse':
diffuse_reflectance = torch.tensor([0.5, 0.5, 0.5])
diffuse_uv_scale = torch.tensor([1.0, 1.0])
specular_reflectance = torch.tensor([0.0, 0.0, 0.0])
specular_uv_scale = torch.tensor([1.0, 1.0])
roughness = torch.tensor([1.0])
for child in node:
if child.attrib['name'] == 'reflectance':
if child.tag == 'texture':
for grandchild in child:
if grandchild.attrib['name'] == 'filename':
diffuse_reflectance = pyredner.imread(
grandchild.attrib['value'])
elif grandchild.attrib['name'] == 'uscale':
diffuse_uv_scale[0] = float(
grandchild.attrib['value'])
elif grandchild.attrib['name'] == 'vscale':
diffuse_uv_scale[1] = float(
grandchild.attrib['value'])
elif child.tag == 'rgb':
diffuse_reflectance = parse_vector(child.attrib['value'])
elif child.attrib['name'] == 'specular':
if child.tag == 'texture':
for grandchild in child:
if grandchild.attrib['name'] == 'filename':
specular_reflectance = image.imread(
grandchild.attrib['value'])
elif grandchild.attrib['name'] == 'uscale':
specular_uv_scale[0] = float(
grandchild.attrib['value'])
elif grandchild.attrib['name'] == 'vscale':
specular_uv_scale[1] = float(
grandchild.attrib['value'])
elif child.tag == 'rgb':
specular_reflectance = parse_vector(child.attrib['value'])
elif child.attrib['name'] == 'roughness':
roughness = torch.tensor(float(child.attrib['value']))
if pyredner.get_use_gpu():
# Copy to GPU
diffuse_reflectance = diffuse_reflectance.cuda()
specular_reflectance = specular_reflectance.cuda()
roughness = roughness.cuda()
return (node_id, pyredner.Material(\
diffuse_reflectance = pyredner.Texture(diffuse_reflectance, diffuse_uv_scale),
specular_reflectance = pyredner.Texture(specular_reflectance, specular_uv_scale),
roughness = pyredner.Texture(roughness),
two_sided = two_sided))
elif node.attrib['type'] == 'twosided':
ret = parse_material(node[0], True)
return (node_id, ret[1])
def replace_tex_with_color(self, color):
self.tex_backup = self.mesh.material.diffuse_reflectance.texels.clone().detach()
new_tex_img = torch.ones(self.tex_backup.shape, device=pyr.get_device())
new_tex_img[:,:,0] = color[0]
new_tex_img[:,:,1] = color[1]
new_tex_img[:,:,2] = color[2]
self.mesh.material.diffuse_reflectance = pyr.Texture(new_tex_img)
def free_tex(self):
self.learning_texture = True
self.learned_tex = self.mesh.material.diffuse_reflectance.texels.clone().detach()
self.learned_tex.requires_grad = True
self.params.append(self.learned_tex)
self.mesh.material.diffuse_reflectance = pyr.Texture(self.learned_tex)
return self
def __init__(self, values, env_to_world = torch.eye(4, 4)):
# Convert to constant texture if necessary
if isinstance(values, torch.Tensor):
values = pyredner.Texture(values)
assert(env_to_world.device.type == 'cpu')
assert(env_to_world.dtype == torch.float32)
assert(env_to_world.is_contiguous())
# Build sampling table
luminance = 0.212671 * values.texels[:, :, 0] + \
0.715160 * values.texels[:, :, 1] + \
0.072169 * values.texels[:, :, 2]
# For each y, compute CDF over x
sample_cdf_xs_ = torch.cumsum(luminance, dim = 1)
y_weight = torch.sin(\
math.pi * (torch.arange(luminance.shape[0],
dtype = torch.float32, device = luminance.device) + 0.5) \
/ float(luminance.shape[0]))
# Compute CDF for x
sample_cdf_ys_ = torch.cumsum(sample_cdf_xs_[:, -1] * y_weight, dim = 0)
pdf_norm = (luminance.shape[0] * luminance.shape[1]) / \
(sample_cdf_ys_[-1].item() * (2 * math.pi * math.pi))
# Normalize to [0, 1)
sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \
torch.max(sample_cdf_xs_[:, (luminance.shape[1] - 1):luminance.shape[1]],
1e-8 * torch.ones(sample_cdf_xs_.shape[0], 1, device = sample_cdf_ys_.device))
sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \
torch.max(sample_cdf_ys_[-1], torch.tensor([1e-8], device = sample_cdf_ys_.device))
self.values = values
self.env_to_world = env_to_world
self.world_to_env = torch.inverse(env_to_world).contiguous()
self.sample_cdf_ys = sample_cdf_ys.contiguous()
self.sample_cdf_xs = sample_cdf_xs.contiguous()
self.pdf_norm = pdf_norm
def restore_tex(self):
'''
Restore the texture from the backup. Only works once per backup!
'''
assert self.tex_backup is not None
self.mesh.material.diffuse_reflectance = pyr.Texture(self.tex_backup)
self.tex_backup = None
def __init__(
self,
diffuse_reflectance: Optional[Union[torch.Tensor,
pyredner.Texture]] = None,
specular_reflectance: Optional[Union[torch.Tensor,
pyredner.Texture]] = None,
roughness: Optional[Union[torch.Tensor, pyredner.Texture]] = None,
generic_texture: Optional[Union[torch.Tensor,
pyredner.Texture]] = None,
normal_map: Optional[Union[torch.Tensor, pyredner.Texture]] = None,
two_sided: bool = False,
use_vertex_color: bool = False):
if diffuse_reflectance is None:
diffuse_reflectance = pyredner.Texture(\
torch.zeros(3, device = pyredner.get_device()))
if specular_reflectance is None:
specular_reflectance = pyredner.Texture(\
torch.zeros(3, device = pyredner.get_device()))
compute_specular_lighting = False
else:
compute_specular_lighting = True
if roughness is None:
roughness = pyredner.Texture(\
torch.tensor([1.0], device = pyredner.get_device()))
# Convert to constant texture if necessary
if isinstance(diffuse_reflectance, torch.Tensor):
diffuse_reflectance = pyredner.Texture(diffuse_reflectance)
if isinstance(specular_reflectance, torch.Tensor):
specular_reflectance = pyredner.Texture(specular_reflectance)
if isinstance(roughness, torch.Tensor):
roughness = pyredner.Texture(roughness)
if generic_texture is not None and isinstance(generic_texture,
torch.Tensor):
generic_texture = pyredner.Texture(generic_texture)
if normal_map is not None and isinstance(normal_map, torch.Tensor):
normal_map = pyredner.Texture(normal_map)
'''
assert((len(diffuse_reflectance.texels.shape) == 1 and diffuse_reflectance.texels.shape[0] == 3) or \
(len(diffuse_reflectance.texels.shape) == 3 and diffuse_reflectance.texels.shape[2] == 3))
assert((len(specular_reflectance.texels.shape) == 1 and specular_reflectance.texels.shape[0] == 3) or \
(len(specular_reflectance.texels.shape) == 3 and specular_reflectance.texels.shape[2] == 3))
assert((len(roughness.texels.shape) == 1 and roughness.texels.shape[0] == 1) or \
(len(roughness.texels.shape) == 3 and roughness.texels.shape[2] == 1))
if normal_map is not None:
assert((len(normal_map.texels.shape) == 1 and normal_map.texels.shape[0] == 3) or \
(len(normal_map.texels.shape) == 3 and normal_map.texels.shape[2] == 3))
'''
self.diffuse_reflectance = diffuse_reflectance
self._specular_reflectance = specular_reflectance
self.compute_specular_lighting = compute_specular_lighting
self.roughness = roughness
self.generic_texture = generic_texture
self.normal_map = normal_map
self.two_sided = two_sided
self.use_vertex_color = use_vertex_color
def specular_reflectance(self, value):
self._specular_reflectance = value
if value is not None:
self.compute_specular_lighting = True
else:
self._specular_reflectance = pyredner.Texture(\
torch.zeros(3, device = pyredner.get_device()))
self.compute_specular_lighting = False
def __init__(self,
diffuse_reflectance,
specular_reflectance=None,
roughness=None,
normal_map=None,
two_sided=False):
if specular_reflectance is None:
specular_reflectance = pyredner.Texture(\
torch.tensor([0.0,0.0,0.0], device = pyredner.get_device()))
if roughness is None:
roughness = pyredner.Texture(\
torch.tensor([1.0], device = pyredner.get_device()))
# Convert to constant texture if necessary
if isinstance(diffuse_reflectance, torch.Tensor):
diffuse_reflectance = pyredner.Texture(diffuse_reflectance)
if isinstance(specular_reflectance, torch.Tensor):
specular_reflectance = pyredner.Texture(specular_reflectance)
if isinstance(roughness, torch.Tensor):
roughness = pyredner.Texture(roughness)
if normal_map is not None and isinstance(normal_map, torch.Tensor):
normal_map = pyredner.Texture(normal_map)
self.diffuse_reflectance = diffuse_reflectance
self.specular_reflectance = specular_reflectance
self.roughness = roughness
self.normal_map = normal_map
self.two_sided = two_sided
def __init__(self,
diffuse_reflectance,
specular_reflectance=None,
roughness=None,
two_sided=False):
if specular_reflectance is None:
specular_reflectance = pyredner.Texture(\
torch.tensor([0.0,0.0,0.0], device = pyredner.get_device()))
if roughness is None:
roughness = pyredner.Texture(\
torch.tensor([1.0], device = pyredner.get_device()))
# Convert to constant texture if necessary
if isinstance(diffuse_reflectance, torch.Tensor):
diffuse_reflectance = pyredner.Texture(diffuse_reflectance)
if isinstance(specular_reflectance, torch.Tensor):
specular_reflectance = pyredner.Texture(specular_reflectance)
if isinstance(roughness, torch.Tensor):
roughness = pyredner.Texture(roughness)
assert (diffuse_reflectance.texels.is_contiguous())
assert (diffuse_reflectance.texels.dtype == torch.float32)
assert (specular_reflectance.texels.is_contiguous())
assert (specular_reflectance.texels.dtype == torch.float32)
assert (roughness.texels.is_contiguous())
assert (roughness.texels.dtype == torch.float32)
if pyredner.get_use_gpu():
assert (diffuse_reflectance.texels.is_cuda)
assert (specular_reflectance.texels.is_cuda)
assert (roughness.texels.is_cuda)
else:
assert (not diffuse_reflectance.texels.is_cuda)
assert (not specular_reflectance.texels.is_cuda)
assert (not roughness.texels.is_cuda)
self.diffuse_reflectance = diffuse_reflectance
self.specular_reflectance = specular_reflectance
self.roughness = roughness
self.two_sided = two_sided
def __init__(self,
values: Union[torch.Tensor, pyredner.Texture],
env_to_world: torch.Tensor = torch.eye(4, 4)):
# Convert to constant texture if necessary
if isinstance(values, torch.Tensor):
values = pyredner.Texture(values)
assert(env_to_world.dtype == torch.float32)
assert(env_to_world.is_contiguous())
self._values = values
self._env_to_world = env_to_world
self.world_to_env = torch.inverse(env_to_world).contiguous()
self.generate_envmap_pdf()
def model(cam_poses, cam_look_ats, shape_coeffs, color_coeffs, resolution):
# First rotate around center, then translation
imgs = []
vertices = (shape_mean + shape_basis @ shape_coeffs).view(-1, 3)
normals = pyredner.compute_vertex_normal(vertices, indices)
colors = (color_mean + color_basis @ color_coeffs).view(-1, 3)
m = pyredner.Material(use_vertex_color=False,
specular_reflectance=torch.tensor(
[1., 1., 1.], device=pyredner.get_device()),
roughness=torch.tensor([0.02]))
obj = pyredner.Object(vertices=vertices,
indices=indices,
normals=normals,
material=m,
colors=colors)
obj = pyredner.load_obj('generated/env_dataset_oness_n/tgt_obj.obj',
return_objects=True)[0]
obj.material.specular_reflectance = pyredner.Texture(
torch.tensor([0.05, 0.05, 0.05], device=pyredner.get_device()))
obj.material.roughness = pyredner.Texture(torch.tensor([0.02]))
pyredner.save_obj(obj, "generated/senv_dataset_" + name + '/tgt_obj.obj')
for i in range(len(cam_poses)):
cam = pyredner.Camera(
position=cam_poses[i],
look_at=cam_look_ats[i %
len(cam_look_ats)], # Center of the vertices
up=torch.tensor([0.0, 1.0, 0.0]),
fov=torch.tensor([45.0]),
resolution=resolution)
scene = pyredner.Scene(camera=cam, objects=[obj], envmap=envmap)
img = pyredner.render_pathtracing(scene=scene, num_samples=(128, 4))
imgs.append(img)
return imgs
def load_3d(mesh_name):
'''
Loads a 3D model, computing vertex normals as needed
'''
dpath = os.path.join(mydir, mesh_name)
fpath = os.path.join(dpath, "mesh.obj")
if os.path.isfile(fpath):
obj = pyr.load_obj(fpath, return_objects=True)[0]
recompute_normals(obj)
texpath = os.path.join(dpath, "texture.png")
if os.path.isfile(texpath):
tex_img = pyr.imread(texpath)
obj.material.diffuse_reflectance = pyr.Texture(tex_img)
return obj
else:
raise FileNotFoundError(f"Could not find {mesh_name}.obj")
def __init__(self, values, env_to_world=tf.eye(4, 4)):
# Convert to constant texture if necessary
if isinstance(values, tf.Tensor):
values = pyredner.Texture(values)
assert (values.texels.is_contiguous())
assert (values.texels.dtype == tf.float32)
if pyredner.get_use_gpu():
assert (values.texels.is_cuda)
else:
assert (not values.texels.is_cuda)
# Build sampling table
luminance = 0.212671 * values.texels[:, :, 0] + \
0.715160 * values.texels[:, :, 1] + \
0.072169 * values.texels[:, :, 2]
# For each y, compute CDF over x
sample_cdf_xs_ = tf.cumsum(luminance, axis=1)
y_weight = tf.sin(\
math.pi * (tf.range(luminance.shape[0],
dtype = tf.float32, name='EnvironmentMap.y_weight') + 0.5) \
/ float(luminance.shape[0]))
# Compute CDF for x
sample_cdf_ys_ = tf.cumsum(sample_cdf_xs_[:, -1] * y_weight, axis=0)
pdf_norm = (luminance.shape[0] * luminance.shape[1]) / \
(sample_cdf_ys_[-1].item() * (2 * math.pi * math.pi))
# Normalize to [0, 1)
sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \
tf.math.maximum(sample_cdf_xs_[:, (luminance.shape[1] - 1):luminance.shape[1]],
1e-8 * tf.ones(
(sample_cdf_xs_.shape[0], 1)
)
)
sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \
tf.math.maximum(sample_cdf_ys_[-1], tf.constant([1e-8]))
self.values = values
self.env_to_world = env_to_world
self.world_to_env = tf.linalg.inv(env_to_world).contiguous()
self.sample_cdf_ys = sample_cdf_ys.contiguous()
self.sample_cdf_xs = sample_cdf_xs.contiguous()
self.pdf_norm = pdf_norm
def parse_material(node, two_sided=False):
def parse_material_bitmap(node, scale=None):
reflectance_texture = None
uv_scale = torch.tensor([1.0, 1.0])
for grandchild in node:
if grandchild.attrib['name'] == 'filename':
reflectance_texture = pyredner.imread(
grandchild.attrib['value'])
if scale:
reflectance_texture = reflectance_texture * scale
elif grandchild.attrib['name'] == 'uscale':
uv_scale[0] = float(grandchild.attrib['value'])
elif grandchild.attrib['name'] == 'vscale':
uv_scale[1] = float(grandchild.attrib['value'])
assert reflectance_texture is not None
return reflectance_texture, uv_scale
# support mitsuba pulgin 'scale' for texture
def parse_texture(node):
if node.attrib['type'] == 'scale':
scale_value = None
for grandchild in node:
if grandchild.attrib[
'name'] == 'scale' and grandchild.tag == 'float':
scale_value = float(grandchild.attrib['value'])
elif grandchild.attrib[
'type'] == 'bitmap' and grandchild.tag == 'texture':
assert scale_value is not None # avoid 'scale' element is declared below the 'bitmap'
return parse_material_bitmap(grandchild, scale_value)
else:
raise NotImplementedError(
'Unsupported scale param type {}'.format(
grandchild.child['type']))
elif node.attrib['type'] == 'bitmap':
return parse_material_bitmap(node)
else:
raise NotImplementedError('Unsupported Texture type {}'.format(
node.attrib['type']))
node_id = None
if 'id' in node.attrib:
node_id = node.attrib['id']
if node.attrib['type'] == 'diffuse':
diffuse_reflectance = torch.tensor([0.5, 0.5, 0.5])
diffuse_uv_scale = torch.tensor([1.0, 1.0])
specular_reflectance = torch.tensor([0.0, 0.0, 0.0])
specular_uv_scale = torch.tensor([1.0, 1.0])
roughness = torch.tensor([1.0])
for child in node:
if child.attrib['name'] == 'reflectance':
if child.tag == 'texture':
diffuse_reflectance, diffuse_uv_scale = parse_texture(
child)
elif child.tag == 'rgb' or child.tag == 'spectrum':
diffuse_reflectance = parse_vector(child.attrib['value'])
elif child.attrib['name'] == 'specular':
if child.tag == 'texture':
specular_reflectance, specular_uv_scale = parse_texture(
child)
elif child.tag == 'rgb' or child.tag == 'spectrum':
specular_reflectance = parse_vector(child.attrib['value'])
elif child.attrib['name'] == 'roughness':
roughness = torch.tensor([float(child.attrib['value'])])
if pyredner.get_use_gpu():
# Copy to GPU
diffuse_reflectance = diffuse_reflectance.cuda()
specular_reflectance = specular_reflectance.cuda()
roughness = roughness.cuda()
return (node_id, pyredner.Material(\
diffuse_reflectance = pyredner.Texture(diffuse_reflectance, diffuse_uv_scale),
specular_reflectance = pyredner.Texture(specular_reflectance, specular_uv_scale),
roughness = pyredner.Texture(roughness),
two_sided = two_sided))
elif node.attrib['type'] == 'roughplastic':
diffuse_reflectance = torch.tensor([0.5, 0.5, 0.5])
diffuse_uv_scale = torch.tensor([1.0, 1.0])
specular_reflectance = torch.tensor([0.0, 0.0, 0.0])
specular_uv_scale = torch.tensor([1.0, 1.0])
roughness = torch.tensor([1.0])
roughness_uv_scale = torch.tensor([1.0, 1.0])
for child in node:
if child.attrib['name'] == 'diffuseReflectance':
if child.tag == 'texture':
diffuse_reflectance, diffuse_uv_scale = parse_texture(
child)
elif child.tag == 'rgb' or child.tag == 'spectrum':
diffuse_reflectance = parse_vector(child.attrib['value'])
elif child.attrib['name'] == 'specularReflectance':
if child.tag == 'texture':
specular_reflectance, specular_uv_scale = parse_texture(
child)
elif child.tag == 'rgb' or child.tag == 'spectrum':
specular_reflectance = parse_vector(child.attrib['value'])
elif child.attrib['name'] == 'alpha':
# Add 'alpha texture' support
if child.tag == 'texture':
#TODO KJL
roughness, roughness_uv_scale = parse_texture(
child) #? not sure to do square here
elif child.tag == 'float':
alpha = float(child.attrib['value'])
roughness = torch.tensor([alpha * alpha])
if pyredner.get_use_gpu():
# Copy to GPU
diffuse_reflectance = diffuse_reflectance.cuda()
specular_reflectance = specular_reflectance.cuda()
roughness = roughness.cuda()
return (node_id, pyredner.Material(\
diffuse_reflectance = pyredner.Texture(diffuse_reflectance, diffuse_uv_scale),
specular_reflectance = pyredner.Texture(specular_reflectance, specular_uv_scale),
roughness = pyredner.Texture(roughness, roughness_uv_scale),
two_sided = two_sided))
elif node.attrib['type'] == 'twosided':
ret = parse_material(node[0], True)
return (node_id, ret[1])
# Simply bypass mask's opacity
elif node.attrib['type'] == 'mask': #TODO add opacity!!!
ret = parse_material(node[0])
return (node_id, ret[1])
else:
print('Unsupported material type:', node.attrib['type'])
assert (False)
def render(self, scene, svbrdf):
imgs = []
svbrdf = svbrdf.unsqueeze(0) if len(svbrdf.shape) == 3 else svbrdf
sensor_size = (svbrdf.shape[-1], svbrdf.shape[-2])
for svbrdf_single in torch.split(svbrdf, 1, dim=0):
normals, diffuse, roughness, specular = utils.unpack_svbrdf(
svbrdf_single.squeeze(0))
# Redner expects the normal map to be in range [0, 1]
normals = utils.encode_as_unit_interval(normals)
# Redner expects the roughness to have one channel only.
# We also need to convert from GGX roughness to Blinn-Phong power.
# See: https://github.com/iondune/csc473/blob/master/lectures/07-cook-torrance.md
roughness = torch.mean(torch.clamp(roughness, min=0.001),
dim=0,
keepdim=True)**4
# Convert from [c,h,w] to [h,w,c] for redner
normals = normals.permute(1, 2, 0)
diffuse = diffuse.permute(1, 2, 0)
roughness = roughness.permute(1, 2, 0)
specular = specular.permute(1, 2, 0)
material = pyredner.Material(
diffuse_reflectance=pyredner.Texture(
diffuse.to(self.redner_device)),
specular_reflectance=pyredner.Texture(
specular.to(self.redner_device)),
roughness=pyredner.Texture(roughness.to(self.redner_device)),
normal_map=pyredner.Texture(normals.to(self.redner_device)))
material_patch = pyredner.Object(vertices=self.patch_vertices,
uvs=self.patch_uvs,
indices=self.patch_indices,
material=material)
# Define the camera parameters (focused at the middle of the patch) and make sure we always have a valid 'up' direction
position = np.array(scene.camera.pos)
lookat = np.array([0.0, 0.0, 0.0])
cz = lookat - position # Principal axis
up = np.array([0.0, 0.0, 1.0])
if np.linalg.norm(np.cross(cz, up)) == 0.0:
up = np.array([0.0, 1.0, 0.0])
camera = pyredner.Camera(
position=torch.FloatTensor(position).to(self.redner_device),
look_at=torch.FloatTensor(lookat).to(self.redner_device),
up=torch.FloatTensor(up).to(self.redner_device),
fov=torch.FloatTensor([90]),
resolution=sensor_size,
camera_type=self.camera_type)
# # The deferred rendering path.
# # It does not have a specular model and therefore is of limited usability for us
# full_scene = pyredner.Scene(camera = camera, objects = [material_patch])
# light = pyredner.PointLight(position = torch.tensor(scene.light.pos).to(self.redner_device),
# intensity = torch.tensor(scene.light.color).to(self.redner_device))
# img = pyredner.render_deferred(scene = full_scene, lights = [light])
light = pyredner.generate_quad_light(
position=torch.Tensor(scene.light.pos).to(self.redner_device),
look_at=torch.zeros(3).to(self.redner_device),
size=torch.Tensor([0.6, 0.6]).to(self.redner_device),
intensity=torch.Tensor(scene.light.color).to(
self.redner_device))
full_scene = pyredner.Scene(camera=camera,
objects=[material_patch, light])
img = pyredner.render_pathtracing(full_scene, num_samples=(16, 8))
# Transform the rendered image back to something torch can interprete
imgs.append(img.permute(2, 0, 1).to(svbrdf.device))
return torch.stack(imgs)
target = pyredner.imread('results/test_multichannels/target_test1.png')
if pyredner.get_use_gpu():
target = target.cuda(device=pyredner.get_device())
generic_texture = generic_texture = torch.zeros(\
128, 128, 16, device = pyredner.get_device(), requires_grad=True)
materials = [pyredner.Material(generic_texture=generic_texture)]
scene = pyredner.Scene(cam, shapes, materials, area_lights=[], envmap=None)
optimizer = torch.optim.Adam([generic_texture], lr=1e-2)
for t in range(200):
print('iteration:', t)
optimizer.zero_grad()
scene.materials[0].generic_texture = pyredner.Texture(generic_texture)
scene_args = pyredner.RenderFunction.serialize_scene(\
scene = scene,
num_samples = 4, # Still need some samples for anti-aliasing
max_bounces = 0,
channels = [redner.channels.generic_texture])
g_buffer = render(t, *scene_args)
img = g_buffer[:, :, 12:15]
loss = (img - target).pow(2).sum()
print('loss:', loss.item())
loss.backward()
optimizer.step()
# Save the images
target = target.cuda()
# Our initial guess is three gray textures
diffuse_tex = torch.tensor(\
np.ones((256, 256, 3), dtype=np.float32) * 0.5,
requires_grad = True,
device = pyredner.get_device())
specular_tex = torch.tensor(\
np.ones((256, 256, 3), dtype=np.float32) * 0.5,
requires_grad = True,
device = pyredner.get_device())
roughness_tex = torch.tensor(\
np.ones((256, 256, 1), dtype=np.float32) * 0.5,
requires_grad = True,
device = pyredner.get_device())
mat_perlin.diffuse_reflectance = pyredner.Texture(diffuse_tex)
mat_perlin.specular_reflectance = pyredner.Texture(specular_tex)
mat_perlin.roughness = pyredner.Texture(roughness_tex)
args = pyredner.RenderFunction.serialize_scene(\
scene = scene,
num_samples = 16,
max_bounces = 1)
# Render the initial guess
img = render(1, *args)
pyredner.imwrite(img.cpu(), 'results/test_svbrdf/init.png')
diff = torch.abs(target - img)
pyredner.imwrite(diff.cpu(), 'results/test_svbrdf/init_diff.png')
# Optimize for triangle vertices
optimizer = torch.optim.Adam([diffuse_tex, specular_tex, roughness_tex],
lr=1e-2)
# Setup camera
cam = pyredner.Camera(
position=torch.tensor([0.0, 30.0, 200.0]),
look_at=torch.tensor([0.0, 30.0, 0.0]),
up=torch.tensor([0.0, 1.0, 0.0]),
fov=torch.tensor([45.0]), # in degree
clip_near=1e-2, # needs to > 0
resolution=(256, 256),
fisheye=False)
# Load normal map (downloaded from https://worldwidemuseum.wordpress.com/2013/01/13/carriage-plates-step-2/)
normal_map = pyredner.imread('scenes/brick_normal.jpg', gamma=1.0)
if pyredner.get_use_gpu():
normal_map = normal_map.cuda(device=pyredner.get_device())
normal_map = pyredner.Texture(normal_map,
uv_scale=torch.tensor(
[4.0, 4.0], device=pyredner.get_device()))
# Setup materials
material_id_map = {}
materials = []
count = 0
for key, value in material_map.items():
material_id_map[key] = count
count += 1
# assign normal map
value.normal_map = normal_map
materials.append(value)
# Setup geometries
shapes = []
import pyredner
import numpy as np
import torch
# Optimize for a textured plane in a specular reflection
# Use GPU if available
pyredner.set_use_gpu(torch.cuda.is_available())
# Load the scene from a Mitsuba scene file
scene = pyredner.load_mitsuba('scenes/teapot_specular.xml')
# The last material is the teapot material, set it to a specular material
scene.materials[-1].diffuse_reflectance = \
pyredner.Texture(torch.tensor([0.15, 0.2, 0.15], device = pyredner.get_device()))
scene.materials[-1].specular_reflectance = \
pyredner.Texture(torch.tensor([0.8, 0.8, 0.8], device = pyredner.get_device()))
scene.materials[-1].roughness = \
pyredner.Texture(torch.tensor([0.0001], device = pyredner.get_device()))
args=pyredner.RenderFunction.serialize_scene(\
scene = scene,
num_samples = 512,
max_bounces = 2)
render = pyredner.RenderFunction.apply
# Render our target. The first argument is the seed for RNG in the renderer.
img = render(0, *args)
pyredner.imwrite(img.cpu(), 'results/test_teapot_specular/target.exr')
pyredner.imwrite(img.cpu(), 'results/test_teapot_specular/target.png')
target = pyredner.imread('results/test_teapot_specular/target.exr')
if pyredner.get_use_gpu():
target_path = sys.argv[2]
diffuse_texels_path = sys.argv[3]
specular_texels_path = sys.argv[4]
gaussian_func = getGaussianFilter()
target_objects = pyredner.load_obj(target_path, return_objects=True)
print(target_objects[0].vertices.shape)
# Load in the mesh colors info.
mesh_colors_resolution = 1
diffuse_texels = torch.tensor(torch.load(diffuse_texels_path),
device=pyredner.get_device())
specular_texels = torch.tensor(torch.load(specular_texels_path),
device=pyredner.get_device())
target_diffuse = pyredner.Texture(
diffuse_texels, mesh_colors_resolution=mesh_colors_resolution)
target_specular = pyredner.Texture(
specular_texels, mesh_colors_resolution=mesh_colors_resolution)
target_roughness = torch.tensor([0.6]) # For now, roughness is fixed.
target_objects[0].material = pyredner.Material(
diffuse_reflectance=target_diffuse,
specular_reflectance=target_specular,
roughness=target_roughness,
normal_map=None)
# Set up the cameras and lights.
resolution = (128, 128)
num_cameras = 8
radius = 2.0
lightLocs = None
camera0 = pyredner.automatic_camera_placement(target_objects, resolution)
pyredner.imwrite(img.cpu(), 'results/coarse_to_fine_estimation/target.exr')
pyredner.imwrite(img.cpu(), 'results/coarse_to_fine_estimation/target.png')
target = pyredner.imread('results/coarse_to_fine_estimation/target.exr')
if pyredner.get_use_gpu():
target = target.cuda()
# Now let's generate an initial guess by perturbing the reference.
# Let's set all the diffuse color to gray by manipulating material.diffuse_reflectance.
# We also store all the material variables to optimize in a list.
material_vars = []
for mi, m in enumerate(scene.materials):
var = torch.tensor([0.5, 0.5, 0.5],
device = pyredner.get_device(),
requires_grad = True)
material_vars.append(var)
m.diffuse_reflectance = pyredner.Texture(var)
# And let's also slightly perturb the camera up vector and field of view a bit
up = torch.tensor([0.2, 0.8, -0.2], requires_grad = True)
fov = torch.tensor([41.0], requires_grad = True)
cam_vars = [up, fov]
scene.camera = pyredner.Camera(\
position = scene.camera.position,
look_at = scene.camera.look_at,
up = up,
fov = fov,
clip_near = scene.camera.clip_near,
resolution = scene.camera.resolution)
# Serialize the scene and render the initial guess
scene_args = pyredner.RenderFunction.serialize_scene(\
scene = scene,
def clean(self):
recompute_normals(self.mesh)
if self.learning_texture:
# Create a new texture with our texels
self.mesh.material.diffuse_reflectance = pyr.Texture(self.learned_tex)
if pyredner.get_use_gpu():
target = target.cuda(device=pyredner.get_device())
batch_dims = 2
diffuse_reflectance = torch.zeros(\
batch_dims, 128, 128, 3, device = pyredner.get_device(), requires_grad=True)
scenes = [scene, scene]
optimizer = torch.optim.Adam([diffuse_reflectance], lr=1e-2)
for t in range(200):
print('iteration:', t)
optimizer.zero_grad()
scene_args_batch = [batch_dims]
for k in range(0, batch_dims):
scenes[k].materials[0].diffuse_reflectance = pyredner.Texture(
diffuse_reflectance[k, :, :, :])
scene_args = pyredner.RenderFunction.serialize_scene(\
scene = scenes[k],
num_samples = 16,
max_bounces = 0,
channels = [redner.channels.position,
redner.channels.shading_normal,
redner.channels.diffuse_reflectance])
scene_args_batch = scene_args_batch + scene_args
g_buffer = batch_render(t, *scene_args_batch)
img1 = g_buffer[0, :, :, 6:9]
img2 = g_buffer[1, :, :, 6:9]
loss = (img1 - target).pow(2).sum() + (img2 - target).pow(2).sum()
print('loss:', loss.item())
loss.backward()
import numpy as np
import torch
import scipy
import scipy.ndimage
# Optimize for material parameters and camera pose
# Use GPU if available
pyredner.set_use_gpu(torch.cuda.is_available())
# Load the scene from a Mitsuba scene file
scene = pyredner.load_mitsuba('scenes/teapot.xml')
# The last material is the teapot material, set it to the target
scene.materials[-1].diffuse_reflectance = \
pyredner.Texture(torch.tensor([0.3, 0.2, 0.2], device = pyredner.get_device()))
scene.materials[-1].specular_reflectance = \
pyredner.Texture(torch.tensor([0.6, 0.6, 0.6], device = pyredner.get_device()))
scene.materials[-1].roughness = \
pyredner.Texture(torch.tensor([0.05], device = pyredner.get_device()))
args = pyredner.RenderFunction.serialize_scene(\
scene = scene,
num_samples = 1024,
max_bounces = 2)
# Alias of the render function
render = pyredner.RenderFunction.apply
# Render our target. The first argument is the seed for RNG in the renderer.
img = render(0, *args)
pyredner.imwrite(img.cpu(), 'results/test_teapot_reflectance/target.exr')
pyredner.imwrite(img.cpu(), 'results/test_teapot_reflectance/target.png')
def model(optim_scenes, num_samples=(64, 64), max_bounces=1):
renders = pyredner.render_pathtracing(scene=optim_scenes,
num_samples=num_samples,
max_bounces=max_bounces)
return renders
res = 3
# Material Optimization
texels = torch.zeros([
target_objects[0].indices.shape[0] * int(((res + 1) * (res + 2)) / 2) * 3
],
device=pyredner.get_device()) + 0.3
diffuse = pyredner.Texture(texels, mesh_colors_resolution=res)
specular = pyredner.Texture(texels.clone(), mesh_colors_resolution=res)
#texels = torch.zeros([target_objects[0].indices.shape[0] * int(((res + 1) * (res + 2)) / 2) * 3], device = pyredner.get_device()) + 0.8
#roughness = pyredner.Texture(texels, mesh_colors_resolution=res)
target_objects[0].material = pyredner.Material(diffuse_reflectance=diffuse,
specular_reflectance=specular,
roughness=None,
normal_map=None)
target_objects[0].material.diffuse_reflectance.texels.requires_grad = True
target_objects[0].material.specular_reflectance.texels.requires_grad = True
#target_objects[0].material.roughness.texels.requires_grad = True
optimizer = torch.optim.Adam([
target_objects[0].material.diffuse_reflectance.texels,
target_objects[0].material.specular_reflectance.texels
'''
diffuse = torch.tensor([0.0, 0.0, 1.0])
optim_objects[0].material = pyredner.Material(diffuse_reflectance=diffuse, specular_reflectance=specular, roughness=roughness, normal_map=normal_map, two_sided=True)
diffuse = torch.tensor([1.0, 0.0, 0.0])
optim_objects[1].material = pyredner.Material(diffuse_reflectance=diffuse, specular_reflectance=specular, roughness=roughness, normal_map=normal_map, two_sided=True)
'''
# Material Optimization
params = []
for ind, obj in enumerate(optim_objects):
#obj.material.diffuse_reflectance.texels = torch.tensor([0.5, 0.5, 0.5])
texels = torch.zeros([optim_objects[ind].indices.shape[0] * int(((res + 1) * (res + 2)) / 2) * 3], device = pyredner.get_device()) + 1.0
diffuse = pyredner.Texture(texels, mesh_colors_resolution=res)
optim_objects[ind].material = pyredner.Material(diffuse_reflectance=diffuse, specular_reflectance=specular_render, roughness=roughness, normal_map=normal_map, two_sided=True)
#obj.material = pyredner.Material(diffuse_reflectance=diffuse, specular_reflectance=specular, roughness=roughness, normal_map=None)
params.append(optim_objects[ind].material.diffuse_reflectance.texels.clone())
params[ind].requires_grad = True
optimizer = torch.optim.Adam(params, lr=1e-2)
prev_loss = 10000000000
i = 0
while i < 1000:
optimizer.zero_grad()
for ind, obj in enumerate(optim_objects):
with torch.no_grad():
i = 0
for subdiv in range(100):
print("Subdivision: ", subdiv)
# Material Optimization
print("Material Optimization")
optim_objects = pyredner.load_obj(refinemesh, return_objects=True)
#texels = torch.zeros([optim_objects[0].indices.shape[0] * int(((res + 1) * (res + 2)) / 2) * 3], device = pyredner.get_device()) + 0.003
# 0.0207, 0.0003, 0.0000
#average = torch.tensor([0.03, 0.005, 0.003], device=pyredner.get_device())
if (subdiv < 10):
average = torch.tensor([0.0206, 0.0003, 0.001],
device=pyredner.get_device())
diffuse = pyredner.Texture(average)
average_roughness = torch.tensor([0.8], device=pyredner.get_device())
roughness = pyredner.Texture(average_roughness)
texture_range = 5
optim_objects[0].material = pyredner.Material(
diffuse_reflectance=diffuse,
specular_reflectance=specular_uv,
roughness=roughness,
normal_map=None)
optim_objects[
0].material.diffuse_reflectance.texels.requires_grad = True
optimizer = torch.optim.Adam(
[optim_objects[0].material.diffuse_reflectance.texels],
lr=0.01,
betas=(0.5, 0.99))
