def test_kernel_dict_post_load(mode_mono):
from mitsuba.python.util import traverse
kernel_dict = KernelDict(
data={
"type": "directional",
"irradiance": {
"type": "irregular",
"wavelengths": "400, 500",
"values": "1, 1",
},
},
post_load={
"irradiance.wavelengths": np.array([400.0, 500.0, 600.0]),
"irradiance.values": np.array([0.0, 1.0, 2.0]),
},
)
# Without post-load update, buffers are initialised as in data
obj = kernel_dict.load(post_load_update=False)
params = traverse(obj)
assert params["irradiance.wavelengths"] == np.array([400.0, 500.0])
assert params["irradiance.values"] == np.array([1.0, 1.0])
# Without post-load update, buffers are initialised as in post_load
obj = kernel_dict.load(post_load_update=True)
params = traverse(obj)
assert params["irradiance.wavelengths"] == np.array([400.0, 500.0, 600.0])
assert params["irradiance.values"] == np.array([0.0, 1.0, 2.0])
def test11_parameters_grad_enabled(variant_gpu_autodiff_rgb):
from mitsuba.core.xml import load_string
shape = load_string('''
<shape type="obj" version="2.0.0">
<string name="filename" value="resources/data/common/meshes/rectangle.obj"/>
</shape>
''')
assert shape.parameters_grad_enabled() == False
# Get the shape's parameters
params = traverse(shape)
# Only parameters of the shape should affect the result of that method
bsdf_param_key = 'bsdf.reflectance.value'
ek.set_requires_gradient(params[bsdf_param_key])
params.set_dirty(bsdf_param_key)
params.update()
assert shape.parameters_grad_enabled() == False
# When setting one of the shape's param to require gradient, method should return True
shape_param_key = 'vertex_positions_buf'
ek.set_requires_gradient(params[shape_param_key])
params.set_dirty(shape_param_key)
params.update()
assert shape.parameters_grad_enabled() == True
def create_stairs(num_steps):
import numpy as np
from mitsuba.render import Mesh
size_step = 1.0 / num_steps
m = Mesh("stairs", 4 * num_steps, 4 * num_steps - 2)
params = traverse(m)
v = np.zeros((4 * num_steps, 3))
f = np.zeros((4 * num_steps - 2, 3))
for i in range(num_steps):
h = i * size_step
s1 = i * size_step
s2 = (i + 1) * size_step
k = 4 * i
v[k + 0] = [0.0, s1, h]
v[k + 1] = [1.0, s1, h]
v[k + 2] = [0.0, s2, h]
v[k + 3] = [1.0, s2, h]
f[k] = [k, k + 1, k + 2]
f[k + 1] = [k + 1, k + 3, k + 2]
if i < num_steps - 1:
f[k + 2] = [k + 2, k + 3, k + 5]
f[k + 3] = [k + 5, k + 4, k + 2]
m.vertex_positions_buffer()[:] = v.reshape(-1)
m.faces_buffer()[:] = f.reshape(-1)
m.recompute_bbox()
return m
def test03_shapes_parameters_grad_enabled(variant_gpu_autodiff_rgb):
from mitsuba.core.xml import load_string
from mitsuba.python.util import traverse
scene = load_string("""
<scene version="2.0.0">
<shape type="obj" id="box">
<string name="filename" value="resources/data/tests/obj/cbox_smallbox.obj"/>
</shape>
<shape type="sphere"/>
</scene>
""")
# Initial scene should always return False
assert scene.shapes_grad_enabled() == False
# Get scene parameters
params = traverse(scene)
# Only parameters of the shape should affect the result of that method
bsdf_param_key = 'box.bsdf.reflectance.value'
ek.set_requires_gradient(params[bsdf_param_key])
params.set_dirty(bsdf_param_key)
params.update()
assert scene.shapes_grad_enabled() == False
# When setting one of the shape's param to require gradient, method should return True
shape_param_key = 'box.vertex_positions_buf'
ek.set_requires_gradient(params[shape_param_key])
params.set_dirty(shape_param_key)
params.update()
assert scene.shapes_grad_enabled() == True
def test_finite_difference(test_name, make_scene, diff_integrator, diff_spp,
diff_passes, fd_integrator, fd_spp, fd_passes,
fd_eps):
print("Running test:", test_name)
path = "output/" + test_name + "/"
if not os.path.isdir(path):
os.makedirs(path)
print("Rendering finite differences...")
scene_fd0 = make_scene(fd_integrator, fd_spp, 0)
scene_fd1 = make_scene(fd_integrator, fd_spp, fd_eps)
fsize = scene_fd0.sensors()[0].film().size()
for i in range(fd_passes):
print("pass:"******"Writing " + path + 'radiance_fd0.exr')
Bitmap(values_fd0).write(path + 'radiance_fd0.exr')
print("Writing " + path + 'radiance_fd1.exr')
Bitmap(values_fd1).write(path + 'radiance_fd1.exr')
gradient_fd = (values_fd1 - values_fd0) / fd_eps
gradient_fd = gradient_fd[:, :, [0]]
scale = np.abs(gradient_fd).max()
write_gradient_image(gradient_fd / scale, path + 'gradient_fd', fsize)
del scene_fd0, scene_fd1, values_fd0, values_fd1, channels, gradient_fd
print("Rendering gradients... ({} spp, {} passes)".format(
diff_spp, diff_passes))
scene = make_scene(diff_integrator, diff_spp, None)
assert scene is not None
params = traverse(scene)
params.keep(['SDF.data'])
gradient_rp_np = render_gradient(scene, diff_spp, diff_passes, scale, path,
params, fd_eps)
def create_regular_tetrahedron():
from mitsuba.render import Mesh
m = Mesh("tetrahedron", 4, 4)
params = traverse(m)
m.vertex_positions_buffer()[:] = [
0, 0, 0, 0.8, 0.8, 0, 0.8, 0, 0.8, 0, 0.8, 0.8
]
m.faces_buffer()[:] = [0, 1, 2, 1, 2, 3, 0, 2, 2, 1, 3, 3, 3, 1, 0]
m.recompute_bbox()
return m
def get_diff_param(scene):
# Create a differentiable hyperparameter
diff_param = Float(0.0)
ek.set_requires_gradient(diff_param)
# Update vertices so that they depend on diff_param
params = traverse(scene)
t = Transform4f.translate(Vector3f(1.0) * diff_param)
vertex_positions = params['light_shape.vertex_positions']
vertex_positions_t = t.transform_point(vertex_positions)
params['light_shape.vertex_positions'] = vertex_positions_t
# Update the scene
params.update()
return diff_param
def get_diff_param(scene):
# Create a differentiable hyperparameter
diff_param = mitsuba.core.Float(0.0)
ek.set_requires_gradient(diff_param)
# Update vertices so that they depend on diff_param
properties = traverse(scene)
t = mitsuba.core.Transform4f.translate(mitsuba.core.Vector3f(1.0, 0.0, 0.0) * diff_param)
vertex_positions = properties['object.vertex_positions']
vertex_positions_t = t.transform_point(vertex_positions)
properties['object.vertex_positions'] = vertex_positions_t
# Update the scene
properties.update()
return diff_param
def update_medium(scene, medium):
"""
Update medium parameters in a scene object
Args:
scene: Scene object including medium
medium: List including medium parameters, i.e.,
- albedo
- eta (refractive index)
- g (anisotropic index)
"""
params = traverse(scene)
params["Plane_001-mesh_0.interior_medium.albedo.color.value"] = medium[
"albedo"]
params["medium_bsdf.eta"] = medium["eta"]
params["myphase.g"] = medium["g"]
params.update()
def test12_differentiable_surface_interaction_automatic(
variant_gpu_autodiff_rgb):
from mitsuba.core import xml, Ray3f, Vector3f, UInt32
from mitsuba.render import HitComputeFlags
scene = xml.load_string('''
<scene version="2.0.0">
<shape type="obj" id="rect">
<string name="filename" value="resources/data/common/meshes/rectangle.obj"/>
</shape>
</scene>
''')
ray = Ray3f(Vector3f(-0.3, -0.3, -10.0), Vector3f(0.0, 0.0, 1.0), 0, [])
pi = scene.ray_intersect_preliminary(ray)
# si should not be attached if not necessary
si = pi.compute_surface_interaction(ray)
assert not ek.requires_gradient(si.t)
assert not ek.requires_gradient(si.p)
# si should be attached if ray is attached
ek.set_requires_gradient(ray.o)
si = pi.compute_surface_interaction(ray)
assert ek.requires_gradient(si.t)
assert ek.requires_gradient(si.p)
# si should not be attached if falgs says so
ek.set_requires_gradient(ray.o)
si = pi.compute_surface_interaction(ray, HitComputeFlags.NonDifferentiable)
assert not ek.requires_gradient(si.t)
assert not ek.requires_gradient(si.p)
# si should be attached if shape parameters are attached
params = traverse(scene)
shape_param_key = 'rect.vertex_positions_buf'
ek.set_requires_gradient(params[shape_param_key])
params.set_dirty(shape_param_key)
params.update()
ek.set_requires_gradient(ray.o, False)
si = pi.compute_surface_interaction(ray)
assert ek.requires_gradient(si.t)
assert ek.requires_gradient(si.p)
import enoki as ek
import mitsuba
mitsuba.set_variant('gpu_autodiff_rgb')
from mitsuba.core import Thread, Vector3f
from mitsuba.core.xml import load_file
from mitsuba.python.util import traverse
from mitsuba.python.autodiff import render_torch, write_bitmap
import torch
import time
Thread.thread().file_resolver().append('cbox')
scene = load_file('cbox/cbox.xml')
# Find differentiable scene parameters
params = traverse(scene)
# Discard all parameters except for one we want to differentiate
params.keep(['red.reflectance.value'])
# Print the current value and keep a backup copy
param_ref = params['red.reflectance.value'].torch()
print(param_ref)
# Render a reference image (no derivatives used yet)
image_ref = render_torch(scene, spp=8)
crop_size = scene.sensors()[0].film().crop_size()
write_bitmap('out_ref.png', image_ref, crop_size)
# Change the left wall into a bright white surface
params['red.reflectance.value'] = [.9, .9, .9]
res=cam_res)
scene_target = create_scene(None, interpolation, cams_origins, fov, img_res)
images_ref = list(
render_torch(scene_target, spp=spp_ref, sensor_index=i)
for i in range(len(cams_origins)))
crop_size = scene_target.sensors()[0].film().crop_size()
for i in range(len(cams_origins)):
write_bitmap(f'{out_path}{i}_target.png', images_ref[i], crop_size)
# Find differentiable scene parameters
sdf_file = "data/sdf/init.vol" if restart_epoch == 0 else f"{out_path}sdf_e{restart_epoch}.vol"
scene_init = create_scene(sdf_file, interpolation, cams_origins, fov, img_res)
params = traverse(scene_init)
#print(params)
params.keep(['SDF.data', 'SDF.bsdf.reflectance.data'])
if restart_epoch > 0 and increase_res:
sdf = params['SDF.data'].numpy().reshape([sdf_res] * 3)
sdf = double_sdf_res(sdf)
params['SDF.data'] = sdf.flatten()
params.update()
params_torch = params.torch()
opt = Adam(params_torch.values(), lr=lr)
lr_scheduler = ExponentialLR(opt, gamma=lr_gamma)
T_max = 0
def test16_differentiable_surface_interaction_params_backward(
variant_gpu_autodiff_rgb):
from mitsuba.core import xml, Float, Ray3f, Vector3f, UInt32, Transform4f
scene = xml.load_string('''
<scene version="2.0.0">
<shape type="obj" id="rect">
<string name="filename" value="resources/data/common/meshes/rectangle.obj"/>
</shape>
</scene>
''')
params = traverse(scene)
vertex_pos_key = 'rect.vertex_positions_buf'
vertex_normals_key = 'rect.vertex_normals_buf'
vertex_texcoords_key = 'rect.vertex_texcoords_buf'
ek.set_requires_gradient(params[vertex_pos_key])
ek.set_requires_gradient(params[vertex_normals_key])
ek.set_requires_gradient(params[vertex_texcoords_key])
params.set_dirty(vertex_pos_key)
params.set_dirty(vertex_normals_key)
params.set_dirty(vertex_texcoords_key)
params.update()
# Hit the upper right corner of the rectancle (the 4th vertex)
ray = Ray3f(Vector3f(0.99999, 0.99999, -10.0), Vector3f(0.0, 0.0, 1.0), 0,
[])
pi = scene.ray_intersect_preliminary(ray)
# ---------------------------------------
# Test vertex posistions
# If si.t changes, so the 4th vertex should move along the z-axis
si = pi.compute_surface_interaction(ray)
ek.backward(si.t)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
atol=1e-5)
# If si.p moves along the z-axis, so does the 4th vertex
si = pi.compute_surface_interaction(ray)
ek.backward(si.p.z)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
atol=1e-5)
# To increase si.dp_du along the x-axis, we need to strech the upper edge of the rectangle
si = pi.compute_surface_interaction(ray)
ek.backward(si.dp_du.x)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 0],
atol=1e-5)
# To increase si.dp_du along the y-axis, we need to transform the rectangle into a trapezoid
si = pi.compute_surface_interaction(ray)
ek.backward(si.dp_du.y)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0],
atol=1e-5)
# To increase si.dp_dv along the x-axis, we need to transform the rectangle into a trapezoid
si = pi.compute_surface_interaction(ray)
ek.backward(si.dp_dv.x)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[-1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0],
atol=1e-5)
# To increase si.dp_dv along the y-axis, we need to strech the right edge of the rectangle
si = pi.compute_surface_interaction(ray)
ek.backward(si.dp_dv.y)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0],
atol=1e-5)
# To increase si.n along the x-axis, we need to rotate the right edge around the y axis
si = pi.compute_surface_interaction(ray)
ek.backward(si.n.x)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0, 0, 0, 0.5, 0, 0, 0, 0, 0, -0.5],
atol=1e-5)
# To increase si.n along the y-axis, we need to rotate the top edge around the x axis
si = pi.compute_surface_interaction(ray)
ek.backward(si.n.y)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0.5, 0, 0, 0, 0, 0, 0, 0, 0, -0.5],
atol=1e-5)
# To increase si.sh_frame.n along the x-axis, we need to rotate the right edge around the y axis
params.set_dirty(vertex_pos_key)
params.update()
si = pi.compute_surface_interaction(ray)
ek.backward(si.sh_frame.n.x)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0, 0, 0, 0.5, 0, 0, 0, 0, 0, -0.5],
atol=1e-5)
# To increase si.sh_frame.n along the y-axis, we need to rotate the top edge around the x axis
params.set_dirty(vertex_pos_key)
params.update()
si = pi.compute_surface_interaction(ray)
ek.backward(si.sh_frame.n.y)
assert ek.allclose(ek.gradient(params[vertex_pos_key]),
[0, 0, 0.5, 0, 0, 0, 0, 0, 0, 0, 0, -0.5],
atol=1e-5)
# ---------------------------------------
# Test vertex texcoords
# To increase si.uv along the x-axis, we need to move the uv of the 4th vertex along the x-axis
si = pi.compute_surface_interaction(ray)
ek.backward(si.uv.x)
assert ek.allclose(ek.gradient(params[vertex_texcoords_key]),
[0, 0, 0, 0, 0, 0, 1, 0],
atol=1e-5)
# To increase si.uv along the y-axis, we need to move the uv of the 4th vertex along the y-axis
si = pi.compute_surface_interaction(ray)
ek.backward(si.uv.y)
assert ek.allclose(ek.gradient(params[vertex_texcoords_key]),
[0, 0, 0, 0, 0, 0, 0, 1],
atol=1e-5)
# To increase si.dp_du along the x-axis, we need to shrink the uv along the top edge of the rectangle
si = pi.compute_surface_interaction(ray)
ek.backward(si.dp_du.x)
assert ek.allclose(ek.gradient(params[vertex_texcoords_key]),
[0, 0, 2, 0, 0, 0, -2, 0],
atol=1e-5)
# To increase si.dp_du along the y-axis, we need to shrink the uv along the right edge of the rectangle
si = pi.compute_surface_interaction(ray)
ek.backward(si.dp_dv.y)
assert ek.allclose(ek.gradient(params[vertex_texcoords_key]),
[0, 2, 0, 0, 0, 0, 0, -2],
atol=1e-5)
def test15_differentiable_surface_interaction_params_forward(
variant_gpu_autodiff_rgb):
from mitsuba.core import xml, Float, Ray3f, Vector3f, UInt32, Transform4f
# Convert flat array into a vector of arrays (will be included in next enoki release)
def ravel(buf, dim=3):
idx = dim * UInt32.arange(ek.slices(buf) // dim)
if dim == 2:
return Vector2f(ek.gather(buf, idx), ek.gather(buf, idx + 1))
elif dim == 3:
return Vector3f(ek.gather(buf, idx), ek.gather(buf, idx + 1),
ek.gather(buf, idx + 2))
# Return contiguous flattened array (will be included in next enoki release)
def unravel(source, target, dim=3):
idx = UInt32.arange(ek.slices(source))
for i in range(dim):
ek.scatter(target, source[i], dim * idx + i)
scene = xml.load_string('''
<scene version="2.0.0">
<shape type="obj" id="rect">
<string name="filename" value="resources/data/common/meshes/rectangle.obj"/>
</shape>
</scene>
''')
params = traverse(scene)
shape_param_key = 'rect.vertex_positions_buf'
positions_buf = params[shape_param_key]
positions_initial = ravel(positions_buf)
# Create differential parameter to be optimized
diff_vector = Vector3f(0.0)
ek.set_requires_gradient(diff_vector)
# Apply the transformation to mesh vertex position and update scene
def apply_transformation(trasfo):
trasfo = trasfo(diff_vector)
new_positions = trasfo.transform_point(positions_initial)
unravel(new_positions, params[shape_param_key])
params.set_dirty(shape_param_key)
params.update()
# ---------------------------------------
# Test translation
ray = Ray3f(Vector3f(-0.2, -0.3, -10.0), Vector3f(0.0, 0.0, 1.0), 0, [])
pi = scene.ray_intersect_preliminary(ray)
# # If the vertices are shifted along z-axis, so does si.t
apply_transformation(lambda v: Transform4f.translate(v))
si = pi.compute_surface_interaction(ray)
ek.forward(diff_vector.z)
assert ek.allclose(ek.gradient(si.t), 1)
# If the vertices are shifted along z-axis, so does si.p
apply_transformation(lambda v: Transform4f.translate(v))
si = pi.compute_surface_interaction(ray)
ek.forward(diff_vector.z)
assert ek.allclose(ek.gradient(si.p), [0.0, 0.0, 1.0])
# If the vertices are shifted along x-axis, so does si.uv (times 0.5)
apply_transformation(lambda v: Transform4f.translate(v))
si = pi.compute_surface_interaction(ray)
ek.forward(diff_vector.x)
assert ek.allclose(ek.gradient(si.uv), [-0.5, 0.0])
# If the vertices are shifted along y-axis, so does si.uv (times 0.5)
apply_transformation(lambda v: Transform4f.translate(v))
si = pi.compute_surface_interaction(ray)
ek.forward(diff_vector.y)
assert ek.allclose(ek.gradient(si.uv), [0.0, -0.5])
# ---------------------------------------
# Test rotation
ray = Ray3f(Vector3f(-0.99999, -0.99999, -10.0), Vector3f(0.0, 0.0, 1.0),
0, [])
pi = scene.ray_intersect_preliminary(ray)
# If the vertices are rotated around the center, so does si.uv (times 0.5)
apply_transformation(lambda v: Transform4f.rotate([0, 0, 1], v.x))
si = pi.compute_surface_interaction(ray)
ek.forward(diff_vector.x)
du = 0.5 * ek.sin(2 * ek.pi / 360.0)
assert ek.allclose(ek.gradient(si.uv), [-du, du], atol=1e-6)
def load(
self, strip: bool = True, post_load_update: bool = True
) -> mitsuba.core.Object:
"""
Call :func:`~mitsuba.core.xml.load_dict` on self. In addition, a
post-load update can be applied.
If the encapsulated dictionary misses a ``"type"`` key, it will be
promoted to a scene dictionary through the addition of
``{"type": "scene"}``. For instance, it means that
.. code:: python
{
"shape1": {"type": "sphere"},
"shape2": {"type": "sphere"},
}
will be interpreted as
.. code:: python
{
"type": "scene",
"shape1": {"type": "sphere"},
"shape2": {"type": "sphere"},
}
.. note::
Requires a valid selected operational mode.
Parameters
----------
strip : bool
If ``True``, if ``data`` has no ``'type'`` entry and if ``data``
consists of one nested dictionary, it will be loaded directly.
For instance, it means that
.. code:: python
{"phase": {"type": "rayleigh"}}
will be stripped to
.. code:: python
{"type": "rayleigh"}
post_load_update : bool
If ``True``, use :func:`~mitsuba.python.util.traverse` and update
loaded scene parameters according to data stored in ``post_load``.
Returns
-------
:class:`mitsuba.core.Object`
Loaded Mitsuba object.
"""
from mitsuba.core.xml import load_dict
from mitsuba.python.util import traverse
d = self.data
d_extra = {}
if "type" not in self:
if len(self) == 1 and strip:
# Extract plugin dictionary
d = onedict_value(d)
else:
# Promote to scene dictionary
d_extra = {"type": "scene"}
obj = load_dict({**d, **d_extra})
if self.post_load and post_load_update:
params = traverse(obj)
params.keep(list(self.post_load.keys()))
for k, v in self.post_load.items():
params[k] = v
params.update()
return obj
def render(scene_dict,
info,
in_q,
out_q,
dones_sh,
texture_sh,
render_sh,
uv_sh,
loop_forever=True):
a = time()
scene = load_dict(scene_dict)
print(f"Scene loading time: {time() - a:.2f}s")
bsdf_params = traverse(scene.shapes()[0].bsdf())
integrator = scene.integrator()
# Numpy versions of the shared tensors
texture_sh_np = texture_sh.numpy()
render_sh_np = render_sh.numpy()
uv_sh_np = uv_sh.numpy()
while True:
camera_pos = sphere_sample(info['scale_range'])
camera = load_dict({
"type":
"perspective",
"to_world":
ScalarTransform4f.look_at(origin=camera_pos,
target=[0, 0, 0],
up=[0, 0, 1]),
"myfilm": {
"type": "hdrfilm",
"width": info['image_size'],
"height": info['image_size']
},
"mysampler": {
"type": "independent",
"sample_count": info['samples']
}
})
ind = in_q.get()
tex = texture_sh_np[ind]
bsdf_params['diffuse_reflectance.data'] = tex
integrator.render(scene, camera)
film = camera.film()
im = film.bitmap(raw=False).split()
im_arr = np.array(im[0][1].convert(Bitmap.PixelFormat.RGBA,
Struct.Type.Float32,
srgb_gamma=False))
u = np.array(im[1][1].convert(Bitmap.PixelFormat.Y,
Struct.Type.Float32,
srgb_gamma=False))
v = np.array(im[2][1].convert(Bitmap.PixelFormat.Y,
Struct.Type.Float32,
srgb_gamma=False))
uv_arr = np.concatenate(
[u, v, np.zeros_like(u), (u + v > 0).astype(np.float32)], axis=2)
render_sh_np[ind, ...] = im_arr
uv_sh_np[ind, ...] = uv_arr
dones_sh[ind] = True
if not loop_forever:
print("Done")
return
