def test_kernel_dict_post_load(mode_mono): from mitsuba.python.util import traverse kernel_dict = KernelDict( data={ "type": "directional", "irradiance": { "type": "irregular", "wavelengths": "400, 500", "values": "1, 1", }, }, post_load={ "irradiance.wavelengths": np.array([400.0, 500.0, 600.0]), "irradiance.values": np.array([0.0, 1.0, 2.0]), }, ) # Without post-load update, buffers are initialised as in data obj = kernel_dict.load(post_load_update=False) params = traverse(obj) assert params["irradiance.wavelengths"] == np.array([400.0, 500.0]) assert params["irradiance.values"] == np.array([1.0, 1.0]) # Without post-load update, buffers are initialised as in post_load obj = kernel_dict.load(post_load_update=True) params = traverse(obj) assert params["irradiance.wavelengths"] == np.array([400.0, 500.0, 600.0]) assert params["irradiance.values"] == np.array([0.0, 1.0, 2.0])
def test11_parameters_grad_enabled(variant_gpu_autodiff_rgb): from mitsuba.core.xml import load_string shape = load_string(''' <shape type="obj" version="2.0.0"> <string name="filename" value="resources/data/common/meshes/rectangle.obj"/> </shape> ''') assert shape.parameters_grad_enabled() == False # Get the shape's parameters params = traverse(shape) # Only parameters of the shape should affect the result of that method bsdf_param_key = 'bsdf.reflectance.value' ek.set_requires_gradient(params[bsdf_param_key]) params.set_dirty(bsdf_param_key) params.update() assert shape.parameters_grad_enabled() == False # When setting one of the shape's param to require gradient, method should return True shape_param_key = 'vertex_positions_buf' ek.set_requires_gradient(params[shape_param_key]) params.set_dirty(shape_param_key) params.update() assert shape.parameters_grad_enabled() == True
def create_stairs(num_steps): import numpy as np from mitsuba.render import Mesh size_step = 1.0 / num_steps m = Mesh("stairs", 4 * num_steps, 4 * num_steps - 2) params = traverse(m) v = np.zeros((4 * num_steps, 3)) f = np.zeros((4 * num_steps - 2, 3)) for i in range(num_steps): h = i * size_step s1 = i * size_step s2 = (i + 1) * size_step k = 4 * i v[k + 0] = [0.0, s1, h] v[k + 1] = [1.0, s1, h] v[k + 2] = [0.0, s2, h] v[k + 3] = [1.0, s2, h] f[k] = [k, k + 1, k + 2] f[k + 1] = [k + 1, k + 3, k + 2] if i < num_steps - 1: f[k + 2] = [k + 2, k + 3, k + 5] f[k + 3] = [k + 5, k + 4, k + 2] m.vertex_positions_buffer()[:] = v.reshape(-1) m.faces_buffer()[:] = f.reshape(-1) m.recompute_bbox() return m
def test03_shapes_parameters_grad_enabled(variant_gpu_autodiff_rgb): from mitsuba.core.xml import load_string from mitsuba.python.util import traverse scene = load_string(""" <scene version="2.0.0"> <shape type="obj" id="box"> <string name="filename" value="resources/data/tests/obj/cbox_smallbox.obj"/> </shape> <shape type="sphere"/> </scene> """) # Initial scene should always return False assert scene.shapes_grad_enabled() == False # Get scene parameters params = traverse(scene) # Only parameters of the shape should affect the result of that method bsdf_param_key = 'box.bsdf.reflectance.value' ek.set_requires_gradient(params[bsdf_param_key]) params.set_dirty(bsdf_param_key) params.update() assert scene.shapes_grad_enabled() == False # When setting one of the shape's param to require gradient, method should return True shape_param_key = 'box.vertex_positions_buf' ek.set_requires_gradient(params[shape_param_key]) params.set_dirty(shape_param_key) params.update() assert scene.shapes_grad_enabled() == True
def test_finite_difference(test_name, make_scene, diff_integrator, diff_spp, diff_passes, fd_integrator, fd_spp, fd_passes, fd_eps): print("Running test:", test_name) path = "output/" + test_name + "/" if not os.path.isdir(path): os.makedirs(path) print("Rendering finite differences...") scene_fd0 = make_scene(fd_integrator, fd_spp, 0) scene_fd1 = make_scene(fd_integrator, fd_spp, fd_eps) fsize = scene_fd0.sensors()[0].film().size() for i in range(fd_passes): print("pass:"******"Writing " + path + 'radiance_fd0.exr') Bitmap(values_fd0).write(path + 'radiance_fd0.exr') print("Writing " + path + 'radiance_fd1.exr') Bitmap(values_fd1).write(path + 'radiance_fd1.exr') gradient_fd = (values_fd1 - values_fd0) / fd_eps gradient_fd = gradient_fd[:, :, [0]] scale = np.abs(gradient_fd).max() write_gradient_image(gradient_fd / scale, path + 'gradient_fd', fsize) del scene_fd0, scene_fd1, values_fd0, values_fd1, channels, gradient_fd print("Rendering gradients... ({} spp, {} passes)".format( diff_spp, diff_passes)) scene = make_scene(diff_integrator, diff_spp, None) assert scene is not None params = traverse(scene) params.keep(['SDF.data']) gradient_rp_np = render_gradient(scene, diff_spp, diff_passes, scale, path, params, fd_eps)
def create_regular_tetrahedron(): from mitsuba.render import Mesh m = Mesh("tetrahedron", 4, 4) params = traverse(m) m.vertex_positions_buffer()[:] = [ 0, 0, 0, 0.8, 0.8, 0, 0.8, 0, 0.8, 0, 0.8, 0.8 ] m.faces_buffer()[:] = [0, 1, 2, 1, 2, 3, 0, 2, 2, 1, 3, 3, 3, 1, 0] m.recompute_bbox() return m
def get_diff_param(scene): # Create a differentiable hyperparameter diff_param = Float(0.0) ek.set_requires_gradient(diff_param) # Update vertices so that they depend on diff_param params = traverse(scene) t = Transform4f.translate(Vector3f(1.0) * diff_param) vertex_positions = params['light_shape.vertex_positions'] vertex_positions_t = t.transform_point(vertex_positions) params['light_shape.vertex_positions'] = vertex_positions_t # Update the scene params.update() return diff_param
def get_diff_param(scene): # Create a differentiable hyperparameter diff_param = mitsuba.core.Float(0.0) ek.set_requires_gradient(diff_param) # Update vertices so that they depend on diff_param properties = traverse(scene) t = mitsuba.core.Transform4f.translate(mitsuba.core.Vector3f(1.0, 0.0, 0.0) * diff_param) vertex_positions = properties['object.vertex_positions'] vertex_positions_t = t.transform_point(vertex_positions) properties['object.vertex_positions'] = vertex_positions_t # Update the scene properties.update() return diff_param
def update_medium(scene, medium): """ Update medium parameters in a scene object Args: scene: Scene object including medium medium: List including medium parameters, i.e., - albedo - eta (refractive index) - g (anisotropic index) """ params = traverse(scene) params["Plane_001-mesh_0.interior_medium.albedo.color.value"] = medium[ "albedo"] params["medium_bsdf.eta"] = medium["eta"] params["myphase.g"] = medium["g"] params.update()
def test12_differentiable_surface_interaction_automatic( variant_gpu_autodiff_rgb): from mitsuba.core import xml, Ray3f, Vector3f, UInt32 from mitsuba.render import HitComputeFlags scene = xml.load_string(''' <scene version="2.0.0"> <shape type="obj" id="rect"> <string name="filename" value="resources/data/common/meshes/rectangle.obj"/> </shape> </scene> ''') ray = Ray3f(Vector3f(-0.3, -0.3, -10.0), Vector3f(0.0, 0.0, 1.0), 0, []) pi = scene.ray_intersect_preliminary(ray) # si should not be attached if not necessary si = pi.compute_surface_interaction(ray) assert not ek.requires_gradient(si.t) assert not ek.requires_gradient(si.p) # si should be attached if ray is attached ek.set_requires_gradient(ray.o) si = pi.compute_surface_interaction(ray) assert ek.requires_gradient(si.t) assert ek.requires_gradient(si.p) # si should not be attached if falgs says so ek.set_requires_gradient(ray.o) si = pi.compute_surface_interaction(ray, HitComputeFlags.NonDifferentiable) assert not ek.requires_gradient(si.t) assert not ek.requires_gradient(si.p) # si should be attached if shape parameters are attached params = traverse(scene) shape_param_key = 'rect.vertex_positions_buf' ek.set_requires_gradient(params[shape_param_key]) params.set_dirty(shape_param_key) params.update() ek.set_requires_gradient(ray.o, False) si = pi.compute_surface_interaction(ray) assert ek.requires_gradient(si.t) assert ek.requires_gradient(si.p)
import enoki as ek import mitsuba mitsuba.set_variant('gpu_autodiff_rgb') from mitsuba.core import Thread, Vector3f from mitsuba.core.xml import load_file from mitsuba.python.util import traverse from mitsuba.python.autodiff import render_torch, write_bitmap import torch import time Thread.thread().file_resolver().append('cbox') scene = load_file('cbox/cbox.xml') # Find differentiable scene parameters params = traverse(scene) # Discard all parameters except for one we want to differentiate params.keep(['red.reflectance.value']) # Print the current value and keep a backup copy param_ref = params['red.reflectance.value'].torch() print(param_ref) # Render a reference image (no derivatives used yet) image_ref = render_torch(scene, spp=8) crop_size = scene.sensors()[0].film().crop_size() write_bitmap('out_ref.png', image_ref, crop_size) # Change the left wall into a bright white surface params['red.reflectance.value'] = [.9, .9, .9]
res=cam_res) scene_target = create_scene(None, interpolation, cams_origins, fov, img_res) images_ref = list( render_torch(scene_target, spp=spp_ref, sensor_index=i) for i in range(len(cams_origins))) crop_size = scene_target.sensors()[0].film().crop_size() for i in range(len(cams_origins)): write_bitmap(f'{out_path}{i}_target.png', images_ref[i], crop_size) # Find differentiable scene parameters sdf_file = "data/sdf/init.vol" if restart_epoch == 0 else f"{out_path}sdf_e{restart_epoch}.vol" scene_init = create_scene(sdf_file, interpolation, cams_origins, fov, img_res) params = traverse(scene_init) #print(params) params.keep(['SDF.data', 'SDF.bsdf.reflectance.data']) if restart_epoch > 0 and increase_res: sdf = params['SDF.data'].numpy().reshape([sdf_res] * 3) sdf = double_sdf_res(sdf) params['SDF.data'] = sdf.flatten() params.update() params_torch = params.torch() opt = Adam(params_torch.values(), lr=lr) lr_scheduler = ExponentialLR(opt, gamma=lr_gamma) T_max = 0
def test16_differentiable_surface_interaction_params_backward( variant_gpu_autodiff_rgb): from mitsuba.core import xml, Float, Ray3f, Vector3f, UInt32, Transform4f scene = xml.load_string(''' <scene version="2.0.0"> <shape type="obj" id="rect"> <string name="filename" value="resources/data/common/meshes/rectangle.obj"/> </shape> </scene> ''') params = traverse(scene) vertex_pos_key = 'rect.vertex_positions_buf' vertex_normals_key = 'rect.vertex_normals_buf' vertex_texcoords_key = 'rect.vertex_texcoords_buf' ek.set_requires_gradient(params[vertex_pos_key]) ek.set_requires_gradient(params[vertex_normals_key]) ek.set_requires_gradient(params[vertex_texcoords_key]) params.set_dirty(vertex_pos_key) params.set_dirty(vertex_normals_key) params.set_dirty(vertex_texcoords_key) params.update() # Hit the upper right corner of the rectancle (the 4th vertex) ray = Ray3f(Vector3f(0.99999, 0.99999, -10.0), Vector3f(0.0, 0.0, 1.0), 0, []) pi = scene.ray_intersect_preliminary(ray) # --------------------------------------- # Test vertex posistions # If si.t changes, so the 4th vertex should move along the z-axis si = pi.compute_surface_interaction(ray) ek.backward(si.t) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], atol=1e-5) # If si.p moves along the z-axis, so does the 4th vertex si = pi.compute_surface_interaction(ray) ek.backward(si.p.z) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1], atol=1e-5) # To increase si.dp_du along the x-axis, we need to strech the upper edge of the rectangle si = pi.compute_surface_interaction(ray) ek.backward(si.dp_du.x) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0, 0], atol=1e-5) # To increase si.dp_du along the y-axis, we need to transform the rectangle into a trapezoid si = pi.compute_surface_interaction(ray) ek.backward(si.dp_du.y) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0, 0, -1, 0, 0, 0, 0, 0, 1, 0], atol=1e-5) # To increase si.dp_dv along the x-axis, we need to transform the rectangle into a trapezoid si = pi.compute_surface_interaction(ray) ek.backward(si.dp_dv.x) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [-1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], atol=1e-5) # To increase si.dp_dv along the y-axis, we need to strech the right edge of the rectangle si = pi.compute_surface_interaction(ray) ek.backward(si.dp_dv.y) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0], atol=1e-5) # To increase si.n along the x-axis, we need to rotate the right edge around the y axis si = pi.compute_surface_interaction(ray) ek.backward(si.n.x) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0, 0, 0, 0.5, 0, 0, 0, 0, 0, -0.5], atol=1e-5) # To increase si.n along the y-axis, we need to rotate the top edge around the x axis si = pi.compute_surface_interaction(ray) ek.backward(si.n.y) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0.5, 0, 0, 0, 0, 0, 0, 0, 0, -0.5], atol=1e-5) # To increase si.sh_frame.n along the x-axis, we need to rotate the right edge around the y axis params.set_dirty(vertex_pos_key) params.update() si = pi.compute_surface_interaction(ray) ek.backward(si.sh_frame.n.x) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0, 0, 0, 0.5, 0, 0, 0, 0, 0, -0.5], atol=1e-5) # To increase si.sh_frame.n along the y-axis, we need to rotate the top edge around the x axis params.set_dirty(vertex_pos_key) params.update() si = pi.compute_surface_interaction(ray) ek.backward(si.sh_frame.n.y) assert ek.allclose(ek.gradient(params[vertex_pos_key]), [0, 0, 0.5, 0, 0, 0, 0, 0, 0, 0, 0, -0.5], atol=1e-5) # --------------------------------------- # Test vertex texcoords # To increase si.uv along the x-axis, we need to move the uv of the 4th vertex along the x-axis si = pi.compute_surface_interaction(ray) ek.backward(si.uv.x) assert ek.allclose(ek.gradient(params[vertex_texcoords_key]), [0, 0, 0, 0, 0, 0, 1, 0], atol=1e-5) # To increase si.uv along the y-axis, we need to move the uv of the 4th vertex along the y-axis si = pi.compute_surface_interaction(ray) ek.backward(si.uv.y) assert ek.allclose(ek.gradient(params[vertex_texcoords_key]), [0, 0, 0, 0, 0, 0, 0, 1], atol=1e-5) # To increase si.dp_du along the x-axis, we need to shrink the uv along the top edge of the rectangle si = pi.compute_surface_interaction(ray) ek.backward(si.dp_du.x) assert ek.allclose(ek.gradient(params[vertex_texcoords_key]), [0, 0, 2, 0, 0, 0, -2, 0], atol=1e-5) # To increase si.dp_du along the y-axis, we need to shrink the uv along the right edge of the rectangle si = pi.compute_surface_interaction(ray) ek.backward(si.dp_dv.y) assert ek.allclose(ek.gradient(params[vertex_texcoords_key]), [0, 2, 0, 0, 0, 0, 0, -2], atol=1e-5)
def test15_differentiable_surface_interaction_params_forward( variant_gpu_autodiff_rgb): from mitsuba.core import xml, Float, Ray3f, Vector3f, UInt32, Transform4f # Convert flat array into a vector of arrays (will be included in next enoki release) def ravel(buf, dim=3): idx = dim * UInt32.arange(ek.slices(buf) // dim) if dim == 2: return Vector2f(ek.gather(buf, idx), ek.gather(buf, idx + 1)) elif dim == 3: return Vector3f(ek.gather(buf, idx), ek.gather(buf, idx + 1), ek.gather(buf, idx + 2)) # Return contiguous flattened array (will be included in next enoki release) def unravel(source, target, dim=3): idx = UInt32.arange(ek.slices(source)) for i in range(dim): ek.scatter(target, source[i], dim * idx + i) scene = xml.load_string(''' <scene version="2.0.0"> <shape type="obj" id="rect"> <string name="filename" value="resources/data/common/meshes/rectangle.obj"/> </shape> </scene> ''') params = traverse(scene) shape_param_key = 'rect.vertex_positions_buf' positions_buf = params[shape_param_key] positions_initial = ravel(positions_buf) # Create differential parameter to be optimized diff_vector = Vector3f(0.0) ek.set_requires_gradient(diff_vector) # Apply the transformation to mesh vertex position and update scene def apply_transformation(trasfo): trasfo = trasfo(diff_vector) new_positions = trasfo.transform_point(positions_initial) unravel(new_positions, params[shape_param_key]) params.set_dirty(shape_param_key) params.update() # --------------------------------------- # Test translation ray = Ray3f(Vector3f(-0.2, -0.3, -10.0), Vector3f(0.0, 0.0, 1.0), 0, []) pi = scene.ray_intersect_preliminary(ray) # # If the vertices are shifted along z-axis, so does si.t apply_transformation(lambda v: Transform4f.translate(v)) si = pi.compute_surface_interaction(ray) ek.forward(diff_vector.z) assert ek.allclose(ek.gradient(si.t), 1) # If the vertices are shifted along z-axis, so does si.p apply_transformation(lambda v: Transform4f.translate(v)) si = pi.compute_surface_interaction(ray) ek.forward(diff_vector.z) assert ek.allclose(ek.gradient(si.p), [0.0, 0.0, 1.0]) # If the vertices are shifted along x-axis, so does si.uv (times 0.5) apply_transformation(lambda v: Transform4f.translate(v)) si = pi.compute_surface_interaction(ray) ek.forward(diff_vector.x) assert ek.allclose(ek.gradient(si.uv), [-0.5, 0.0]) # If the vertices are shifted along y-axis, so does si.uv (times 0.5) apply_transformation(lambda v: Transform4f.translate(v)) si = pi.compute_surface_interaction(ray) ek.forward(diff_vector.y) assert ek.allclose(ek.gradient(si.uv), [0.0, -0.5]) # --------------------------------------- # Test rotation ray = Ray3f(Vector3f(-0.99999, -0.99999, -10.0), Vector3f(0.0, 0.0, 1.0), 0, []) pi = scene.ray_intersect_preliminary(ray) # If the vertices are rotated around the center, so does si.uv (times 0.5) apply_transformation(lambda v: Transform4f.rotate([0, 0, 1], v.x)) si = pi.compute_surface_interaction(ray) ek.forward(diff_vector.x) du = 0.5 * ek.sin(2 * ek.pi / 360.0) assert ek.allclose(ek.gradient(si.uv), [-du, du], atol=1e-6)
def load( self, strip: bool = True, post_load_update: bool = True ) -> mitsuba.core.Object: """ Call :func:`~mitsuba.core.xml.load_dict` on self. In addition, a post-load update can be applied. If the encapsulated dictionary misses a ``"type"`` key, it will be promoted to a scene dictionary through the addition of ``{"type": "scene"}``. For instance, it means that .. code:: python { "shape1": {"type": "sphere"}, "shape2": {"type": "sphere"}, } will be interpreted as .. code:: python { "type": "scene", "shape1": {"type": "sphere"}, "shape2": {"type": "sphere"}, } .. note:: Requires a valid selected operational mode. Parameters ---------- strip : bool If ``True``, if ``data`` has no ``'type'`` entry and if ``data`` consists of one nested dictionary, it will be loaded directly. For instance, it means that .. code:: python {"phase": {"type": "rayleigh"}} will be stripped to .. code:: python {"type": "rayleigh"} post_load_update : bool If ``True``, use :func:`~mitsuba.python.util.traverse` and update loaded scene parameters according to data stored in ``post_load``. Returns ------- :class:`mitsuba.core.Object` Loaded Mitsuba object. """ from mitsuba.core.xml import load_dict from mitsuba.python.util import traverse d = self.data d_extra = {} if "type" not in self: if len(self) == 1 and strip: # Extract plugin dictionary d = onedict_value(d) else: # Promote to scene dictionary d_extra = {"type": "scene"} obj = load_dict({**d, **d_extra}) if self.post_load and post_load_update: params = traverse(obj) params.keep(list(self.post_load.keys())) for k, v in self.post_load.items(): params[k] = v params.update() return obj
def render(scene_dict, info, in_q, out_q, dones_sh, texture_sh, render_sh, uv_sh, loop_forever=True): a = time() scene = load_dict(scene_dict) print(f"Scene loading time: {time() - a:.2f}s") bsdf_params = traverse(scene.shapes()[0].bsdf()) integrator = scene.integrator() # Numpy versions of the shared tensors texture_sh_np = texture_sh.numpy() render_sh_np = render_sh.numpy() uv_sh_np = uv_sh.numpy() while True: camera_pos = sphere_sample(info['scale_range']) camera = load_dict({ "type": "perspective", "to_world": ScalarTransform4f.look_at(origin=camera_pos, target=[0, 0, 0], up=[0, 0, 1]), "myfilm": { "type": "hdrfilm", "width": info['image_size'], "height": info['image_size'] }, "mysampler": { "type": "independent", "sample_count": info['samples'] } }) ind = in_q.get() tex = texture_sh_np[ind] bsdf_params['diffuse_reflectance.data'] = tex integrator.render(scene, camera) film = camera.film() im = film.bitmap(raw=False).split() im_arr = np.array(im[0][1].convert(Bitmap.PixelFormat.RGBA, Struct.Type.Float32, srgb_gamma=False)) u = np.array(im[1][1].convert(Bitmap.PixelFormat.Y, Struct.Type.Float32, srgb_gamma=False)) v = np.array(im[2][1].convert(Bitmap.PixelFormat.Y, Struct.Type.Float32, srgb_gamma=False)) uv_arr = np.concatenate( [u, v, np.zeros_like(u), (u + v > 0).astype(np.float32)], axis=2) render_sh_np[ind, ...] = im_arr uv_sh_np[ind, ...] = uv_arr dones_sh[ind] = True if not loop_forever: print("Done") return