def automatic_camera_placement(shapes: List,
resolution: Tuple[int, int]):
"""
Given a list of shapes, generates camera parameters automatically
using the bounding boxes of the shapes. Place the camera at
some distances from the shapes, so that it can see all of them.
Inspired by https://github.com/mitsuba-renderer/mitsuba/blob/master/src/librender/scene.cpp#L286
"""
assert(tf.executing_eagerly())
aabb_min = tf.constant((float('inf'), float('inf'), float('inf')))
aabb_max = -tf.constant((float('inf'), float('inf'), float('inf')))
for shape in shapes:
v = shape.vertices
v_min = tf.reduce_min(v, 0)
v_max = tf.reduce_max(v, 0)
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
v_min = tf.identity(v_min)
v_max = tf.identity(v_max)
aabb_min = tf.minimum(aabb_min, v_min)
aabb_max = tf.maximum(aabb_max, v_max)
assert(tf.reduce_all(tf.math.is_finite(aabb_min)) and tf.reduce_all(tf.math.is_finite(aabb_max)))
center = (aabb_max + aabb_min) * 0.5
extents = aabb_max - aabb_min
max_extents_xy = tf.maximum(extents[0], extents[1])
distance = max_extents_xy / (2 * math.tan(45 * 0.5 * math.pi / 180.0))
max_extents_xyz = tf.maximum(extents[2], max_extents_xy)
return Camera(position = tf.stack((center[0], center[1], aabb_min[2] - distance)),
look_at = center,
up = tf.constant((0.0, 1.0, 0.0)),
fov = tf.constant([45.0]),
clip_near = 0.001 * float(distance),
resolution = resolution)
def intrinsic_mat(self, value):
if value is not None:
self._intrinsic_mat = value
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self.intrinsic_mat_inv = tf.linalg.inv(self._intrinsic_mat)
else:
assert(self.fov is not None)
self.fov = self._fov
def cam_to_world(self, value):
if value is not None:
self._cam_to_world = value
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self.world_to_cam = tf.linalg.inv(self.cam_to_world)
else:
self._cam_to_world = None
self.world_to_cam = None
def fov(self, value):
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self._fov = tf.identity(value).cpu()
fov_factor = 1.0 / tf.tan(transform.radians(0.5 * self._fov))
o = tf.convert_to_tensor(np.ones([1], dtype=np.float32),
dtype=tf.float32)
diag = tf.concat([fov_factor, fov_factor, o], 0)
self._cam_to_ndc = tf.linalg.tensor_diag(diag)
self.ndc_to_cam = tf.linalg.inv(self._cam_to_ndc)
def fov(self, value):
if value is not None:
self._fov = value
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
fov_factor = 1.0 / tf.tan(transform.radians(0.5 * self._fov))
o = tf.ones([1], dtype=tf.float32)
diag = tf.concat([fov_factor, fov_factor, o], 0)
self._intrinsic_mat = tf.linalg.tensor_diag(diag)
self.intrinsic_mat_inv = tf.linalg.inv(self._intrinsic_mat)
else:
self._fov = None
def __init__(self,
position: Optional[tf.Tensor] = None,
look_at: Optional[tf.Tensor] = None,
up: Optional[tf.Tensor] = None,
fov: Optional[tf.Tensor] = None,
clip_near: float = 1e-4,
resolution: Tuple[int] = (256, 256),
cam_to_world: Optional[tf.Tensor] = None,
intrinsic_mat: Optional[tf.Tensor] = None,
camera_type = pyredner.camera_type.perspective,
fisheye: bool = False):
assert(tf.executing_eagerly())
if position is not None:
assert(position.dtype == tf.float32)
assert(len(position.shape) == 1 and position.shape[0] == 3)
if look_at is not None:
assert(look_at.dtype == tf.float32)
assert(len(look_at.shape) == 1 and look_at.shape[0] == 3)
if up is not None:
assert(up.dtype == tf.float32)
assert(len(up.shape) == 1 and up.shape[0] == 3)
if fov is not None:
assert(fov.dtype == tf.float32)
assert(len(fov.shape) == 1 and fov.shape[0] == 1)
assert(isinstance(clip_near, float))
if position is None and look_at is None and up is None:
assert(cam_to_world is not None)
self.position = position
self.look_at = look_at
self.up = up
self.fov = fov
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if cam_to_world is not None:
self.cam_to_world = cam_to_world
else:
self.cam_to_world = None
if intrinsic_mat is None:
if camera_type == redner.CameraType.perspective:
fov_factor = 1.0 / tf.tan(transform.radians(0.5 * fov))
o = tf.ones([1], dtype=tf.float32)
diag = tf.concat([fov_factor, fov_factor, o], 0)
self._intrinsic_mat = tf.linalg.tensor_diag(diag)
else:
self._intrinsic_mat = tf.eye(3, dtype=tf.float32)
else:
self._intrinsic_mat = intrinsic_mat
self.intrinsic_mat_inv = tf.linalg.inv(self._intrinsic_mat)
self.clip_near = clip_near
self.resolution = resolution
self.camera_type = camera_type
if fisheye:
self.camera_type = redner.CameraType.fisheye
def compute_uvs(vertices, indices, print_progress = True):
"""
Compute UV coordinates of a given mesh using a charting algorithm
with least square conformal mapping. This calls the `xatlas <https://github.com/jpcy/xatlas>`_ library.
Args
====
vertices: tf.Tensor
3D position of vertices
float32 tensor with size num_vertices x 3
indices: tf.Tensor
vertex indices of triangle faces.
int32 tensor with size num_triangles x 3
Returns
=======
tf.Tensor
uv vertices pool, float32 Tensor with size num_uv_vertices x 3
tf.Tensor
uv indices, int32 Tensor with size num_triangles x 3
"""
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
vertices = tf.identity(vertices)
indices = tf.identity(indices)
uv_trimesh = redner.UVTriMesh(redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(0),
redner.int_ptr(0),
int(vertices.shape[0]),
0,
int(indices.shape[0]))
atlas = redner.TextureAtlas()
num_uv_vertices = redner.automatic_uv_map([uv_trimesh], atlas, print_progress)[0]
uvs = tf.zeros([num_uv_vertices, 2], dtype=tf.float32)
uv_indices = tf.zeros_like(indices)
uv_trimesh.uvs = redner.float_ptr(pyredner.data_ptr(uvs))
uv_trimesh.uv_indices = redner.int_ptr(pyredner.data_ptr(uv_indices))
uv_trimesh.num_uv_vertices = num_uv_vertices
redner.copy_texture_atlas(atlas, [uv_trimesh])
with tf.device(pyredner.get_device_name()):
vertices = tf.identity(vertices)
indices = tf.identity(indices)
uvs = tf.identity(uvs)
uv_indices = tf.identity(uv_indices)
return uvs, uv_indices
def __init__(self, values, env_to_world=tf.eye(4, 4)):
assert (tf.executing_eagerly())
# Convert to constant texture if necessary
if tf.is_tensor(values):
values = pyredner.Texture(values)
# assert(values.texels.is_contiguous())
assert (values.texels.dtype == tf.float32)
with tf.device(pyredner.get_device_name()):
# Build sampling table
luminance = 0.212671 * values.texels[:, :, 0] + \
0.715160 * values.texels[:, :, 1] + \
0.072169 * values.texels[:, :, 2]
# For each y, compute CDF over x
sample_cdf_xs_ = tf.cumsum(luminance, axis=1)
y_weight = tf.sin(
math.pi *
(tf.cast(tf.range(luminance.shape[0].value), tf.float32) + 0.5)
/ float(luminance.shape[0].value))
# Compute CDF for x
sample_cdf_ys_ = tf.cumsum(sample_cdf_xs_[:, -1] * y_weight,
axis=0)
pdf_norm = (luminance.shape[0].value * luminance.shape[1].value) / \
(sample_cdf_ys_[-1] * (2 * math.pi * math.pi))
# Normalize to [0, 1)
sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \
tf.math.maximum(
sample_cdf_xs_[
:,
(luminance.shape[1].value - 1):luminance.shape[1].value],
1e-8 * tf.convert_to_tensor(np.ones((sample_cdf_xs_.shape[0], 1)), dtype=tf.float32)
)
sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \
tf.math.maximum(sample_cdf_ys_[-1], tf.constant([1e-8]))
self.values = values
self.sample_cdf_ys = sample_cdf_ys
self.sample_cdf_xs = sample_cdf_xs
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self.pdf_norm = pdf_norm.cpu()
env_to_world = tf.identity(env_to_world).cpu()
self.env_to_world = env_to_world
self.world_to_env = tf.linalg.inv(env_to_world)
def __init__(self,
position: tf.Tensor,
look_at: tf.Tensor,
up: tf.Tensor,
fov: tf.Tensor,
clip_near: float,
resolution: Tuple[int],
cam_to_ndc: tf.Tensor = None,
camera_type=redner.CameraType.perspective,
fisheye: bool = False):
assert (tf.executing_eagerly())
assert (position.dtype == tf.float32)
assert (len(position.shape) == 1 and position.shape[0] == 3)
assert (look_at.dtype == tf.float32)
assert (len(look_at.shape) == 1 and look_at.shape[0] == 3)
assert (up.dtype == tf.float32)
assert (len(up.shape) == 1 and up.shape[0] == 3)
if fov is not None:
assert (fov.dtype == tf.float32)
assert (len(fov.shape) == 1 and fov.shape[0] == 1)
assert (isinstance(clip_near, float))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self.position = tf.identity(position).cpu()
self.look_at = tf.identity(look_at).cpu()
self.up = tf.identity(up).cpu()
self.fov = tf.identity(fov).cpu()
if cam_to_ndc is None:
if camera_type == redner.CameraType.perspective:
fov_factor = 1.0 / tf.tan(transform.radians(0.5 * fov))
o = tf.convert_to_tensor(np.ones([1], dtype=np.float32),
dtype=tf.float32)
diag = tf.concat([fov_factor, fov_factor, o], 0)
self._cam_to_ndc = tf.linalg.tensor_diag(diag)
else:
self._cam_to_ndc = tf.eye(3, dtype=tf.float32)
else:
self._cam_to_ndc = tf.identity(cam_to_ndc).cpu()
self.ndc_to_cam = tf.linalg.inv(self.cam_to_ndc)
self.clip_near = clip_near
self.resolution = resolution
self.camera_type = camera_type
if fisheye:
self.camera_type = redner.CameraType.fisheye
def unpack_args(seed,
args,
use_primary_edge_sampling=None,
use_secondary_edge_sampling=None):
"""
Given a list of serialized scene arguments, unpack
all information into a Context.
"""
# Unpack arguments
current_index = 0
num_shapes = int(args[current_index])
current_index += 1
num_materials = int(args[current_index])
current_index += 1
num_lights = int(args[current_index])
current_index += 1
# Camera arguments
cam_position = args[current_index]
current_index += 1
cam_look_at = args[current_index]
current_index += 1
cam_up = args[current_index]
current_index += 1
cam_to_world = args[current_index]
current_index += 1
world_to_cam = args[current_index]
current_index += 1
intrinsic_mat_inv = args[current_index]
current_index += 1
intrinsic_mat = args[current_index]
current_index += 1
clip_near = float(args[current_index])
current_index += 1
resolution = args[current_index].numpy() # Tuple[int, int]
current_index += 1
viewport = args[current_index].numpy() # Tuple[int, int, int, int]
current_index += 1
camera_type = RednerCameraType.asCameraType(
args[current_index]) # FIXME: Map to custom type
current_index += 1
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if is_empty_tensor(cam_to_world):
camera = redner.Camera(
resolution[1],
resolution[0],
redner.float_ptr(pyredner.data_ptr(cam_position)),
redner.float_ptr(pyredner.data_ptr(cam_look_at)),
redner.float_ptr(pyredner.data_ptr(cam_up)),
redner.float_ptr(0), # cam_to_world
redner.float_ptr(0), # world_to_cam
redner.float_ptr(pyredner.data_ptr(intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(intrinsic_mat)),
clip_near,
camera_type,
redner.Vector2i(viewport[1], viewport[0]),
redner.Vector2i(viewport[3], viewport[2]))
else:
camera = redner.Camera(
resolution[1], resolution[0], redner.float_ptr(0),
redner.float_ptr(0), redner.float_ptr(0),
redner.float_ptr(pyredner.data_ptr(cam_to_world)),
redner.float_ptr(pyredner.data_ptr(world_to_cam)),
redner.float_ptr(pyredner.data_ptr(intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(intrinsic_mat)), clip_near,
camera_type, redner.Vector2i(viewport[1], viewport[0]),
redner.Vector2i(viewport[3], viewport[2]))
with tf.device(pyredner.get_device_name()):
shapes = []
for i in range(num_shapes):
vertices = args[current_index]
current_index += 1
indices = args[current_index]
current_index += 1
uvs = args[current_index]
current_index += 1
normals = args[current_index]
current_index += 1
uv_indices = args[current_index]
current_index += 1
normal_indices = args[current_index]
current_index += 1
colors = args[current_index]
current_index += 1
material_id = int(args[current_index])
current_index += 1
light_id = int(args[current_index])
current_index += 1
shapes.append(redner.Shape(\
redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(pyredner.data_ptr(uvs) if not is_empty_tensor(uvs) else 0),
redner.float_ptr(pyredner.data_ptr(normals) if not is_empty_tensor(normals) else 0),
redner.int_ptr(pyredner.data_ptr(uv_indices) if not is_empty_tensor(uv_indices) else 0),
redner.int_ptr(pyredner.data_ptr(normal_indices) if not is_empty_tensor(normal_indices) else 0),
redner.float_ptr(pyredner.data_ptr(colors) if not is_empty_tensor(colors) else 0),
int(vertices.shape[0]),
int(uvs.shape[0]) if not is_empty_tensor(uvs) else 0,
int(normals.shape[0]) if not is_empty_tensor(normals) else 0,
int(indices.shape[0]),
material_id,
light_id))
materials = []
with tf.device(pyredner.get_device_name()):
for i in range(num_materials):
num_levels = int(args[current_index])
current_index += 1
diffuse_reflectance = []
for j in range(num_levels):
diffuse_reflectance.append(args[current_index])
current_index += 1
diffuse_uv_scale = args[current_index]
current_index += 1
num_levels = int(args[current_index])
current_index += 1
specular_reflectance = []
for j in range(num_levels):
specular_reflectance.append(args[current_index])
current_index += 1
specular_uv_scale = args[current_index]
current_index += 1
num_levels = int(args[current_index])
current_index += 1
roughness = []
for j in range(num_levels):
roughness.append(args[current_index])
current_index += 1
roughness_uv_scale = args[current_index]
current_index += 1
num_levels = int(args[current_index])
current_index += 1
generic_texture = []
if num_levels > 0:
for j in range(num_levels):
generic_texture.append(args[current_index])
current_index += 1
generic_uv_scale = args[current_index]
current_index += 1
else:
generic_uv_scale = None
num_levels = int(args[current_index])
current_index += 1
normal_map = []
if num_levels > 0:
for j in range(num_levels):
normal_map.append(args[current_index])
current_index += 1
normal_map_uv_scale = args[current_index]
current_index += 1
else:
normal_map_uv_scale = None
compute_specular_lighting = bool(args[current_index])
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
use_vertex_color = bool(args[current_index])
current_index += 1
if get_tensor_dimension(diffuse_reflectance[0]) == 1:
diffuse_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(diffuse_reflectance[0]))],
[0],
[0],
3, redner.float_ptr(pyredner.data_ptr(diffuse_uv_scale)))
else:
assert (get_tensor_dimension(diffuse_reflectance[0]) == 3)
diffuse_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in diffuse_reflectance],
[x.shape[1] for x in diffuse_reflectance],
[x.shape[0] for x in diffuse_reflectance],
3,
redner.float_ptr(pyredner.data_ptr(diffuse_uv_scale)))
if get_tensor_dimension(specular_reflectance[0]) == 1:
specular_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(specular_reflectance[0]))],
[0],
[0],
3, redner.float_ptr(pyredner.data_ptr(specular_uv_scale)))
else:
assert (get_tensor_dimension(specular_reflectance[0]) == 3)
specular_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in specular_reflectance],
[x.shape[1] for x in specular_reflectance],
[x.shape[0] for x in specular_reflectance],
3,
redner.float_ptr(pyredner.data_ptr(specular_uv_scale)))
if get_tensor_dimension(roughness[0]) == 1:
roughness = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(roughness[0]))],
[0],
[0],
1, redner.float_ptr(pyredner.data_ptr(roughness_uv_scale)))
else:
assert (get_tensor_dimension(roughness[0]) == 3)
roughness = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in roughness],
[x.shape[1] for x in roughness],
[x.shape[0] for x in roughness],
3,
redner.float_ptr(pyredner.data_ptr(roughness_uv_scale)))
if len(generic_texture) > 0:
generic_texture = redner.TextureN(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in generic_texture],
[x.shape[1] for x in generic_texture],
[x.shape[0] for x in generic_texture],
generic_texture[0].shape[2],
redner.float_ptr(pyredner.data_ptr(generic_uv_scale)))
else:
generic_texture = redner.TextureN(\
[], [], [], 0, redner.float_ptr(0))
if len(normal_map) > 0:
normal_map = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in normal_map],
[x.shape[1] for x in normal_map],
[x.shape[0] for x in normal_map],
normal_map[0].shape[2],
redner.float_ptr(pyredner.data_ptr(normal_map_uv_scale)))
else:
normal_map = redner.Texture3(\
[], [], [], 0, redner.float_ptr(0))
materials.append(redner.Material(\
diffuse_reflectance,
specular_reflectance,
roughness,
generic_texture,
normal_map,
compute_specular_lighting,
two_sided,
use_vertex_color))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
area_lights = []
for i in range(num_lights):
shape_id = int(args[current_index])
current_index += 1
intensity = args[current_index]
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
directly_visible = bool(args[current_index])
current_index += 1
area_lights.append(
redner.AreaLight(
shape_id, redner.float_ptr(pyredner.data_ptr(intensity)),
two_sided, directly_visible))
envmap = None
if not is_empty_tensor(args[current_index]):
num_levels = args[current_index]
current_index += 1
values = []
for j in range(num_levels):
values.append(args[current_index])
current_index += 1
envmap_uv_scale = args[current_index]
current_index += 1
env_to_world = args[current_index]
current_index += 1
world_to_env = args[current_index]
current_index += 1
sample_cdf_ys = args[current_index]
current_index += 1
sample_cdf_xs = args[current_index]
current_index += 1
pdf_norm = float(args[current_index])
current_index += 1
directly_visible = bool(args[current_index])
current_index += 1
assert isinstance(pdf_norm, float)
with tf.device(pyredner.get_device_name()):
sample_cdf_ys = redner.float_ptr(pyredner.data_ptr(sample_cdf_ys))
sample_cdf_xs = redner.float_ptr(pyredner.data_ptr(sample_cdf_xs))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
env_to_world = redner.float_ptr(pyredner.data_ptr(env_to_world))
world_to_env = redner.float_ptr(pyredner.data_ptr(world_to_env))
with tf.device(pyredner.get_device_name()):
values = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in values],
[x.shape[1] for x in values], # width
[x.shape[0] for x in values], # height
3, # channels
redner.float_ptr(pyredner.data_ptr(envmap_uv_scale)))
envmap = redner.EnvironmentMap(\
values,
env_to_world,
world_to_env,
sample_cdf_ys,
sample_cdf_xs,
pdf_norm,
directly_visible)
else:
current_index += 1
# Options
num_samples = args[current_index]
current_index += 1
if len(num_samples.shape) == 0 or num_samples.shape[0] == 1:
num_samples = int(num_samples)
else:
assert (num_samples.shape[0] == 2)
num_samples = (int(num_samples[0]), int(num_samples[1]))
max_bounces = int(args[current_index])
current_index += 1
num_channel_args = int(args[current_index])
current_index += 1
channels = []
for _ in range(num_channel_args):
ch = args[current_index]
ch = RednerChannels.asChannel(ch)
channels.append(ch)
current_index += 1
sampler_type = args[current_index]
sampler_type = RednerSamplerType.asSamplerType(sampler_type)
current_index += 1
use_primary_edge_sampling = args[current_index]
current_index += 1
use_secondary_edge_sampling = args[current_index]
current_index += 1
sample_pixel_center = args[current_index]
current_index += 1
start = time.time()
scene = redner.Scene(camera, shapes, materials, area_lights, envmap,
pyredner.get_use_gpu(), pyredner.get_gpu_device_id(),
use_primary_edge_sampling,
use_secondary_edge_sampling)
time_elapsed = time.time() - start
if get_print_timing():
print('Scene construction, time: %.5f s' % time_elapsed)
# check that num_samples is a tuple
if isinstance(num_samples, int):
num_samples = (num_samples, num_samples)
options = redner.RenderOptions(seed, num_samples[0], max_bounces, channels,
sampler_type, sample_pixel_center)
ctx = Context()
ctx.channels = channels
ctx.options = options
ctx.resolution = resolution
ctx.viewport = viewport
ctx.scene = scene
ctx.camera = camera
ctx.shapes = shapes
ctx.materials = materials
ctx.area_lights = area_lights
ctx.envmap = envmap
ctx.scene = scene
ctx.options = options
ctx.num_samples = num_samples
ctx.num_channel_args = num_channel_args
return ctx
def forward(seed: int, *args):
"""
Forward rendering pass: given a scene and output an image.
"""
global __ctx
ctx = __ctx
# Unpack arguments
current_index = 0
num_shapes = int(args[current_index])
current_index += 1
num_materials = int(args[current_index])
current_index += 1
num_lights = int(args[current_index])
current_index += 1
# Camera arguments
cam_position = args[current_index]
current_index += 1
cam_look_at = args[current_index]
current_index += 1
cam_up = args[current_index]
current_index += 1
ndc_to_cam = args[current_index]
current_index += 1
cam_to_ndc = args[current_index]
current_index += 1
clip_near = float(args[current_index])
current_index += 1
resolution = args[current_index].numpy() # Tuple[int, int]
current_index += 1
camera_type = pyredner.RednerCameraType.asCameraType(
args[current_index]) # FIXME: Map to custom type
current_index += 1
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
camera = redner.Camera(
resolution[1], resolution[0],
redner.float_ptr(pyredner.data_ptr(cam_position)),
redner.float_ptr(pyredner.data_ptr(cam_look_at)),
redner.float_ptr(pyredner.data_ptr(cam_up)),
redner.float_ptr(pyredner.data_ptr(ndc_to_cam)),
redner.float_ptr(pyredner.data_ptr(cam_to_ndc)), clip_near,
camera_type)
with tf.device(pyredner.get_device_name()):
shapes = []
for i in range(num_shapes):
vertices = args[current_index]
current_index += 1
indices = args[current_index]
current_index += 1
uvs = args[current_index]
current_index += 1
normals = args[current_index]
current_index += 1
material_id = int(args[current_index])
current_index += 1
light_id = int(args[current_index])
current_index += 1
shapes.append(redner.Shape(\
redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(pyredner.data_ptr(uvs) if uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(normals) if normals is not None else 0),
int(vertices.shape[0]),
int(indices.shape[0]),
material_id,
light_id))
materials = []
with tf.device(pyredner.get_device_name()):
for i in range(num_materials):
diffuse_reflectance = args[current_index]
current_index += 1
diffuse_uv_scale = args[current_index]
current_index += 1
specular_reflectance = args[current_index]
current_index += 1
specular_uv_scale = args[current_index]
current_index += 1
roughness = args[current_index]
current_index += 1
roughness_uv_scale = args[current_index]
current_index += 1
normal_map = args[current_index]
current_index += 1
normal_map_uv_scale = args[current_index]
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
diffuse_reflectance_ptr = redner.float_ptr(
pyredner.data_ptr(diffuse_reflectance))
specular_reflectance_ptr = redner.float_ptr(
pyredner.data_ptr(specular_reflectance))
roughness_ptr = redner.float_ptr(pyredner.data_ptr(roughness))
if normal_map.shape[0] > 0:
normal_map_ptr = redner.float_ptr(
pyredner.data_ptr(normal_map))
diffuse_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(diffuse_uv_scale))
specular_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(specular_uv_scale))
roughness_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(roughness_uv_scale))
if normal_map.shape[0] > 0:
normal_map_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(normal_map_uv_scale))
if get_tensor_dimension(diffuse_reflectance) == 1:
diffuse_reflectance = redner.Texture3(diffuse_reflectance_ptr,
0, 0, 0,
diffuse_uv_scale_ptr)
else:
diffuse_reflectance = redner.Texture3(\
diffuse_reflectance_ptr,
int(diffuse_reflectance.shape[2]), # width
int(diffuse_reflectance.shape[1]), # height
int(diffuse_reflectance.shape[0]), # num levels
diffuse_uv_scale_ptr)
if get_tensor_dimension(specular_reflectance) == 1:
specular_reflectance = redner.Texture3(
specular_reflectance_ptr, 0, 0, 0, specular_uv_scale_ptr)
else:
specular_reflectance = redner.Texture3(\
specular_reflectance_ptr,
int(specular_reflectance.shape[2]), # width
int(specular_reflectance.shape[1]), # height
int(specular_reflectance.shape[0]), # num levels
specular_uv_scale_ptr)
if get_tensor_dimension(roughness) == 1:
roughness = redner.Texture1(roughness_ptr, 0, 0, 0,
roughness_uv_scale_ptr)
else:
assert (get_tensor_dimension(roughness) == 4)
roughness = redner.Texture1(\
roughness_ptr,
int(roughness.shape[2]), # width
int(roughness.shape[1]), # height
int(roughness.shape[0]), # num levels
roughness_uv_scale_ptr)
if normal_map.shape[0] > 0:
normal_map = redner.Texture3(\
normal_map_ptr,
int(normal_map.shape[2]),
int(normal_map.shape[1]),
int(normal_map.shape[0]),
normal_map_uv_scale_ptr)
else:
normal_map = redner.Texture3(\
redner.float_ptr(0), 0, 0, 0, redner.float_ptr(0))
materials.append(redner.Material(\
diffuse_reflectance,
specular_reflectance,
roughness,
normal_map,
two_sided))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
area_lights = []
for i in range(num_lights):
shape_id = int(args[current_index])
current_index += 1
intensity = args[current_index]
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
area_lights.append(
redner.AreaLight(
shape_id, redner.float_ptr(pyredner.data_ptr(intensity)),
two_sided))
envmap = None
if not is_empty_tensor(args[current_index]):
values = args[current_index]
current_index += 1
envmap_uv_scale = args[current_index]
current_index += 1
env_to_world = args[current_index]
current_index += 1
world_to_env = args[current_index]
current_index += 1
sample_cdf_ys = args[current_index]
current_index += 1
sample_cdf_xs = args[current_index]
current_index += 1
pdf_norm = float(args[current_index])
current_index += 1
assert isinstance(pdf_norm, float)
with tf.device(pyredner.get_device_name()):
values_ptr = redner.float_ptr(pyredner.data_ptr(values))
sample_cdf_ys = redner.float_ptr(pyredner.data_ptr(sample_cdf_ys))
sample_cdf_xs = redner.float_ptr(pyredner.data_ptr(sample_cdf_xs))
envmap_uv_scale = redner.float_ptr(
pyredner.data_ptr(envmap_uv_scale))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
env_to_world = redner.float_ptr(pyredner.data_ptr(env_to_world))
world_to_env = redner.float_ptr(pyredner.data_ptr(world_to_env))
values = redner.Texture3(
values_ptr,
int(values.shape[2]), # width
int(values.shape[1]), # height
int(values.shape[0]), # num levels
envmap_uv_scale)
envmap = redner.EnvironmentMap(\
values,
env_to_world,
world_to_env,
sample_cdf_ys,
sample_cdf_xs,
pdf_norm)
else:
current_index += 7
# Options
num_samples = int(args[current_index])
current_index += 1
max_bounces = int(args[current_index])
current_index += 1
__num_channels = int(args[current_index])
current_index += 1
channels = []
for _ in range(__num_channels):
ch = args[current_index]
ch = pyredner.RednerChannels.asChannel(ch)
channels.append(ch)
current_index += 1
sampler_type = args[current_index]
sampler_type = pyredner.RednerSamplerType.asSamplerType(sampler_type)
current_index += 1
use_primary_edge_sampling = args[current_index]
current_index += 1
use_secondary_edge_sampling = args[current_index]
current_index += 1
scene = redner.Scene(camera, shapes, materials, area_lights, envmap,
pyredner.get_use_gpu(), pyredner.get_gpu_device_id(),
use_primary_edge_sampling,
use_secondary_edge_sampling)
# check that num_samples is a tuple
if isinstance(num_samples, int):
num_samples = (num_samples, num_samples)
options = redner.RenderOptions(seed, num_samples[0], max_bounces, channels,
sampler_type)
num_channels = redner.compute_num_channels(channels)
with tf.device(pyredner.get_device_name()):
rendered_image = tf.zeros(
shape=[resolution[0], resolution[1], num_channels],
dtype=tf.float32)
start = time.time()
# pdb.set_trace()
redner.render(scene, options,
redner.float_ptr(pyredner.data_ptr(rendered_image)),
redner.float_ptr(0), None, redner.float_ptr(0))
time_elapsed = time.time() - start
if print_timing:
print('Forward pass, time: %.5f s' % time_elapsed)
# # For debugging
# debug_img = tf.zeros((256, 256, 3), dtype=tf.float32)
# redner.render(scene,
# options,
# redner.float_ptr(pyredner.data_ptr(rendered_image)),
# redner.float_ptr(0),
# None,
# redner.float_ptr(pyredner.data_ptr(debug_img)))
# pyredner.imwrite(debug_img, 'debug.png')
# exit()
# import pdb; pdb.set_trace()
ctx.shapes = shapes
ctx.materials = materials
ctx.area_lights = area_lights
ctx.envmap = envmap
ctx.scene = scene
ctx.options = options
ctx.num_samples = num_samples
ctx.num_channels = __num_channels
return rendered_image
np.min(specular))
specular = tf.convert_to_tensor(np.tile(np.reshape(specular, (256, 256, 1)),
(1, 1, 3)),
dtype=tf.float32)
roughness = perlin(x, y, seed=2)
roughness = (roughness - np.min(roughness) + 1e-3) / (np.max(roughness) -
np.min(roughness))
roughness = tf.convert_to_tensor(np.reshape(roughness, (256, 256, 1)),
dtype=tf.float32)
# Use GPU if available
pyredner.set_use_gpu(
tf.test.is_gpu_available(cuda_only=True, min_cuda_compute_capability=None))
# Set up the scene using Pytorch tensor
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
position = tf.Variable([0.0, 0.0, -5.0], dtype=tf.float32)
look_at = tf.Variable([0.0, 0.0, 0.0], dtype=tf.float32)
up = tf.Variable([0.0, 1.0, 0.0], dtype=tf.float32)
fov = tf.Variable([45.0], dtype=tf.float32)
clip_near = 1e-2
resolution = (256, 256)
cam = pyredner.Camera(position=position,
look_at=look_at,
up=up,
fov=fov,
clip_near=clip_near,
resolution=resolution)
mat_perlin = pyredner.Material(diffuse_reflectance=diffuse,
specular_reflectance=specular,
def env_to_world(self, value):
self._env_to_world = value
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self.world_to_env = tf.linalg.inv(self._env_to_world)
def cam_to_ndc(self, value):
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self._cam_to_ndc = tf.identity(value).cpu()
self.ndc_to_cam = tf.linalg.inv(self._cam_to_ndc)
def serialize_scene(scene: pyredner.Scene,
num_samples: Union[int, Tuple[int, int]],
max_bounces: int,
channels=[redner.channels.radiance],
sampler_type=redner.SamplerType.independent,
use_primary_edge_sampling=True,
use_secondary_edge_sampling=True) -> List:
"""
Given a pyredner scene & rendering options, convert them to a linear list of argument,
so that we can use it in PyTorch.
Args
====
scene: pyredner.Scene
num_samples: int
number of samples per pixel for forward and backward passes
can be an integer or a tuple of 2 integers
if a single integer is provided, use the same number of samples
for both
max_bounces: int
number of bounces for global illumination
1 means direct lighting only
channels: List[redner.channels]
| A list of channels that should present in the output image
| following channels are supported\:
| redner.channels.radiance,
| redner.channels.alpha,
| redner.channels.depth,
| redner.channels.position,
| redner.channels.geometry_normal,
| redner.channels.shading_normal,
| redner.channels.uv,
| redner.channels.diffuse_reflectance,
| redner.channels.specular_reflectance,
| redner.channels.vertex_color,
| redner.channels.roughness,
| redner.channels.generic_texture,
| redner.channels.shape_id,
| redner.channels.material_id
| all channels, except for shape id and material id, are differentiable
sampler_type: redner.SamplerType
| Which sampling pattern to use?
| see `Chapter 7 of the PBRT book <http://www.pbr-book.org/3ed-2018/Sampling_and_Reconstruction.html>`
for an explanation of the difference between different samplers.
| Following samplers are supported:
| redner.SamplerType.independent
| redner.SamplerType.sobol
use_primary_edge_sampling: bool
use_secondary_edge_sampling: bool
"""
cam = scene.camera
num_shapes = len(scene.shapes)
num_materials = len(scene.materials)
num_lights = len(scene.area_lights)
num_channels = len(channels)
for light_id, light in enumerate(scene.area_lights):
scene.shapes[light.shape_id].light_id = light_id
args = []
args.append(tf.constant(num_shapes))
args.append(tf.constant(num_materials))
args.append(tf.constant(num_lights))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if cam.position is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.position))
args.append(tf.identity(cam.look_at))
args.append(tf.identity(cam.up))
if cam.cam_to_world is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.cam_to_world))
args.append(tf.identity(cam.world_to_cam))
args.append(tf.identity(cam.intrinsic_mat_inv))
args.append(tf.identity(cam.intrinsic_mat))
args.append(tf.constant(cam.clip_near))
args.append(tf.constant(cam.resolution))
args.append(RednerCameraType.asTensor(cam.camera_type))
for shape in scene.shapes:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(shape.vertices))
args.append(tf.identity(shape.indices))
if shape.uvs is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.uvs))
if shape.normals is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.normals))
if shape.uv_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.uv_indices))
if shape.normal_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.normal_indices))
if shape.colors is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.colors))
args.append(tf.constant(shape.material_id))
args.append(tf.constant(shape.light_id))
for material in scene.materials:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(material.diffuse_reflectance.mipmap))
args.append(tf.identity(material.diffuse_reflectance.uv_scale))
args.append(tf.identity(material.specular_reflectance.mipmap))
args.append(tf.identity(material.specular_reflectance.uv_scale))
args.append(tf.identity(material.roughness.mipmap))
args.append(tf.identity(material.roughness.uv_scale))
if material.generic_texture is not None:
args.append(tf.identity(material.generic_texture.mipmap))
args.append(tf.identity(material.generic_texture.uv_scale))
else:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
if material.normal_map is not None:
args.append(tf.identity(material.normal_map.mipmap))
args.append(tf.identity(material.normal_map.uv_scale))
else:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(tf.constant(material.compute_specular_lighting))
args.append(tf.constant(material.two_sided))
args.append(tf.constant(material.use_vertex_color))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
for light in scene.area_lights:
args.append(tf.constant(light.shape_id))
args.append(tf.identity(light.intensity))
args.append(tf.constant(light.two_sided))
if scene.envmap is not None:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(scene.envmap.values.mipmap))
args.append(tf.identity(scene.envmap.values.uv_scale))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
args.append(tf.identity(scene.envmap.env_to_world))
args.append(tf.identity(scene.envmap.world_to_env))
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(scene.envmap.sample_cdf_ys))
args.append(tf.identity(scene.envmap.sample_cdf_xs))
args.append(scene.envmap.pdf_norm)
else:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(tf.constant(num_samples))
args.append(tf.constant(max_bounces))
args.append(tf.constant(num_channels))
for ch in channels:
args.append(RednerChannels.asTensor(ch))
args.append(RednerSamplerType.asTensor(sampler_type))
args.append(tf.constant(use_primary_edge_sampling))
args.append(tf.constant(use_secondary_edge_sampling))
return args
def serialize_scene(scene: pyredner.Scene,
num_samples: Union[int, Tuple[int, int]],
max_bounces: int,
channels=[redner.channels.radiance],
sampler_type=redner.SamplerType.independent,
use_primary_edge_sampling=True,
use_secondary_edge_sampling=True,
sample_pixel_center: bool = False) -> List:
"""
Given a pyredner scene & rendering options, convert them to a linear list of argument,
so that we can use it in TensorFlow.
Args
====
scene: pyredner.Scene
num_samples: int
number of samples per pixel for forward and backward passes
can be an integer or a tuple of 2 integers
if a single integer is provided, use the same number of samples
for both
max_bounces: int
number of bounces for global illumination
1 means direct lighting only
channels: List[redner.channels]
| A list of channels that should present in the output image
| following channels are supported\:
| redner.channels.radiance,
| redner.channels.alpha,
| redner.channels.depth,
| redner.channels.position,
| redner.channels.geometry_normal,
| redner.channels.shading_normal,
| redner.channels.uv,
| redner.channels.diffuse_reflectance,
| redner.channels.specular_reflectance,
| redner.channels.vertex_color,
| redner.channels.roughness,
| redner.channels.generic_texture,
| redner.channels.shape_id,
| redner.channels.triangle_id,
| redner.channels.material_id
| all channels, except for shape id, triangle id and material id, are differentiable
sampler_type: redner.SamplerType
| Which sampling pattern to use?
| see `Chapter 7 of the PBRT book <http://www.pbr-book.org/3ed-2018/Sampling_and_Reconstruction.html>`
for an explanation of the difference between different samplers.
| Following samplers are supported:
| redner.SamplerType.independent
| redner.SamplerType.sobol
use_primary_edge_sampling: bool
use_secondary_edge_sampling: bool
sample_pixel_center: bool
Always sample at the pixel center when rendering.
This trades noise with aliasing.
If this option is activated, the rendering becomes non-differentiable
(since there is no antialiasing integral),
and redner's edge sampling becomes an approximation to the gradients of the aliased rendering.
"""
# TODO: figure out a way to determine whether a TF tensor requires gradient or not
cam = scene.camera
num_shapes = len(scene.shapes)
num_materials = len(scene.materials)
num_lights = len(scene.area_lights)
num_channels = len(channels)
for light_id, light in enumerate(scene.area_lights):
scene.shapes[light.shape_id].light_id = light_id
if max_bounces == 0:
use_secondary_edge_sampling = False
args = []
args.append(tf.constant(num_shapes))
args.append(tf.constant(num_materials))
args.append(tf.constant(num_lights))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if cam.position is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.position))
args.append(tf.identity(cam.look_at))
args.append(tf.identity(cam.up))
if cam.cam_to_world is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.cam_to_world))
args.append(tf.identity(cam.world_to_cam))
args.append(tf.identity(cam.intrinsic_mat_inv))
args.append(tf.identity(cam.intrinsic_mat))
args.append(tf.constant(cam.clip_near))
args.append(tf.constant(cam.resolution))
viewport = cam.viewport
if viewport is None:
viewport = (0, 0, cam.resolution[0], cam.resolution[1])
# Clamp the viewport if necessary
viewport = (max(viewport[0], 0), max(viewport[1], 0),
min(viewport[2],
cam.resolution[0]), min(viewport[3], cam.resolution[1]))
args.append(tf.constant(viewport))
args.append(RednerCameraType.asTensor(cam.camera_type))
for shape in scene.shapes:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(shape.vertices))
# HACK: tf.bitcast forces tensorflow to copy int32 to GPU memory.
# tf.identity stopped working since TF 2.1 (if you print the device
# it will say it's on GPU, but the address returned by data_ptr is wrong).
# Hopefully TF people will fix this in the future.
args.append(tf.bitcast(shape.indices, type=tf.int32))
if shape.uvs is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.uvs))
if shape.normals is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.normals))
if shape.uv_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.bitcast(shape.uv_indices, type=tf.int32))
if shape.normal_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.bitcast(shape.normal_indices, type=tf.int32))
if shape.colors is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.colors))
args.append(tf.constant(shape.material_id))
args.append(tf.constant(shape.light_id))
for material in scene.materials:
serialize_texture(material.diffuse_reflectance, args)
serialize_texture(material.specular_reflectance, args)
serialize_texture(material.roughness, args)
serialize_texture(material.generic_texture, args)
serialize_texture(material.normal_map, args)
args.append(tf.constant(material.compute_specular_lighting))
args.append(tf.constant(material.two_sided))
args.append(tf.constant(material.use_vertex_color))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
for light in scene.area_lights:
args.append(tf.constant(light.shape_id))
args.append(tf.identity(light.intensity))
args.append(tf.constant(light.two_sided))
args.append(tf.constant(light.directly_visible))
if scene.envmap is not None:
serialize_texture(scene.envmap.values, args)
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
args.append(tf.identity(scene.envmap.env_to_world))
args.append(tf.identity(scene.envmap.world_to_env))
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(scene.envmap.sample_cdf_ys))
args.append(tf.identity(scene.envmap.sample_cdf_xs))
args.append(scene.envmap.pdf_norm)
args.append(scene.envmap.directly_visible)
else:
args.append(__EMPTY_TENSOR)
args.append(tf.constant(num_samples))
args.append(tf.constant(max_bounces))
args.append(tf.constant(num_channels))
for ch in channels:
args.append(RednerChannels.asTensor(ch))
args.append(RednerSamplerType.asTensor(sampler_type))
args.append(tf.constant(use_primary_edge_sampling))
args.append(tf.constant(use_secondary_edge_sampling))
args.append(tf.constant(sample_pixel_center))
return args
def parse_shape(node, material_dict, shape_id):
if node.attrib['type'] == 'obj' or node.attrib['type'] == 'serialized':
to_world = tf.eye(4)
serialized_shape_id = 0
mat_id = -1
light_intensity = None
filename = ''
max_smooth_angle = -1
for child in node:
if 'name' in child.attrib:
if child.attrib['name'] == 'filename':
filename = child.attrib['value']
elif child.attrib['name'] == 'toWorld':
to_world = parse_transform(child)
elif child.attrib['name'] == 'shapeIndex':
serialized_shape_id = int(child.attrib['value'])
elif child.attrib['name'] == 'maxSmoothAngle':
max_smooth_angle = float(child.attrib['value'])
if child.tag == 'ref':
mat_id = material_dict[child.attrib['id']]
elif child.tag == 'emitter':
for grandchild in child:
if grandchild.attrib['name'] == 'radiance':
light_intensity = parse_vector(
grandchild.attrib['value'])
if light_intensity.shape[0] == 1:
light_intensity = tf.constant([
light_intensity[0], light_intensity[0],
light_intensity[0]
])
if node.attrib['type'] == 'obj':
_, mesh_list, _ = pyredner.load_obj(filename, obj_group=False)
# Convert to CPU for rebuild_topology
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
vertices = tf.identity(mesh_list[0][1].vertices)
indices = tf.identity(mesh_list[0][1].indices)
uvs = mesh_list[0][1].uvs
normals = mesh_list[0][1].normals
uv_indices = mesh_list[0][1].uv_indices
normal_indices = mesh_list[0][1].normal_indices
if uvs is not None:
uvs = tf.identity(uvs)
if normals is not None:
normals = tf.identity(normals)
if uv_indices is not None:
uv_indices = tf.identity(uv_indices)
else:
assert (node.attrib['type'] == 'serialized')
mitsuba_tri_mesh = redner.load_serialized(filename,
serialized_shape_id)
vertices = tf.convert_to_tensor(mitsuba_tri_mesh.vertices)
indices = tf.convert_to_tensor(mitsuba_tri_mesh.indices)
uvs = tf.convert_to_tensor(mitsuba_tri_mesh.uvs)
normals = tf.convert_to_tensor(mitsuba_tri_mesh.normals)
if uvs.shape[0] == 0:
uvs = None
if normals.shape[0] == 0:
normals = None
uv_indices = None # Serialized doesn't use different indices for UV & normal
normal_indices = None
# Transform the vertices and normals
vertices = tf.concat(
(vertices, tf.ones([vertices.shape[0], 1], dtype=tf.float32)),
axis=1)
vertices = vertices @ tf.transpose(to_world, [1, 0])
vertices = vertices / vertices[:, 3:4]
vertices = vertices[:, 0:3]
if normals is not None:
normals = normals @ (tf.linalg.inv(tf.transpose(to_world,
[0, 1]))[:3, :3])
assert (vertices is not None)
assert (indices is not None)
if max_smooth_angle >= 0:
if normals is None:
normals = tf.zeros_like(vertices)
new_num_vertices = redner.rebuild_topology(\
redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(pyredner.data_ptr(uvs) if uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(normals) if normals is not None else 0),
redner.int_ptr(pyredner.data_ptr(uv_indices) if uv_indices is not None else 0),
int(vertices.shape[0]),
int(indices.shape[0]),
max_smooth_angle)
print('Rebuilt topology, original vertices size: {}, new vertices size: {}'.format(\
int(vertices.shape[0]), new_num_vertices))
vertices.resize_(new_num_vertices, 3)
if uvs is not None:
uvs.resize_(new_num_vertices, 2)
if normals is not None:
normals.resize_(new_num_vertices, 3)
lgt = None
if light_intensity is not None:
lgt = pyredner.AreaLight(shape_id, light_intensity)
return pyredner.Shape(vertices=vertices,
indices=indices,
uvs=uvs,
normals=normals,
uv_indices=uv_indices,
normal_indices=normal_indices,
material_id=mat_id), lgt
elif node.attrib['type'] == 'rectangle':
indices = tf.constant([[0, 2, 1], [1, 2, 3]], dtype=tf.int32)
vertices = tf.constant([[-1.0, -1.0, 0.0], [-1.0, 1.0, 0.0],
[1.0, -1.0, 0.0], [1.0, 1.0, 0.0]])
uvs = None
normals = None
to_world = tf.eye(4)
mat_id = -1
light_intensity = None
for child in node:
if 'name' in child.attrib:
if child.attrib['name'] == 'toWorld':
to_world = parse_transform(child)
if child.tag == 'ref':
mat_id = material_dict[child.attrib['id']]
elif child.tag == 'emitter':
for grandchild in child:
if grandchild.attrib['name'] == 'radiance':
light_intensity = parse_vector(
grandchild.attrib['value'])
if light_intensity.shape[0] == 1:
light_intensity = tf.constant([
light_intensity[0], light_intensity[0],
light_intensity[0]
])
# Transform the vertices and normals
vertices = tf.concat(
(vertices,
tf.convert_to_tensor(np.ones(vertices.shape[0], 1),
dtype=tf.float32)),
axis=1)
vertices = vertices @ tf.transpose(to_world, [0, 1])
vertices = vertices / vertices[:, 3:4]
vertices = vertices[:, 0:3]
if normals is not None:
normals = normals @ (tf.linalg.inv(tf.transpose(to_world,
[0, 1]))[:3, :3])
assert (vertices is not None)
assert (indices is not None)
lgt = None
if light_intensity is not None:
lgt = pyrender.Light(shape_id, light_intensity)
return pyredner.Shape(vertices=vertices,
indices=indices,
uvs=uvs,
normals=normals,
material_id=mat_id), lgt
else:
assert (False)
def backward(grad_img):
scene = ctx.scene
options = ctx.options
buffers = create_gradient_buffers(ctx)
if not get_use_correlated_random_number():
# Decod_uple the forward/backward random numbers by adding a big prime number
options.seed += 1000003
start = time.time()
options.num_samples = ctx.num_samples[1]
with tf.device(pyredner.get_device_name()):
grad_img = tf.identity(grad_img)
redner.render(
scene,
options,
redner.float_ptr(0), # rendered_image
redner.float_ptr(pyredner.data_ptr(grad_img)),
buffers.d_scene,
redner.float_ptr(0), # translational_gradient_image
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if get_print_timing():
print('Backward pass, time: %.5f s' % time_elapsed)
ret_list = []
ret_list.append(None) # seed
ret_list.append(None) # num_shapes
ret_list.append(None) # num_materials
ret_list.append(None) # num_lights
if ctx.camera.use_look_at:
ret_list.append(buffers.d_position)
ret_list.append(buffers.d_look_at)
ret_list.append(buffers.d_up)
ret_list.append(None) # cam_to_world
ret_list.append(None) # world_to_cam
else:
ret_list.append(None) # pos
ret_list.append(None) # look
ret_list.append(None) # up
ret_list.append(buffers.d_cam_to_world)
ret_list.append(buffers.d_world_to_cam)
ret_list.append(buffers.d_intrinsic_mat_inv)
ret_list.append(buffers.d_intrinsic_mat)
ret_list.append(None) # clip near
ret_list.append(None) # resolution
ret_list.append(None) # viewport
ret_list.append(None) # camera_type
num_shapes = len(ctx.shapes)
for i in range(num_shapes):
ret_list.append(buffers.d_vertices_list[i])
ret_list.append(None) # indices
ret_list.append(buffers.d_uvs_list[i])
ret_list.append(buffers.d_normals_list[i])
ret_list.append(None) # uv_indices
ret_list.append(None) # normal_indices
ret_list.append(buffers.d_colors_list[i])
ret_list.append(None) # material id
ret_list.append(None) # light id
num_materials = len(ctx.materials)
for i in range(num_materials):
ret_list.append(None) # num_levels
for d_diffuse in buffers.d_diffuse_list[i]:
ret_list.append(d_diffuse)
ret_list.append(buffers.d_diffuse_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_specular in buffers.d_specular_list[i]:
ret_list.append(d_specular)
ret_list.append(buffers.d_specular_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_roughness in buffers.d_roughness_list[i]:
ret_list.append(d_roughness)
ret_list.append(buffers.d_roughness_uv_scale_list[i])
if buffers.d_generic_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_generic in buffers.d_generic_list[i]:
ret_list.append(d_generic)
ret_list.append(buffers.d_generic_uv_scale_list[i])
if buffers.d_normal_map_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_normal_map in buffers.d_normal_map_list[i]:
ret_list.append(d_normal_map)
ret_list.append(buffers.d_normal_map_uv_scale_list[i])
ret_list.append(None) # compute_specular_lighting
ret_list.append(None) # two sided
ret_list.append(None) # use_vertex_color
num_area_lights = len(ctx.area_lights)
for i in range(num_area_lights):
ret_list.append(None) # shape id
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(buffers.d_intensity_list[i]))
ret_list.append(None) # two_sided
ret_list.append(None) # directly_visible
if ctx.envmap is not None:
ret_list.append(None) # num_levels
for d_values in buffers.d_envmap_values:
ret_list.append(d_values)
ret_list.append(buffers.d_envmap_uv_scale)
ret_list.append(None) # env_to_world
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(buffers.d_world_to_env))
ret_list.append(None) # sample_cdf_ys
ret_list.append(None) # sample_cdf_xs
ret_list.append(None) # pdf_norm
ret_list.append(None) # directly_visible
else:
ret_list.append(None)
ret_list.append(None) # num samples
ret_list.append(None) # num bounces
ret_list.append(None) # num channels
for _ in range(ctx.num_channel_args):
ret_list.append(None) # channel
ret_list.append(None) # sampler type
ret_list.append(None) # use_primary_edge_sampling
ret_list.append(None) # use_secondary_edge_sampling
ret_list.append(None) # sample_pixel_center
return ret_list
def backward(grad_img):
camera = ctx.camera
scene = ctx.scene
options = ctx.options
with tf.device(pyredner.get_device_name()):
if camera.use_look_at:
d_position = tf.zeros(3, dtype=tf.float32)
d_look_at = tf.zeros(3, dtype=tf.float32)
d_up = tf.zeros(3, dtype=tf.float32)
d_cam_to_world = None
d_world_to_cam = None
else:
d_position = None
d_look_at = None
d_up = None
d_cam_to_world = tf.zeros([4, 4], dtype=tf.float32)
d_world_to_cam = tf.zeros([4, 4], dtype=tf.float32)
d_intrinsic_mat_inv = tf.zeros([3, 3], dtype=tf.float32)
d_intrinsic_mat = tf.zeros([3, 3], dtype=tf.float32)
if camera.use_look_at:
d_camera = redner.DCamera(
redner.float_ptr(pyredner.data_ptr(d_position)),
redner.float_ptr(pyredner.data_ptr(d_look_at)),
redner.float_ptr(pyredner.data_ptr(d_up)),
redner.float_ptr(0), # cam_to_world
redner.float_ptr(0), # world_to_cam
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat)))
else:
d_camera = redner.DCamera(
redner.float_ptr(0), redner.float_ptr(0),
redner.float_ptr(0),
redner.float_ptr(pyredner.data_ptr(d_cam_to_world)),
redner.float_ptr(pyredner.data_ptr(d_world_to_cam)),
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat)))
d_vertices_list = []
d_uvs_list = []
d_normals_list = []
d_colors_list = []
d_shapes = []
with tf.device(pyredner.get_device_name()):
for i, shape in enumerate(ctx.shapes):
num_vertices = shape.num_vertices
d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32)
d_uvs = tf.zeros([num_vertices, 2],
dtype=tf.float32) if shape.has_uvs() else None
d_normals = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_normals() else None
d_colors = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_colors() else None
d_vertices_list.append(d_vertices)
d_uvs_list.append(d_uvs)
d_normals_list.append(d_normals)
d_colors_list.append(d_colors)
d_shapes.append(redner.DShape(\
redner.float_ptr(pyredner.data_ptr(d_vertices)),
redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_colors) if d_colors is not None else 0)))
d_diffuse_list = []
d_specular_list = []
d_roughness_list = []
d_normal_map_list = []
d_diffuse_uv_scale_list = []
d_specular_uv_scale_list = []
d_roughness_uv_scale_list = []
d_generic_list = []
d_generic_uv_scale_list = []
d_normal_map_uv_scale_list = []
d_materials = []
with tf.device(pyredner.get_device_name()):
for material in ctx.materials:
if material.get_diffuse_size(0)[0] == 0:
d_diffuse = [tf.zeros(3, dtype=tf.float32)]
else:
d_diffuse = []
for l in range(material.get_diffuse_levels()):
diffuse_size = material.get_diffuse_size(l)
d_diffuse.append(\
tf.zeros([diffuse_size[1],
diffuse_size[0],
3], dtype=tf.float32))
if material.get_specular_size(0)[0] == 0:
d_specular = [tf.zeros(3, dtype=tf.float32)]
else:
d_specular = []
for l in range(material.get_specular_levels()):
specular_size = material.get_specular_size(l)
d_specular.append(\
tf.zeros([specular_size[1],
specular_size[0],
3], dtype=tf.float32))
if material.get_roughness_size(0)[0] == 0:
d_roughness = [tf.zeros(1, dtype=tf.float32)]
else:
d_roughness = []
for l in range(material.get_roughness_levels()):
roughness_size = material.get_roughness_size(l)
d_roughness.append(\
tf.zeros([roughness_size[1],
roughness_size[0],
1], dtype=tf.float32))
# HACK: tensorflow's eager mode uses a cache to store scalar
# constants to avoid memory copy. If we pass scalar tensors
# into the C++ code and modify them, we would corrupt the
# cache, causing incorrect result in future scalar constant
# creations. Thus we force tensorflow to copy by plusing a zero.
# (also see https://github.com/tensorflow/tensorflow/issues/11186
# for more discussion regarding copying tensors)
if d_roughness[0].shape.num_elements() == 1:
d_roughness[0] = d_roughness[0] + 0
if material.get_generic_levels() == 0:
d_generic = None
else:
d_generic = []
for l in range(material.get_generic_levels()):
generic_size = material.get_generic_size(l)
d_generic.append(\
tf.zeros([generic_size[2],
generic_size[1],
generic_size[0]], dtype=tf.float32))
if material.get_normal_map_levels() == 0:
d_normal_map = None
else:
d_normal_map = []
for l in range(material.get_normal_map_levels()):
normal_map_size = material.get_normal_map_size(l)
d_normal_map.append(\
tf.zeros([normal_map_size[1],
normal_map_size[0],
3], dtype=tf.float32))
d_diffuse_list.append(d_diffuse)
d_specular_list.append(d_specular)
d_roughness_list.append(d_roughness)
d_generic_list.append(d_generic)
d_normal_map_list.append(d_normal_map)
d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32)
d_specular_uv_scale = tf.zeros([2], dtype=tf.float32)
d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32)
if d_generic is None:
d_generic_uv_scale = None
else:
d_generic_uv_scale = tf.zeros([2], dtype=tf.float32)
if d_normal_map is None:
d_normal_map_uv_scale = None
else:
d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32)
d_diffuse_uv_scale_list.append(d_diffuse_uv_scale)
d_specular_uv_scale_list.append(d_specular_uv_scale)
d_roughness_uv_scale_list.append(d_roughness_uv_scale)
d_generic_uv_scale_list.append(d_generic_uv_scale)
d_normal_map_uv_scale_list.append(d_normal_map_uv_scale)
if len(d_diffuse[0].shape) == 1:
d_diffuse_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(d_diffuse[0]))],
[0],
[0],
3,
redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale)))
else:
d_diffuse_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_diffuse],
[x.shape[1] for x in d_diffuse],
[x.shape[0] for x in d_diffuse],
3,
redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale)))
if len(d_specular[0].shape) == 1:
d_specular_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(d_specular[0]))],
[0],
[0],
3,
redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale)))
else:
d_specular_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_specular],
[x.shape[1] for x in d_specular],
[x.shape[0] for x in d_specular],
3,
redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale)))
if len(d_roughness[0].shape) == 1:
d_roughness_tex = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(d_roughness[0]))],
[0],
[0],
1,
redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale)))
else:
d_roughness_tex = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_roughness],
[x.shape[1] for x in d_roughness],
[x.shape[0] for x in d_roughness],
1,
redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale)))
if d_generic is None:
d_generic_tex = redner.TextureN(\
[], [], [], 0, redner.float_ptr(0))
else:
d_generic_tex = redner.TextureN(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_generic],
[x.shape[1] for x in d_generic],
[x.shape[0] for x in d_generic],
d_generic[0].shape[2],
redner.float_ptr(pyredner.data_ptr(d_generic_uv_scale)))
if d_normal_map is None:
d_normal_map = redner.Texture3(\
[], [], [], 0, redner.float_ptr(0))
else:
d_normal_map = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_normal_map],
[x.shape[1] for x in d_normal_map],
[x.shape[0] for x in d_normal_map],
3,
redner.float_ptr(pyredner.data_ptr(d_normal_map_uv_scale)))
d_materials.append(redner.DMaterial(\
d_diffuse_tex, d_specular_tex, d_roughness_tex,
d_generic_tex, d_normal_map))
d_intensity_list = []
d_area_lights = []
with tf.device(pyredner.get_device_name()):
for light in ctx.area_lights:
d_intensity = tf.zeros(3, dtype=tf.float32)
d_intensity_list.append(d_intensity)
d_area_lights.append(\
redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity))))
d_envmap = None
if ctx.envmap is not None:
envmap = ctx.envmap
with tf.device(pyredner.get_device_name()):
d_envmap_values = []
for l in range(envmap.get_levels()):
size = envmap.get_size(l)
d_envmap_values.append(\
tf.zeros([size[1],
size[0],
3], dtype=tf.float32))
d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32)
d_world_to_env = tf.zeros([4, 4], dtype=tf.float32)
d_envmap_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_envmap_values],
[x.shape[1] for x in d_envmap_values],
[x.shape[0] for x in d_envmap_values],
3,
redner.float_ptr(pyredner.data_ptr(d_envmap_uv_scale)))
d_envmap = redner.DEnvironmentMap(
d_envmap_tex,
redner.float_ptr(pyredner.data_ptr(d_world_to_env)))
d_scene = redner.DScene(d_camera, d_shapes, d_materials, d_area_lights,
d_envmap, pyredner.get_use_gpu(),
pyredner.get_gpu_device_id())
if not get_use_correlated_random_number():
# Decod_uple the forward/backward random numbers by adding a big prime number
options.seed += 1000003
start = time.time()
options.num_samples = ctx.num_samples[1]
with tf.device(pyredner.get_device_name()):
grad_img = tf.identity(grad_img)
redner.render(
scene,
options,
redner.float_ptr(0), # rendered_image
redner.float_ptr(pyredner.data_ptr(grad_img)),
d_scene,
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if print_timing:
print('Backward pass, time: %.5f s' % time_elapsed)
# # For debugging
# pyredner.imwrite(grad_img, 'grad_img.exr')
# grad_img = tf.ones([256, 256, 3], dtype=tf.float32)
# debug_img = tf.zeros([256, 256, 3], dtype=tf.float32)
# redner.render(scene, options,
# redner.float_ptr(0),
# redner.float_ptr(pyredner.data_ptr(grad_img)),
# d_scene,
# redner.float_ptr(pyredner.data_ptr(debug_img)))
# pyredner.imwrite(debug_img, 'debug.exr')
# pyredner.imwrite(-debug_img, 'debug_.exr')
# exit()
ret_list = []
ret_list.append(None) # seed
ret_list.append(None) # num_shapes
ret_list.append(None) # num_materials
ret_list.append(None) # num_lights
if camera.use_look_at:
ret_list.append(d_position)
ret_list.append(d_look_at)
ret_list.append(d_up)
ret_list.append(None) # cam_to_world
ret_list.append(None) # world_to_cam
else:
ret_list.append(None) # pos
ret_list.append(None) # look
ret_list.append(None) # up
ret_list.append(d_cam_to_world)
ret_list.append(d_world_to_cam)
ret_list.append(d_intrinsic_mat_inv)
ret_list.append(d_intrinsic_mat)
ret_list.append(None) # clip near
ret_list.append(None) # resolution
ret_list.append(None) # camera_type
num_shapes = len(ctx.shapes)
for i in range(num_shapes):
ret_list.append(d_vertices_list[i])
ret_list.append(None) # indices
ret_list.append(d_uvs_list[i])
ret_list.append(d_normals_list[i])
ret_list.append(None) # uv_indices
ret_list.append(None) # normal_indices
ret_list.append(d_colors_list[i])
ret_list.append(None) # material id
ret_list.append(None) # light id
num_materials = len(ctx.materials)
for i in range(num_materials):
ret_list.append(None) # num_levels
for d_diffuse in d_diffuse_list[i]:
ret_list.append(d_diffuse)
ret_list.append(d_diffuse_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_specular in d_specular_list[i]:
ret_list.append(d_specular)
ret_list.append(d_specular_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_roughness in d_roughness_list[i]:
ret_list.append(d_roughness)
ret_list.append(d_roughness_uv_scale_list[i])
if d_generic_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_generic in d_generic_list[i]:
ret_list.append(d_generic)
ret_list.append(d_generic_uv_scale_list[i])
if d_normal_map_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_normal_map in d_normal_map_list[i]:
ret_list.append(d_normal_map)
ret_list.append(d_normal_map_uv_scale_list[i])
ret_list.append(None) # compute_specular_lighting
ret_list.append(None) # two sided
ret_list.append(None) # use_vertex_color
num_area_lights = len(ctx.area_lights)
for i in range(num_area_lights):
ret_list.append(None) # shape id
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(d_intensity_list[i]))
ret_list.append(None) # two sided
if ctx.envmap is not None:
ret_list.append(None) # num_levels
for d_values in d_envmap_values:
ret_list.append(d_values)
ret_list.append(d_envmap_uv_scale)
ret_list.append(None) # env_to_world
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(d_world_to_env))
ret_list.append(None) # sample_cdf_ys
ret_list.append(None) # sample_cdf_xs
ret_list.append(None) # pdf_norm
else:
ret_list.append(None)
ret_list.append(None) # num samples
ret_list.append(None) # num bounces
ret_list.append(None) # num channels
for _ in range(ctx.num_channels):
ret_list.append(None) # channel
ret_list.append(None) # sampler type
ret_list.append(None) # use_primary_edge_sampling
ret_list.append(None) # use_secondary_edge_sampling
ret_list.append(None) # sample_pixel_center
return ret_list
