def generate_mipmap(self):
texels = self._texels
if len(texels.shape) >= 2:
with tf.device(pyredner.get_device_name()):
# Build a mipmap for texels
width = max(texels.shape[0], texels.shape[1])
num_levels = math.ceil(math.log(width, 2) + 1)
mipmap = tf.broadcast_to(texels, [num_levels, *texels.shape])
if len(mipmap.shape) == 3:
mipmap = tf.expand_dims(mipmap, axis=-1)
num_channels = mipmap.shape[-1]
"""NOTE: conv2d kernel axes
torch: (outchannels, in_channels / groups, kH, kW)
tf: [filter_height, filter_width, in_channels, out_channels]
"""
box_filter = tf.ones([2, 2, num_channels, 1],
dtype=tf.float32) / 4.0
# (TF) [batch, in_height, in_width, in_channels], i.e. NHWC
base_level = tf.transpose(tf.expand_dims(texels, axis=0),
perm=[0, 3, 1, 2])
mipmap = [base_level]
prev_lvl = base_level
for l in range(1, num_levels):
dilation_size = 2**(l - 1)
# Pad for circular boundary condition
# This is slow. The hope is at some point Tensorflow will support
# circular boundary condition for conv2d
desired_height = prev_lvl.shape[2] + dilation_size
while prev_lvl.shape[2] < desired_height:
prev_lvl = tf.concat([
prev_lvl, prev_lvl[:, :, 0:(desired_height -
prev_lvl.shape[2])]
], 2)
desired_width = prev_lvl.shape[3] + dilation_size
while prev_lvl.shape[3] < desired_width:
prev_lvl = tf.concat(
[prev_lvl, prev_lvl[:, :, :, 0:dilation_size]], 3)
prev_lvl = tf.transpose(prev_lvl, perm=[0, 2, 3, 1])
current_lvl = tf.nn.depthwise_conv2d(
prev_lvl,
box_filter, # [filter_height, filter_width, in_channels, out_channels]
dilations=[dilation_size, dilation_size],
strides=[1, 1, 1, 1],
padding="VALID", # No padding
data_format="NHWC")
current_lvl = tf.transpose(current_lvl, [0, 3, 1, 2])
mipmap.append(current_lvl)
prev_lvl = current_lvl
mipmap = tf.concat(mipmap, 0)
# Convert from NCHW to NHWC
mipmap = tf.transpose(mipmap, perm=[0, 2, 3, 1])
texels = mipmap
self.mipmap = texels
def generate_envmap_pdf(self):
assert (tf.executing_eagerly())
values = self.values
with tf.device(pyredner.get_device_name()):
# Build sampling table
luminance = 0.212671 * values.texels[:, :, 0] + \
0.715160 * values.texels[:, :, 1] + \
0.072169 * values.texels[:, :, 2]
# For each y, compute CDF over x
sample_cdf_xs_ = tf.cumsum(luminance, axis=1)
y_weight = tf.sin(
math.pi *
(tf.cast(tf.range(luminance.shape[0]), tf.float32) + 0.5) /
float(luminance.shape[0]))
# Compute CDF for x
sample_cdf_ys_ = tf.cumsum(sample_cdf_xs_[:, -1] * y_weight,
axis=0)
pdf_norm = (luminance.shape[0] * luminance.shape[1]) / \
(sample_cdf_ys_[-1] * (2 * math.pi * math.pi))
# Normalize to [0, 1)
sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \
tf.math.maximum(
sample_cdf_xs_[
:,
(luminance.shape[1] - 1):luminance.shape[1]],
1e-8 * tf.convert_to_tensor(np.ones((sample_cdf_xs_.shape[0], 1)), dtype=tf.float32)
)
sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \
tf.math.maximum(sample_cdf_ys_[-1], tf.constant([1e-8]))
self.sample_cdf_ys = sample_cdf_ys
self.sample_cdf_xs = sample_cdf_xs
self.pdf_norm = pdf_norm
def serialize_texture(texture, args):
if texture is None:
args.append(tf.constant(0))
return
args.append(tf.constant(len(texture.mipmap)))
with tf.device(pyredner.get_device_name()):
for mipmap in texture.mipmap:
args.append(tf.identity(mipmap))
args.append(tf.identity(texture.uv_scale))
def forward(seed: int, *args):
"""
Forward rendering pass: given a serialized scene and output an image.
"""
args_ctx = unpack_args(seed, args)
area_lights = args_ctx.area_lights
camera = args_ctx.camera
channels = args_ctx.channels
envmap = args_ctx.envmap
materials = args_ctx.materials
num_samples = args_ctx.num_samples
options = args_ctx.options
resolution = args_ctx.resolution
viewport = args_ctx.viewport
scene = args_ctx.scene
shapes = args_ctx.shapes
num_channel_args = args_ctx.num_channel_args
num_channels = redner.compute_num_channels(
channels, scene.max_generic_texture_dimension)
with tf.device(pyredner.get_device_name()):
img_height = viewport[2] - viewport[0]
img_width = viewport[3] - viewport[1]
rendered_image = tf.zeros(shape=[img_height, img_width, num_channels],
dtype=tf.float32)
start = time.time()
redner.render(
scene,
options,
redner.float_ptr(pyredner.data_ptr(rendered_image)),
redner.float_ptr(0), # d_rendered_image
None, # d_scene
redner.float_ptr(0), # translational_gradient_image
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if get_print_timing():
print('Forward pass, time: %.5f s' % time_elapsed)
ctx = Context()
ctx.camera = camera
ctx.shapes = shapes
ctx.materials = materials
ctx.area_lights = area_lights
ctx.envmap = envmap
ctx.scene = scene
ctx.options = options
ctx.num_samples = num_samples
ctx.num_channel_args = num_channel_args
ctx.args = args # important to avoid GC on tf tensors
return rendered_image, ctx
def compute_uvs(vertices, indices, print_progress = True):
"""
Compute UV coordinates of a given mesh using a charting algorithm
with least square conformal mapping. This calls the `xatlas <https://github.com/jpcy/xatlas>`_ library.
Args
====
vertices: tf.Tensor
3D position of vertices
float32 tensor with size num_vertices x 3
indices: tf.Tensor
vertex indices of triangle faces.
int32 tensor with size num_triangles x 3
Returns
=======
tf.Tensor
uv vertices pool, float32 Tensor with size num_uv_vertices x 3
tf.Tensor
uv indices, int32 Tensor with size num_triangles x 3
"""
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
vertices = tf.identity(vertices)
indices = tf.identity(indices)
uv_trimesh = redner.UVTriMesh(redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(0),
redner.int_ptr(0),
int(vertices.shape[0]),
0,
int(indices.shape[0]))
atlas = redner.TextureAtlas()
num_uv_vertices = redner.automatic_uv_map([uv_trimesh], atlas, print_progress)[0]
uvs = tf.zeros([num_uv_vertices, 2], dtype=tf.float32)
uv_indices = tf.zeros_like(indices)
uv_trimesh.uvs = redner.float_ptr(pyredner.data_ptr(uvs))
uv_trimesh.uv_indices = redner.int_ptr(pyredner.data_ptr(uv_indices))
uv_trimesh.num_uv_vertices = num_uv_vertices
redner.copy_texture_atlas(atlas, [uv_trimesh])
with tf.device(pyredner.get_device_name()):
vertices = tf.identity(vertices)
indices = tf.identity(indices)
uvs = tf.identity(uvs)
uv_indices = tf.identity(uv_indices)
return uvs, uv_indices
def __init__(self, values, env_to_world=tf.eye(4, 4)):
assert (tf.executing_eagerly())
# Convert to constant texture if necessary
if tf.is_tensor(values):
values = pyredner.Texture(values)
# assert(values.texels.is_contiguous())
assert (values.texels.dtype == tf.float32)
with tf.device(pyredner.get_device_name()):
# Build sampling table
luminance = 0.212671 * values.texels[:, :, 0] + \
0.715160 * values.texels[:, :, 1] + \
0.072169 * values.texels[:, :, 2]
# For each y, compute CDF over x
sample_cdf_xs_ = tf.cumsum(luminance, axis=1)
y_weight = tf.sin(
math.pi *
(tf.cast(tf.range(luminance.shape[0].value), tf.float32) + 0.5)
/ float(luminance.shape[0].value))
# Compute CDF for x
sample_cdf_ys_ = tf.cumsum(sample_cdf_xs_[:, -1] * y_weight,
axis=0)
pdf_norm = (luminance.shape[0].value * luminance.shape[1].value) / \
(sample_cdf_ys_[-1] * (2 * math.pi * math.pi))
# Normalize to [0, 1)
sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \
tf.math.maximum(
sample_cdf_xs_[
:,
(luminance.shape[1].value - 1):luminance.shape[1].value],
1e-8 * tf.convert_to_tensor(np.ones((sample_cdf_xs_.shape[0], 1)), dtype=tf.float32)
)
sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \
tf.math.maximum(sample_cdf_ys_[-1], tf.constant([1e-8]))
self.values = values
self.sample_cdf_ys = sample_cdf_ys
self.sample_cdf_xs = sample_cdf_xs
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
self.pdf_norm = pdf_norm.cpu()
env_to_world = tf.identity(env_to_world).cpu()
self.env_to_world = env_to_world
self.world_to_env = tf.linalg.inv(env_to_world)
def generate_quad_light(position: tf.Tensor, look_at: tf.Tensor,
size: tf.Tensor, intensity: tf.Tensor):
"""
Generate a pyredner.Object that is a quad light source.
Args
====
position: tf.Tensor
1-d tensor of size 3
look_at: tf.Tensor
1-d tensor of size 3
size: tf.Tensor
1-d tensor of size 2
intensity: tf.Tensor
1-d tensor of size 3
Returns
=======
pyredner.Object
quad light source
"""
d = look_at - position
d = d / tf.norm(d)
# ONB -- generate two axes that are orthogonal to d
a = 1 / (1 + d[2])
b = -d[0] * d[1] * a
x = tf.where(d[2] < (-1 + 1e-6), tf.constant([0.0, -1.0, 0.0]),
tf.stack([1 - d[0] * d[0] * a, b, -d[0]]))
y = tf.where(d[2] < (-1 + 1e-6), tf.constant([-1.0, 0.0, 0.0]),
tf.stack([b, 1 - d[1] * d[1] * a, -d[1]]))
v0 = position - x * size[0] * 0.5 - y * size[1] * 0.5
v1 = position + x * size[0] * 0.5 - y * size[1] * 0.5
v2 = position - x * size[0] * 0.5 + y * size[1] * 0.5
v3 = position + x * size[0] * 0.5 + y * size[1] * 0.5
with tf.device(pyredner.get_device_name()):
vertices = tf.stack((v0, v1, v2, v3), axis=0)
indices = tf.constant([[0, 1, 2], [1, 3, 2]], dtype=tf.int32)
m = pyredner.Material(diffuse_reflectance=tf.constant([0.0, 0.0, 0.0]))
return pyredner.Object(vertices=vertices,
indices=indices,
material=m,
light_intensity=intensity)
def generate_mipmap(self):
texels = self._texels
if len(texels.shape) >= 2:
with tf.device(pyredner.get_device_name()):
# Build a mipmap for texels
width = max(texels.shape[0], texels.shape[1])
num_levels = min(math.ceil(math.log(width, 2) + 1), 8)
num_channels = texels.shape[2]
box_filter = tf.ones([2, 2, num_channels, 1],
dtype=tf.float32) / 4.0
mipmap = [texels]
# HWC -> NHWC
base_level = tf.expand_dims(texels, axis=0)
prev_lvl = base_level
for l in range(1, num_levels):
# Pad for circular boundary condition
prev_lvl = tf.concat([prev_lvl, prev_lvl[:, 0:1, :, :]], 1)
prev_lvl = tf.concat([prev_lvl, prev_lvl[:, :, 0:1, :]], 2)
# Convolve with a box filter
current_lvl = tf.nn.depthwise_conv2d(
prev_lvl,
box_filter, # [filter_height, filter_width, in_channels, out_channels]
strides=[1, 1, 1, 1],
padding='VALID', # No padding
data_format='NHWC')
# Downsample
# tf.image.resize is too slow, so we use average pooling
current_lvl = tf.nn.avg_pool2d(current_lvl,
ksize=2,
strides=2,
padding='SAME')
mipmap.append(tf.squeeze(current_lvl, axis=0))
prev_lvl = current_lvl
else:
mipmap = [texels]
self.mipmap = mipmap
def unpack_args(seed,
args,
use_primary_edge_sampling=None,
use_secondary_edge_sampling=None):
"""
Given a list of serialized scene arguments, unpack
all information into a Context.
"""
# Unpack arguments
current_index = 0
num_shapes = int(args[current_index])
current_index += 1
num_materials = int(args[current_index])
current_index += 1
num_lights = int(args[current_index])
current_index += 1
# Camera arguments
cam_position = args[current_index]
current_index += 1
cam_look_at = args[current_index]
current_index += 1
cam_up = args[current_index]
current_index += 1
cam_to_world = args[current_index]
current_index += 1
world_to_cam = args[current_index]
current_index += 1
intrinsic_mat_inv = args[current_index]
current_index += 1
intrinsic_mat = args[current_index]
current_index += 1
clip_near = float(args[current_index])
current_index += 1
resolution = args[current_index].numpy() # Tuple[int, int]
current_index += 1
viewport = args[current_index].numpy() # Tuple[int, int, int, int]
current_index += 1
camera_type = RednerCameraType.asCameraType(
args[current_index]) # FIXME: Map to custom type
current_index += 1
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if is_empty_tensor(cam_to_world):
camera = redner.Camera(
resolution[1],
resolution[0],
redner.float_ptr(pyredner.data_ptr(cam_position)),
redner.float_ptr(pyredner.data_ptr(cam_look_at)),
redner.float_ptr(pyredner.data_ptr(cam_up)),
redner.float_ptr(0), # cam_to_world
redner.float_ptr(0), # world_to_cam
redner.float_ptr(pyredner.data_ptr(intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(intrinsic_mat)),
clip_near,
camera_type,
redner.Vector2i(viewport[1], viewport[0]),
redner.Vector2i(viewport[3], viewport[2]))
else:
camera = redner.Camera(
resolution[1], resolution[0], redner.float_ptr(0),
redner.float_ptr(0), redner.float_ptr(0),
redner.float_ptr(pyredner.data_ptr(cam_to_world)),
redner.float_ptr(pyredner.data_ptr(world_to_cam)),
redner.float_ptr(pyredner.data_ptr(intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(intrinsic_mat)), clip_near,
camera_type, redner.Vector2i(viewport[1], viewport[0]),
redner.Vector2i(viewport[3], viewport[2]))
with tf.device(pyredner.get_device_name()):
shapes = []
for i in range(num_shapes):
vertices = args[current_index]
current_index += 1
indices = args[current_index]
current_index += 1
uvs = args[current_index]
current_index += 1
normals = args[current_index]
current_index += 1
uv_indices = args[current_index]
current_index += 1
normal_indices = args[current_index]
current_index += 1
colors = args[current_index]
current_index += 1
material_id = int(args[current_index])
current_index += 1
light_id = int(args[current_index])
current_index += 1
shapes.append(redner.Shape(\
redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(pyredner.data_ptr(uvs) if not is_empty_tensor(uvs) else 0),
redner.float_ptr(pyredner.data_ptr(normals) if not is_empty_tensor(normals) else 0),
redner.int_ptr(pyredner.data_ptr(uv_indices) if not is_empty_tensor(uv_indices) else 0),
redner.int_ptr(pyredner.data_ptr(normal_indices) if not is_empty_tensor(normal_indices) else 0),
redner.float_ptr(pyredner.data_ptr(colors) if not is_empty_tensor(colors) else 0),
int(vertices.shape[0]),
int(uvs.shape[0]) if not is_empty_tensor(uvs) else 0,
int(normals.shape[0]) if not is_empty_tensor(normals) else 0,
int(indices.shape[0]),
material_id,
light_id))
materials = []
with tf.device(pyredner.get_device_name()):
for i in range(num_materials):
num_levels = int(args[current_index])
current_index += 1
diffuse_reflectance = []
for j in range(num_levels):
diffuse_reflectance.append(args[current_index])
current_index += 1
diffuse_uv_scale = args[current_index]
current_index += 1
num_levels = int(args[current_index])
current_index += 1
specular_reflectance = []
for j in range(num_levels):
specular_reflectance.append(args[current_index])
current_index += 1
specular_uv_scale = args[current_index]
current_index += 1
num_levels = int(args[current_index])
current_index += 1
roughness = []
for j in range(num_levels):
roughness.append(args[current_index])
current_index += 1
roughness_uv_scale = args[current_index]
current_index += 1
num_levels = int(args[current_index])
current_index += 1
generic_texture = []
if num_levels > 0:
for j in range(num_levels):
generic_texture.append(args[current_index])
current_index += 1
generic_uv_scale = args[current_index]
current_index += 1
else:
generic_uv_scale = None
num_levels = int(args[current_index])
current_index += 1
normal_map = []
if num_levels > 0:
for j in range(num_levels):
normal_map.append(args[current_index])
current_index += 1
normal_map_uv_scale = args[current_index]
current_index += 1
else:
normal_map_uv_scale = None
compute_specular_lighting = bool(args[current_index])
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
use_vertex_color = bool(args[current_index])
current_index += 1
if get_tensor_dimension(diffuse_reflectance[0]) == 1:
diffuse_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(diffuse_reflectance[0]))],
[0],
[0],
3, redner.float_ptr(pyredner.data_ptr(diffuse_uv_scale)))
else:
assert (get_tensor_dimension(diffuse_reflectance[0]) == 3)
diffuse_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in diffuse_reflectance],
[x.shape[1] for x in diffuse_reflectance],
[x.shape[0] for x in diffuse_reflectance],
3,
redner.float_ptr(pyredner.data_ptr(diffuse_uv_scale)))
if get_tensor_dimension(specular_reflectance[0]) == 1:
specular_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(specular_reflectance[0]))],
[0],
[0],
3, redner.float_ptr(pyredner.data_ptr(specular_uv_scale)))
else:
assert (get_tensor_dimension(specular_reflectance[0]) == 3)
specular_reflectance = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in specular_reflectance],
[x.shape[1] for x in specular_reflectance],
[x.shape[0] for x in specular_reflectance],
3,
redner.float_ptr(pyredner.data_ptr(specular_uv_scale)))
if get_tensor_dimension(roughness[0]) == 1:
roughness = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(roughness[0]))],
[0],
[0],
1, redner.float_ptr(pyredner.data_ptr(roughness_uv_scale)))
else:
assert (get_tensor_dimension(roughness[0]) == 3)
roughness = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in roughness],
[x.shape[1] for x in roughness],
[x.shape[0] for x in roughness],
3,
redner.float_ptr(pyredner.data_ptr(roughness_uv_scale)))
if len(generic_texture) > 0:
generic_texture = redner.TextureN(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in generic_texture],
[x.shape[1] for x in generic_texture],
[x.shape[0] for x in generic_texture],
generic_texture[0].shape[2],
redner.float_ptr(pyredner.data_ptr(generic_uv_scale)))
else:
generic_texture = redner.TextureN(\
[], [], [], 0, redner.float_ptr(0))
if len(normal_map) > 0:
normal_map = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in normal_map],
[x.shape[1] for x in normal_map],
[x.shape[0] for x in normal_map],
normal_map[0].shape[2],
redner.float_ptr(pyredner.data_ptr(normal_map_uv_scale)))
else:
normal_map = redner.Texture3(\
[], [], [], 0, redner.float_ptr(0))
materials.append(redner.Material(\
diffuse_reflectance,
specular_reflectance,
roughness,
generic_texture,
normal_map,
compute_specular_lighting,
two_sided,
use_vertex_color))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
area_lights = []
for i in range(num_lights):
shape_id = int(args[current_index])
current_index += 1
intensity = args[current_index]
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
directly_visible = bool(args[current_index])
current_index += 1
area_lights.append(
redner.AreaLight(
shape_id, redner.float_ptr(pyredner.data_ptr(intensity)),
two_sided, directly_visible))
envmap = None
if not is_empty_tensor(args[current_index]):
num_levels = args[current_index]
current_index += 1
values = []
for j in range(num_levels):
values.append(args[current_index])
current_index += 1
envmap_uv_scale = args[current_index]
current_index += 1
env_to_world = args[current_index]
current_index += 1
world_to_env = args[current_index]
current_index += 1
sample_cdf_ys = args[current_index]
current_index += 1
sample_cdf_xs = args[current_index]
current_index += 1
pdf_norm = float(args[current_index])
current_index += 1
directly_visible = bool(args[current_index])
current_index += 1
assert isinstance(pdf_norm, float)
with tf.device(pyredner.get_device_name()):
sample_cdf_ys = redner.float_ptr(pyredner.data_ptr(sample_cdf_ys))
sample_cdf_xs = redner.float_ptr(pyredner.data_ptr(sample_cdf_xs))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
env_to_world = redner.float_ptr(pyredner.data_ptr(env_to_world))
world_to_env = redner.float_ptr(pyredner.data_ptr(world_to_env))
with tf.device(pyredner.get_device_name()):
values = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in values],
[x.shape[1] for x in values], # width
[x.shape[0] for x in values], # height
3, # channels
redner.float_ptr(pyredner.data_ptr(envmap_uv_scale)))
envmap = redner.EnvironmentMap(\
values,
env_to_world,
world_to_env,
sample_cdf_ys,
sample_cdf_xs,
pdf_norm,
directly_visible)
else:
current_index += 1
# Options
num_samples = args[current_index]
current_index += 1
if len(num_samples.shape) == 0 or num_samples.shape[0] == 1:
num_samples = int(num_samples)
else:
assert (num_samples.shape[0] == 2)
num_samples = (int(num_samples[0]), int(num_samples[1]))
max_bounces = int(args[current_index])
current_index += 1
num_channel_args = int(args[current_index])
current_index += 1
channels = []
for _ in range(num_channel_args):
ch = args[current_index]
ch = RednerChannels.asChannel(ch)
channels.append(ch)
current_index += 1
sampler_type = args[current_index]
sampler_type = RednerSamplerType.asSamplerType(sampler_type)
current_index += 1
use_primary_edge_sampling = args[current_index]
current_index += 1
use_secondary_edge_sampling = args[current_index]
current_index += 1
sample_pixel_center = args[current_index]
current_index += 1
start = time.time()
scene = redner.Scene(camera, shapes, materials, area_lights, envmap,
pyredner.get_use_gpu(), pyredner.get_gpu_device_id(),
use_primary_edge_sampling,
use_secondary_edge_sampling)
time_elapsed = time.time() - start
if get_print_timing():
print('Scene construction, time: %.5f s' % time_elapsed)
# check that num_samples is a tuple
if isinstance(num_samples, int):
num_samples = (num_samples, num_samples)
options = redner.RenderOptions(seed, num_samples[0], max_bounces, channels,
sampler_type, sample_pixel_center)
ctx = Context()
ctx.channels = channels
ctx.options = options
ctx.resolution = resolution
ctx.viewport = viewport
ctx.scene = scene
ctx.camera = camera
ctx.shapes = shapes
ctx.materials = materials
ctx.area_lights = area_lights
ctx.envmap = envmap
ctx.scene = scene
ctx.options = options
ctx.num_samples = num_samples
ctx.num_channel_args = num_channel_args
return ctx
def visualize_screen_gradient(grad_img: tf.Tensor,
seed: int,
scene: pyredner.Scene,
num_samples: Union[int, Tuple[int, int]],
max_bounces: int,
channels: List = [redner.channels.radiance],
sampler_type=redner.SamplerType.independent,
use_primary_edge_sampling: bool = True,
use_secondary_edge_sampling: bool = True,
sample_pixel_center: bool = False):
"""
Given a serialized scene and output an 2-channel image,
which visualizes the derivatives of pixel color with respect to
the screen space coordinates.
Args
====
grad_img: Optional[tf.Tensor]
The "adjoint" of the backpropagation gradient. If you don't know
what this means just give None
seed: int
seed for the Monte Carlo random samplers
See serialize_scene for the explanation of the rest of the arguments.
"""
args = serialize_scene(\
scene = scene,
num_samples = num_samples,
max_bounces = max_bounces,
sampler_type = sampler_type,
channels = channels,
sample_pixel_center = sample_pixel_center)
args_ctx = unpack_args(\
seed, args, use_primary_edge_sampling, use_secondary_edge_sampling)
channels = args_ctx.channels
options = args_ctx.options
resolution = args_ctx.resolution
viewport = args_ctx.viewport
scene = args_ctx.scene
buffers = create_gradient_buffers(args_ctx)
num_channels = redner.compute_num_channels(
channels, scene.max_generic_texture_dimension)
with tf.device(pyredner.get_device_name()):
img_height = viewport[2] - viewport[0]
img_width = viewport[3] - viewport[1]
screen_gradient_image = tf.zeros(\
shape = [img_height, img_width, 2],
dtype = tf.float32)
if grad_img is not None:
assert (grad_img.shape[0] == resolution[0])
assert (grad_img.shape[1] == resolution[1])
assert (grad_img.shape[2] == num_channels)
else:
grad_img = tf.ones(\
shape = [img_height, img_width, num_channels],
dtype = tf.float32)
start = time.time()
redner.render(
scene,
options,
redner.float_ptr(0), # rendered_image
redner.float_ptr(pyredner.data_ptr(grad_img)), # d_rendered_image
buffers.d_scene,
redner.float_ptr(pyredner.data_ptr(screen_gradient_image)),
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if get_print_timing():
print('Visualize gradient, time: %.5f s' % time_elapsed)
return screen_gradient_image
def backward(grad_img):
global __ctx
ctx = __ctx
scene = ctx.scene
options = ctx.options
with tf.device(pyredner.get_device_name()):
d_position = tf.zeros(3, dtype=tf.float32)
d_look_at = tf.zeros(3, dtype=tf.float32)
d_up = tf.zeros(3, dtype=tf.float32)
d_ndc_to_cam = tf.zeros([3, 3], dtype=tf.float32)
d_cam_to_ndc = tf.zeros([3, 3], dtype=tf.float32)
d_camera = redner.DCamera(
redner.float_ptr(pyredner.data_ptr(d_position)),
redner.float_ptr(pyredner.data_ptr(d_look_at)),
redner.float_ptr(pyredner.data_ptr(d_up)),
redner.float_ptr(pyredner.data_ptr(d_ndc_to_cam)),
redner.float_ptr(pyredner.data_ptr(d_cam_to_ndc)))
d_vertices_list = []
d_uvs_list = []
d_normals_list = []
d_shapes = []
with tf.device(pyredner.get_device_name()):
for i, shape in enumerate(ctx.shapes):
num_vertices = shape.num_vertices
d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32)
d_uvs = tf.zeros([num_vertices, 2],
dtype=tf.float32) if shape.has_uvs() else None
d_normals = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_normals() else None
d_vertices_list.append(d_vertices)
d_uvs_list.append(d_uvs)
d_normals_list.append(d_normals)
d_shapes.append(redner.DShape(\
redner.float_ptr(pyredner.data_ptr(d_vertices)),
redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0)))
d_diffuse_list = []
d_specular_list = []
d_roughness_list = []
d_normal_map_list = []
d_diffuse_uv_scale_list = []
d_specular_uv_scale_list = []
d_roughness_uv_scale_list = []
d_normal_map_uv_scale_list = []
d_materials = []
with tf.device(pyredner.get_device_name()):
for material in ctx.materials:
diffuse_size = material.get_diffuse_size()
specular_size = material.get_specular_size()
roughness_size = material.get_roughness_size()
normal_map_size = material.get_normal_map_size()
if diffuse_size[0] == 0:
d_diffuse = tf.zeros(3, dtype=tf.float32)
else:
d_diffuse = tf.zeros(
[diffuse_size[2], diffuse_size[1], diffuse_size[0], 3],
dtype=tf.float32)
if specular_size[0] == 0:
d_specular = tf.zeros(3, dtype=tf.float32)
else:
d_specular = tf.zeros([
specular_size[2], specular_size[1], specular_size[0], 3
],
dtype=tf.float32)
if roughness_size[0] == 0:
d_roughness = tf.zeros(1, dtype=tf.float32)
else:
d_roughness = tf.zeros([
roughness_size[2], roughness_size[1],
roughness_size[0], 1
],
dtype=tf.float32)
# HACK: tensorflow's eager mode uses a cache to store scalar
# constants to avoid memory copy. If we pass scalar tensors
# into the C++ code and modify them, we would corrupt the
# cache, causing incorrect result in future scalar constant
# creations. Thus we force tensorflow to copy by plusing a zero
# (also see https://github.com/tensorflow/tensorflow/issues/11186
# for more discussion regarding copying tensors)
if d_roughness.shape.num_elements() == 1:
d_roughness = d_roughness + 0
if normal_map_size[0] == 0:
d_normal_map = None
else:
d_normal_map = tf.zeros([
normal_map_size[2], normal_map_size[1],
normal_map_size[0], 3
],
dtype=tf.float32)
d_diffuse_list.append(d_diffuse)
d_specular_list.append(d_specular)
d_roughness_list.append(d_roughness)
d_normal_map_list.append(d_normal_map)
d_diffuse = redner.float_ptr(pyredner.data_ptr(d_diffuse))
d_specular = redner.float_ptr(pyredner.data_ptr(d_specular))
d_roughness = redner.float_ptr(pyredner.data_ptr(d_roughness))
if normal_map_size[0] > 0:
d_normal_map = redner.float_ptr(
pyredner.data_ptr(d_normal_map))
d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32)
d_specular_uv_scale = tf.zeros([2], dtype=tf.float32)
d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32)
if normal_map_size[0] > 0:
d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32)
else:
d_normal_map_uv_scale = None
d_diffuse_uv_scale_list.append(d_diffuse_uv_scale)
d_specular_uv_scale_list.append(d_specular_uv_scale)
d_roughness_uv_scale_list.append(d_roughness_uv_scale)
d_normal_map_uv_scale_list.append(d_normal_map_uv_scale)
d_diffuse_uv_scale = redner.float_ptr(
pyredner.data_ptr(d_diffuse_uv_scale))
d_specular_uv_scale = redner.float_ptr(
pyredner.data_ptr(d_specular_uv_scale))
d_roughness_uv_scale = redner.float_ptr(
pyredner.data_ptr(d_roughness_uv_scale))
if normal_map_size[0] > 0:
d_normal_map_uv_scale = redner.float_ptr(
pyredner.data_ptr(d_normal_map_uv_scale))
d_diffuse_tex = redner.Texture3(\
d_diffuse, diffuse_size[0], diffuse_size[1], diffuse_size[2], d_diffuse_uv_scale)
d_specular_tex = redner.Texture3(\
d_specular, specular_size[0], specular_size[1], specular_size[2], d_specular_uv_scale)
d_roughness_tex = redner.Texture1(\
d_roughness, roughness_size[0], roughness_size[1], roughness_size[2], d_roughness_uv_scale)
if normal_map_size[0] > 0:
d_normal_map_tex = redner.Texture3(\
d_normal_map, normal_map_size[0], normal_map_size[1], normal_map_size[2], d_normal_map_uv_scale)
else:
d_normal_map_tex = redner.Texture3(\
redner.float_ptr(0), 0, 0, 0, redner.float_ptr(0))
d_materials.append(
redner.DMaterial(d_diffuse_tex, d_specular_tex,
d_roughness_tex, d_normal_map_tex))
d_intensity_list = []
d_area_lights = []
with tf.device(pyredner.get_device_name()):
for light in ctx.area_lights:
d_intensity = tf.zeros(3, dtype=tf.float32)
d_intensity_list.append(d_intensity)
d_area_lights.append(\
redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity))))
d_envmap = None
if ctx.envmap is not None:
envmap = ctx.envmap
size = envmap.get_size()
with tf.device(pyredner.get_device_name()):
d_envmap_values = tf.zeros([size[2], size[1], size[0], 3],
dtype=tf.float32)
d_envmap_values_ptr = redner.float_ptr(
pyredner.data_ptr(d_envmap_values))
d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32)
d_envmap_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(d_envmap_uv_scale))
d_world_to_env = tf.zeros([4, 4], dtype=tf.float32)
d_world_to_env_ptr = redner.float_ptr(
pyredner.data_ptr(d_world_to_env))
d_envmap_tex = redner.Texture3(\
d_envmap_values_ptr, size[0], size[1], size[2], d_envmap_uv_scale_ptr)
d_envmap = redner.DEnvironmentMap(d_envmap_tex, d_world_to_env_ptr)
d_scene = redner.DScene(d_camera, d_shapes, d_materials, d_area_lights,
d_envmap, pyredner.get_use_gpu(), -1)
if not get_use_correlated_random_number():
# Decod_uple the forward/backward random numbers by adding a big prime number
options.seed += 1000003
start = time.time()
options.num_samples = ctx.num_samples[1]
with tf.device(pyredner.get_device_name()):
if pyredner.get_use_gpu():
grad_img = grad_img.gpu(pyredner.get_gpu_device_id())
else:
grad_img = grad_img.cpu()
redner.render(
scene,
options,
redner.float_ptr(0), # rendered_image
redner.float_ptr(pyredner.data_ptr(grad_img)),
d_scene,
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if print_timing:
print('Backward pass, time: %.5f s' % time_elapsed)
# # For debugging
# pyredner.imwrite(grad_img, 'grad_img.exr')
# grad_img = tf.ones([256, 256, 3], dtype=tf.float32)
# debug_img = tf.zeros([256, 256, 3], dtype=tf.float32)
# redner.render(scene, options,
# redner.float_ptr(0),
# redner.float_ptr(pyredner.data_ptr(grad_img)),
# d_scene,
# redner.float_ptr(pyredner.data_ptr(debug_img)))
# pyredner.imwrite(debug_img, 'debug.exr')
# pyredner.imwrite(-debug_img, 'debug_.exr')
# exit()
ret_list = []
ret_list.append(None) # seed
ret_list.append(None) # num_shapes
ret_list.append(None) # num_materials
ret_list.append(None) # num_lights
ret_list.append(d_position)
ret_list.append(d_look_at)
ret_list.append(d_up)
ret_list.append(d_ndc_to_cam)
ret_list.append(d_cam_to_ndc)
ret_list.append(None) # clip near
ret_list.append(None) # resolution
ret_list.append(None) # camera_type
num_shapes = len(ctx.shapes)
for i in range(num_shapes):
ret_list.append(d_vertices_list[i])
ret_list.append(None) # indices
ret_list.append(d_uvs_list[i])
ret_list.append(d_normals_list[i])
ret_list.append(None) # material id
ret_list.append(None) # light id
num_materials = len(ctx.materials)
for i in range(num_materials):
ret_list.append(d_diffuse_list[i])
ret_list.append(d_diffuse_uv_scale_list[i])
ret_list.append(d_specular_list[i])
ret_list.append(d_specular_uv_scale_list[i])
ret_list.append(d_roughness_list[i])
ret_list.append(d_roughness_uv_scale_list[i])
ret_list.append(d_normal_map_list[i])
ret_list.append(d_normal_map_uv_scale_list[i])
ret_list.append(None) # two sided
num_area_lights = len(ctx.area_lights)
for i in range(num_area_lights):
ret_list.append(None) # shape id
ret_list.append(d_intensity_list[i].cpu())
ret_list.append(None) # two sided
if ctx.envmap is not None:
ret_list.append(d_envmap_values)
ret_list.append(d_envmap_uv_scale)
ret_list.append(None) # env_to_world
ret_list.append(d_world_to_env.cpu())
ret_list.append(None) # sample_cdf_ys
ret_list.append(None) # sample_cdf_xs
ret_list.append(None) # pdf_norm
else:
ret_list.append(None)
ret_list.append(None)
ret_list.append(None)
ret_list.append(None)
ret_list.append(None)
ret_list.append(None)
ret_list.append(None)
ret_list.append(None) # num samples
ret_list.append(None) # num bounces
ret_list.append(None) # num channels
for _ in range(ctx.num_channels):
ret_list.append(None) # channel
ret_list.append(None) # sampler type
ret_list.append(None) # use_primary_edge_sampling
ret_list.append(None) # use_secondary_edge_sampling
# pdb.set_trace()
return ret_list
def forward(seed: int, *args):
"""
Forward rendering pass: given a scene and output an image.
"""
global __ctx
ctx = __ctx
# Unpack arguments
current_index = 0
num_shapes = int(args[current_index])
current_index += 1
num_materials = int(args[current_index])
current_index += 1
num_lights = int(args[current_index])
current_index += 1
# Camera arguments
cam_position = args[current_index]
current_index += 1
cam_look_at = args[current_index]
current_index += 1
cam_up = args[current_index]
current_index += 1
ndc_to_cam = args[current_index]
current_index += 1
cam_to_ndc = args[current_index]
current_index += 1
clip_near = float(args[current_index])
current_index += 1
resolution = args[current_index].numpy() # Tuple[int, int]
current_index += 1
camera_type = pyredner.RednerCameraType.asCameraType(
args[current_index]) # FIXME: Map to custom type
current_index += 1
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
camera = redner.Camera(
resolution[1], resolution[0],
redner.float_ptr(pyredner.data_ptr(cam_position)),
redner.float_ptr(pyredner.data_ptr(cam_look_at)),
redner.float_ptr(pyredner.data_ptr(cam_up)),
redner.float_ptr(pyredner.data_ptr(ndc_to_cam)),
redner.float_ptr(pyredner.data_ptr(cam_to_ndc)), clip_near,
camera_type)
with tf.device(pyredner.get_device_name()):
shapes = []
for i in range(num_shapes):
vertices = args[current_index]
current_index += 1
indices = args[current_index]
current_index += 1
uvs = args[current_index]
current_index += 1
normals = args[current_index]
current_index += 1
material_id = int(args[current_index])
current_index += 1
light_id = int(args[current_index])
current_index += 1
shapes.append(redner.Shape(\
redner.float_ptr(pyredner.data_ptr(vertices)),
redner.int_ptr(pyredner.data_ptr(indices)),
redner.float_ptr(pyredner.data_ptr(uvs) if uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(normals) if normals is not None else 0),
int(vertices.shape[0]),
int(indices.shape[0]),
material_id,
light_id))
materials = []
with tf.device(pyredner.get_device_name()):
for i in range(num_materials):
diffuse_reflectance = args[current_index]
current_index += 1
diffuse_uv_scale = args[current_index]
current_index += 1
specular_reflectance = args[current_index]
current_index += 1
specular_uv_scale = args[current_index]
current_index += 1
roughness = args[current_index]
current_index += 1
roughness_uv_scale = args[current_index]
current_index += 1
normal_map = args[current_index]
current_index += 1
normal_map_uv_scale = args[current_index]
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
diffuse_reflectance_ptr = redner.float_ptr(
pyredner.data_ptr(diffuse_reflectance))
specular_reflectance_ptr = redner.float_ptr(
pyredner.data_ptr(specular_reflectance))
roughness_ptr = redner.float_ptr(pyredner.data_ptr(roughness))
if normal_map.shape[0] > 0:
normal_map_ptr = redner.float_ptr(
pyredner.data_ptr(normal_map))
diffuse_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(diffuse_uv_scale))
specular_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(specular_uv_scale))
roughness_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(roughness_uv_scale))
if normal_map.shape[0] > 0:
normal_map_uv_scale_ptr = redner.float_ptr(
pyredner.data_ptr(normal_map_uv_scale))
if get_tensor_dimension(diffuse_reflectance) == 1:
diffuse_reflectance = redner.Texture3(diffuse_reflectance_ptr,
0, 0, 0,
diffuse_uv_scale_ptr)
else:
diffuse_reflectance = redner.Texture3(\
diffuse_reflectance_ptr,
int(diffuse_reflectance.shape[2]), # width
int(diffuse_reflectance.shape[1]), # height
int(diffuse_reflectance.shape[0]), # num levels
diffuse_uv_scale_ptr)
if get_tensor_dimension(specular_reflectance) == 1:
specular_reflectance = redner.Texture3(
specular_reflectance_ptr, 0, 0, 0, specular_uv_scale_ptr)
else:
specular_reflectance = redner.Texture3(\
specular_reflectance_ptr,
int(specular_reflectance.shape[2]), # width
int(specular_reflectance.shape[1]), # height
int(specular_reflectance.shape[0]), # num levels
specular_uv_scale_ptr)
if get_tensor_dimension(roughness) == 1:
roughness = redner.Texture1(roughness_ptr, 0, 0, 0,
roughness_uv_scale_ptr)
else:
assert (get_tensor_dimension(roughness) == 4)
roughness = redner.Texture1(\
roughness_ptr,
int(roughness.shape[2]), # width
int(roughness.shape[1]), # height
int(roughness.shape[0]), # num levels
roughness_uv_scale_ptr)
if normal_map.shape[0] > 0:
normal_map = redner.Texture3(\
normal_map_ptr,
int(normal_map.shape[2]),
int(normal_map.shape[1]),
int(normal_map.shape[0]),
normal_map_uv_scale_ptr)
else:
normal_map = redner.Texture3(\
redner.float_ptr(0), 0, 0, 0, redner.float_ptr(0))
materials.append(redner.Material(\
diffuse_reflectance,
specular_reflectance,
roughness,
normal_map,
two_sided))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
area_lights = []
for i in range(num_lights):
shape_id = int(args[current_index])
current_index += 1
intensity = args[current_index]
current_index += 1
two_sided = bool(args[current_index])
current_index += 1
area_lights.append(
redner.AreaLight(
shape_id, redner.float_ptr(pyredner.data_ptr(intensity)),
two_sided))
envmap = None
if not is_empty_tensor(args[current_index]):
values = args[current_index]
current_index += 1
envmap_uv_scale = args[current_index]
current_index += 1
env_to_world = args[current_index]
current_index += 1
world_to_env = args[current_index]
current_index += 1
sample_cdf_ys = args[current_index]
current_index += 1
sample_cdf_xs = args[current_index]
current_index += 1
pdf_norm = float(args[current_index])
current_index += 1
assert isinstance(pdf_norm, float)
with tf.device(pyredner.get_device_name()):
values_ptr = redner.float_ptr(pyredner.data_ptr(values))
sample_cdf_ys = redner.float_ptr(pyredner.data_ptr(sample_cdf_ys))
sample_cdf_xs = redner.float_ptr(pyredner.data_ptr(sample_cdf_xs))
envmap_uv_scale = redner.float_ptr(
pyredner.data_ptr(envmap_uv_scale))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
env_to_world = redner.float_ptr(pyredner.data_ptr(env_to_world))
world_to_env = redner.float_ptr(pyredner.data_ptr(world_to_env))
values = redner.Texture3(
values_ptr,
int(values.shape[2]), # width
int(values.shape[1]), # height
int(values.shape[0]), # num levels
envmap_uv_scale)
envmap = redner.EnvironmentMap(\
values,
env_to_world,
world_to_env,
sample_cdf_ys,
sample_cdf_xs,
pdf_norm)
else:
current_index += 7
# Options
num_samples = int(args[current_index])
current_index += 1
max_bounces = int(args[current_index])
current_index += 1
__num_channels = int(args[current_index])
current_index += 1
channels = []
for _ in range(__num_channels):
ch = args[current_index]
ch = pyredner.RednerChannels.asChannel(ch)
channels.append(ch)
current_index += 1
sampler_type = args[current_index]
sampler_type = pyredner.RednerSamplerType.asSamplerType(sampler_type)
current_index += 1
use_primary_edge_sampling = args[current_index]
current_index += 1
use_secondary_edge_sampling = args[current_index]
current_index += 1
scene = redner.Scene(camera, shapes, materials, area_lights, envmap,
pyredner.get_use_gpu(), pyredner.get_gpu_device_id(),
use_primary_edge_sampling,
use_secondary_edge_sampling)
# check that num_samples is a tuple
if isinstance(num_samples, int):
num_samples = (num_samples, num_samples)
options = redner.RenderOptions(seed, num_samples[0], max_bounces, channels,
sampler_type)
num_channels = redner.compute_num_channels(channels)
with tf.device(pyredner.get_device_name()):
rendered_image = tf.zeros(
shape=[resolution[0], resolution[1], num_channels],
dtype=tf.float32)
start = time.time()
# pdb.set_trace()
redner.render(scene, options,
redner.float_ptr(pyredner.data_ptr(rendered_image)),
redner.float_ptr(0), None, redner.float_ptr(0))
time_elapsed = time.time() - start
if print_timing:
print('Forward pass, time: %.5f s' % time_elapsed)
# # For debugging
# debug_img = tf.zeros((256, 256, 3), dtype=tf.float32)
# redner.render(scene,
# options,
# redner.float_ptr(pyredner.data_ptr(rendered_image)),
# redner.float_ptr(0),
# None,
# redner.float_ptr(pyredner.data_ptr(debug_img)))
# pyredner.imwrite(debug_img, 'debug.png')
# exit()
# import pdb; pdb.set_trace()
ctx.shapes = shapes
ctx.materials = materials
ctx.area_lights = area_lights
ctx.envmap = envmap
ctx.scene = scene
ctx.options = options
ctx.num_samples = num_samples
ctx.num_channels = __num_channels
return rendered_image
look_at = tf.Variable([0.0, 0.0, 0.0], dtype=tf.float32)
up = tf.Variable([0.0, 1.0, 0.0], dtype=tf.float32)
fov = tf.Variable([45.0], dtype=tf.float32)
clip_near = 1e-2
resolution = (256, 256)
cam = pyredner.Camera(position=position,
look_at=look_at,
up=up,
fov=fov,
clip_near=clip_near,
resolution=resolution)
mat_perlin = pyredner.Material(diffuse_reflectance=diffuse,
specular_reflectance=specular,
roughness=roughness)
with tf.device(pyredner.get_device_name()):
mat_black = pyredner.Material(
diffuse_reflectance=tf.Variable([0.0, 0.0, 0.0], dtype=tf.float32))
materials = [mat_perlin, mat_black]
vertices = tf.Variable([[-1.5, -1.5, 0.0], [-1.5, 1.5, 0.0],
[1.5, -1.5, 0.0], [1.5, 1.5, 0.0]],
dtype=tf.float32)
indices = tf.constant([[0, 1, 2], [1, 3, 2]], dtype=tf.int32)
uvs = tf.Variable([[0.05, 0.05], [0.05, 0.95], [0.95, 0.05], [0.95, 0.95]],
dtype=tf.float32)
shape_plane = pyredner.Shape(vertices, indices, 0, uvs)
light_vertices = tf.Variable([[-1.0, -1.0, -7.0], [1.0, -1.0, -7.0],
[-1.0, 1.0, -7.0], [1.0, 1.0, -7.0]],
dtype=tf.float32)
light_indices = tf.constant([[0, 1, 2], [1, 3, 2]], dtype=tf.int32)
shape_light = pyredner.Shape(light_vertices, light_indices, 1)
def serialize_scene(scene: pyredner.Scene,
num_samples: Union[int, Tuple[int, int]],
max_bounces: int,
channels=[redner.channels.radiance],
sampler_type=redner.SamplerType.independent,
use_primary_edge_sampling=True,
use_secondary_edge_sampling=True) -> List:
"""
Given a pyredner scene & rendering options, convert them to a linear list of argument,
so that we can use it in PyTorch.
Args
====
scene: pyredner.Scene
num_samples: int
number of samples per pixel for forward and backward passes
can be an integer or a tuple of 2 integers
if a single integer is provided, use the same number of samples
for both
max_bounces: int
number of bounces for global illumination
1 means direct lighting only
channels: List[redner.channels]
| A list of channels that should present in the output image
| following channels are supported\:
| redner.channels.radiance,
| redner.channels.alpha,
| redner.channels.depth,
| redner.channels.position,
| redner.channels.geometry_normal,
| redner.channels.shading_normal,
| redner.channels.uv,
| redner.channels.diffuse_reflectance,
| redner.channels.specular_reflectance,
| redner.channels.vertex_color,
| redner.channels.roughness,
| redner.channels.generic_texture,
| redner.channels.shape_id,
| redner.channels.material_id
| all channels, except for shape id and material id, are differentiable
sampler_type: redner.SamplerType
| Which sampling pattern to use?
| see `Chapter 7 of the PBRT book <http://www.pbr-book.org/3ed-2018/Sampling_and_Reconstruction.html>`
for an explanation of the difference between different samplers.
| Following samplers are supported:
| redner.SamplerType.independent
| redner.SamplerType.sobol
use_primary_edge_sampling: bool
use_secondary_edge_sampling: bool
"""
cam = scene.camera
num_shapes = len(scene.shapes)
num_materials = len(scene.materials)
num_lights = len(scene.area_lights)
num_channels = len(channels)
for light_id, light in enumerate(scene.area_lights):
scene.shapes[light.shape_id].light_id = light_id
args = []
args.append(tf.constant(num_shapes))
args.append(tf.constant(num_materials))
args.append(tf.constant(num_lights))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if cam.position is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.position))
args.append(tf.identity(cam.look_at))
args.append(tf.identity(cam.up))
if cam.cam_to_world is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.cam_to_world))
args.append(tf.identity(cam.world_to_cam))
args.append(tf.identity(cam.intrinsic_mat_inv))
args.append(tf.identity(cam.intrinsic_mat))
args.append(tf.constant(cam.clip_near))
args.append(tf.constant(cam.resolution))
args.append(RednerCameraType.asTensor(cam.camera_type))
for shape in scene.shapes:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(shape.vertices))
args.append(tf.identity(shape.indices))
if shape.uvs is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.uvs))
if shape.normals is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.normals))
if shape.uv_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.uv_indices))
if shape.normal_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.normal_indices))
if shape.colors is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.colors))
args.append(tf.constant(shape.material_id))
args.append(tf.constant(shape.light_id))
for material in scene.materials:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(material.diffuse_reflectance.mipmap))
args.append(tf.identity(material.diffuse_reflectance.uv_scale))
args.append(tf.identity(material.specular_reflectance.mipmap))
args.append(tf.identity(material.specular_reflectance.uv_scale))
args.append(tf.identity(material.roughness.mipmap))
args.append(tf.identity(material.roughness.uv_scale))
if material.generic_texture is not None:
args.append(tf.identity(material.generic_texture.mipmap))
args.append(tf.identity(material.generic_texture.uv_scale))
else:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
if material.normal_map is not None:
args.append(tf.identity(material.normal_map.mipmap))
args.append(tf.identity(material.normal_map.uv_scale))
else:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(tf.constant(material.compute_specular_lighting))
args.append(tf.constant(material.two_sided))
args.append(tf.constant(material.use_vertex_color))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
for light in scene.area_lights:
args.append(tf.constant(light.shape_id))
args.append(tf.identity(light.intensity))
args.append(tf.constant(light.two_sided))
if scene.envmap is not None:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(scene.envmap.values.mipmap))
args.append(tf.identity(scene.envmap.values.uv_scale))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
args.append(tf.identity(scene.envmap.env_to_world))
args.append(tf.identity(scene.envmap.world_to_env))
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(scene.envmap.sample_cdf_ys))
args.append(tf.identity(scene.envmap.sample_cdf_xs))
args.append(scene.envmap.pdf_norm)
else:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(tf.constant(num_samples))
args.append(tf.constant(max_bounces))
args.append(tf.constant(num_channels))
for ch in channels:
args.append(RednerChannels.asTensor(ch))
args.append(RednerSamplerType.asTensor(sampler_type))
args.append(tf.constant(use_primary_edge_sampling))
args.append(tf.constant(use_secondary_edge_sampling))
return args
def serialize_scene(scene: pyredner.Scene,
num_samples: Union[int, Tuple[int, int]],
max_bounces: int,
channels=[redner.channels.radiance],
sampler_type=redner.SamplerType.independent,
use_primary_edge_sampling=True,
use_secondary_edge_sampling=True,
sample_pixel_center: bool = False) -> List:
"""
Given a pyredner scene & rendering options, convert them to a linear list of argument,
so that we can use it in TensorFlow.
Args
====
scene: pyredner.Scene
num_samples: int
number of samples per pixel for forward and backward passes
can be an integer or a tuple of 2 integers
if a single integer is provided, use the same number of samples
for both
max_bounces: int
number of bounces for global illumination
1 means direct lighting only
channels: List[redner.channels]
| A list of channels that should present in the output image
| following channels are supported\:
| redner.channels.radiance,
| redner.channels.alpha,
| redner.channels.depth,
| redner.channels.position,
| redner.channels.geometry_normal,
| redner.channels.shading_normal,
| redner.channels.uv,
| redner.channels.diffuse_reflectance,
| redner.channels.specular_reflectance,
| redner.channels.vertex_color,
| redner.channels.roughness,
| redner.channels.generic_texture,
| redner.channels.shape_id,
| redner.channels.triangle_id,
| redner.channels.material_id
| all channels, except for shape id, triangle id and material id, are differentiable
sampler_type: redner.SamplerType
| Which sampling pattern to use?
| see `Chapter 7 of the PBRT book <http://www.pbr-book.org/3ed-2018/Sampling_and_Reconstruction.html>`
for an explanation of the difference between different samplers.
| Following samplers are supported:
| redner.SamplerType.independent
| redner.SamplerType.sobol
use_primary_edge_sampling: bool
use_secondary_edge_sampling: bool
sample_pixel_center: bool
Always sample at the pixel center when rendering.
This trades noise with aliasing.
If this option is activated, the rendering becomes non-differentiable
(since there is no antialiasing integral),
and redner's edge sampling becomes an approximation to the gradients of the aliased rendering.
"""
# TODO: figure out a way to determine whether a TF tensor requires gradient or not
cam = scene.camera
num_shapes = len(scene.shapes)
num_materials = len(scene.materials)
num_lights = len(scene.area_lights)
num_channels = len(channels)
for light_id, light in enumerate(scene.area_lights):
scene.shapes[light.shape_id].light_id = light_id
if max_bounces == 0:
use_secondary_edge_sampling = False
args = []
args.append(tf.constant(num_shapes))
args.append(tf.constant(num_materials))
args.append(tf.constant(num_lights))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
if cam.position is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.position))
args.append(tf.identity(cam.look_at))
args.append(tf.identity(cam.up))
if cam.cam_to_world is None:
args.append(__EMPTY_TENSOR)
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(cam.cam_to_world))
args.append(tf.identity(cam.world_to_cam))
args.append(tf.identity(cam.intrinsic_mat_inv))
args.append(tf.identity(cam.intrinsic_mat))
args.append(tf.constant(cam.clip_near))
args.append(tf.constant(cam.resolution))
viewport = cam.viewport
if viewport is None:
viewport = (0, 0, cam.resolution[0], cam.resolution[1])
# Clamp the viewport if necessary
viewport = (max(viewport[0], 0), max(viewport[1], 0),
min(viewport[2],
cam.resolution[0]), min(viewport[3], cam.resolution[1]))
args.append(tf.constant(viewport))
args.append(RednerCameraType.asTensor(cam.camera_type))
for shape in scene.shapes:
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(shape.vertices))
# HACK: tf.bitcast forces tensorflow to copy int32 to GPU memory.
# tf.identity stopped working since TF 2.1 (if you print the device
# it will say it's on GPU, but the address returned by data_ptr is wrong).
# Hopefully TF people will fix this in the future.
args.append(tf.bitcast(shape.indices, type=tf.int32))
if shape.uvs is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.uvs))
if shape.normals is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.normals))
if shape.uv_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.bitcast(shape.uv_indices, type=tf.int32))
if shape.normal_indices is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.bitcast(shape.normal_indices, type=tf.int32))
if shape.colors is None:
args.append(__EMPTY_TENSOR)
else:
args.append(tf.identity(shape.colors))
args.append(tf.constant(shape.material_id))
args.append(tf.constant(shape.light_id))
for material in scene.materials:
serialize_texture(material.diffuse_reflectance, args)
serialize_texture(material.specular_reflectance, args)
serialize_texture(material.roughness, args)
serialize_texture(material.generic_texture, args)
serialize_texture(material.normal_map, args)
args.append(tf.constant(material.compute_specular_lighting))
args.append(tf.constant(material.two_sided))
args.append(tf.constant(material.use_vertex_color))
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
for light in scene.area_lights:
args.append(tf.constant(light.shape_id))
args.append(tf.identity(light.intensity))
args.append(tf.constant(light.two_sided))
args.append(tf.constant(light.directly_visible))
if scene.envmap is not None:
serialize_texture(scene.envmap.values, args)
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
args.append(tf.identity(scene.envmap.env_to_world))
args.append(tf.identity(scene.envmap.world_to_env))
with tf.device(pyredner.get_device_name()):
args.append(tf.identity(scene.envmap.sample_cdf_ys))
args.append(tf.identity(scene.envmap.sample_cdf_xs))
args.append(scene.envmap.pdf_norm)
args.append(scene.envmap.directly_visible)
else:
args.append(__EMPTY_TENSOR)
args.append(tf.constant(num_samples))
args.append(tf.constant(max_bounces))
args.append(tf.constant(num_channels))
for ch in channels:
args.append(RednerChannels.asTensor(ch))
args.append(RednerSamplerType.asTensor(sampler_type))
args.append(tf.constant(use_primary_edge_sampling))
args.append(tf.constant(use_secondary_edge_sampling))
args.append(tf.constant(sample_pixel_center))
return args
def load_obj(filename: str,
obj_group: bool = True,
flip_tex_coords: bool = True,
use_common_indices: bool = False,
return_objects: bool = False):
"""
Load from a Wavefront obj file as PyTorch tensors.
Args
====
obj_group: bool
split the meshes based on materials
flip_tex_coords: bool
flip the v coordinate of uv by applying v' = 1 - v
use_common_indices: bool
Use the same indices for position, uvs, normals.
Not recommended since texture seams in the objects sharing
the same positions would cause the optimization to "tear" the object
return_objects: bool
Output list of Object instead.
If there is no corresponding material for a shape, assign a grey material.
Returns
=======
if return_objects == True, return a list of Object
if return_objects == False, return (material_map, mesh_list, light_map),
material_map -> Map[mtl_name, WavefrontMaterial]
mesh_list -> List[TriangleMesh]
light_map -> Map[mtl_name, torch.Tensor]
"""
if device_name is None:
device_name = pyredner.get_device_name()
vertices_pool = []
uvs_pool = []
normals_pool = []
indices = []
uv_indices = []
normal_indices = []
vertices = []
uvs = []
normals = []
vertices_map = {}
uvs_map = {}
normals_map = {}
material_map = {}
current_mtllib = {}
current_material_name = None
def create_mesh(indices, uv_indices, normal_indices, vertices, uvs,
normals):
indices = tf.constant(indices, dtype=tf.int32)
if len(uv_indices) == 0:
uv_indices = None
else:
uv_indices = tf.constant(uv_indices, dtype=tf.int32)
if len(normal_indices) == 0:
normal_indices = None
else:
normal_indices = tf.constant(normal_indices, dtype=tf.int32)
vertices = tf.constant(vertices)
if len(uvs) == 0:
uvs = None
else:
uvs = tf.constant(uvs)
if len(normals) == 0:
normals = None
else:
normals = tf.constant(normals)
return TriangleMesh(indices, uv_indices, normal_indices, vertices, uvs,
normals)
mesh_list = []
light_map = {}
with open(filename, 'r') as f:
d = os.path.dirname(filename)
cwd = os.getcwd()
if d != '':
os.chdir(d)
for line in f:
line = line.strip()
splitted = re.split('\ +', line)
if splitted[0] == 'mtllib':
current_mtllib = load_mtl(splitted[1])
elif splitted[0] == 'usemtl':
if len(indices) > 0 and obj_group is True:
# Flush
mesh_list.append(
(current_material_name,
create_mesh(indices, uv_indices, normal_indices,
vertices, uvs, normals)))
indices = []
uv_indices = []
normal_indices = []
vertices = []
normals = []
uvs = []
vertices_map = {}
uvs_map = {}
normals_map = {}
mtl_name = splitted[1]
current_material_name = mtl_name
if mtl_name not in material_map:
m = current_mtllib[mtl_name]
if m.map_Kd is None:
diffuse_reflectance = tf.constant(m.Kd,
dtype=tf.float32)
else:
diffuse_reflectance = pyredner.imread(m.map_Kd)
if m.map_Ks is None:
specular_reflectance = tf.constant(m.Ks,
dtype=tf.float32)
else:
specular_reflectance = pyredner.imread(m.map_Ks)
if m.map_Ns is None:
roughness = tf.constant([2.0 / (m.Ns + 2.0)],
dtype=tf.float32)
else:
roughness = 2.0 / (pyredner.imread(m.map_Ks) + 2.0)
if m.Ke != (0.0, 0.0, 0.0):
light_map[mtl_name] = tf.constant(m.Ke,
dtype=tf.float32)
material_map[mtl_name] = pyredner.Material(
diffuse_reflectance, specular_reflectance, roughness)
elif splitted[0] == 'v':
vertices_pool.append([
float(splitted[1]),
float(splitted[2]),
float(splitted[3])
])
elif splitted[0] == 'vt':
u = float(splitted[1])
v = float(splitted[2])
if flip_tex_coords:
v = 1 - v
uvs_pool.append([u, v])
elif splitted[0] == 'vn':
normals_pool.append([
float(splitted[1]),
float(splitted[2]),
float(splitted[3])
])
elif splitted[0] == 'f':
def num_indices(x):
return len(re.split('/', x))
def get_index(x, i):
return int(re.split('/', x)[i])
def parse_face_index(x, i):
f = get_index(x, i)
if f > 0:
f -= 1
return f
assert (len(splitted) <= 5)
def get_vertex_id(indices):
pi = parse_face_index(indices, 0)
uvi = None
if (num_indices(indices) > 1
and re.split('/', indices)[1] != ''):
uvi = parse_face_index(indices, 1)
ni = None
if (num_indices(indices) > 2
and re.split('/', indices)[2] != ''):
ni = parse_face_index(indices, 2)
if use_common_indices:
# vertex, uv, normals share the same indexing
key = (pi, uvi, ni)
if key in vertices_map:
vertex_id = vertices_map[key]
return vertex_id, vertex_id, vertex_id
vertex_id = len(vertices)
vertices_map[key] = vertex_id
vertices.append(vertices_pool[pi])
if uvi is not None:
uvs.append(uvs_pool[uvi])
if ni is not None:
normals.append(normals_pool[ni])
return vertex_id, vertex_id, vertex_id
else:
# vertex, uv, normals use separate indexing
vertex_id = None
uv_id = None
normal_id = None
if pi in vertices_map:
vertex_id = vertices_map[pi]
else:
vertex_id = len(vertices)
vertices.append(vertices_pool[pi])
vertices_map[pi] = vertex_id
if uvi is not None:
if uvi in uvs_map:
uv_id = uvs_map[uvi]
else:
uv_id = len(uvs)
uvs.append(uvs_pool[uvi])
uvs_map[uvi] = uv_id
if ni is not None:
if ni in normals_map:
normal_id = normals_map[ni]
else:
normal_id = len(normals)
normals.append(normals_pool[ni])
normals_map[ni] = normal_id
return vertex_id, uv_id, normal_id
vid0, uv_id0, n_id0 = get_vertex_id(splitted[1])
vid1, uv_id1, n_id1 = get_vertex_id(splitted[2])
vid2, uv_id2, n_id2 = get_vertex_id(splitted[3])
indices.append([vid0, vid1, vid2])
if uv_id0 is not None:
assert (uv_id1 is not None and uv_id2 is not None)
uv_indices.append([uv_id0, uv_id1, uv_id2])
if n_id0 is not None:
assert (n_id1 is not None and n_id2 is not None)
normal_indices.append([n_id0, n_id1, n_id2])
if (len(splitted) == 5):
vid3, uv_id3, n_id3 = get_vertex_id(splitted[4])
indices.append([vid0, vid2, vid3])
if uv_id0 is not None:
assert (uv_id3 is not None)
uv_indices.append([uv_id0, uv_id2, uv_id3])
if n_id0 is not None:
assert (n_id3 is not None)
normal_indices.append([n_id0, n_id2, n_id3])
mesh_list.append((current_material_name,
create_mesh(indices, uv_indices, normal_indices,
vertices, uvs, normals)))
if d != '':
os.chdir(cwd)
if return_objects:
objects = []
for mtl_name, mesh in mesh_list:
if mtl_name in material_map:
m = material_map[mtl_name]
else:
m = pyredner.Material(diffuse_reflectance = \
tf.constant((0.5, 0.5, 0.5)))
if mtl_name in light_map:
l = light_map[mtl_name]
else:
l = None
objects.append(pyredner.Object(\
vertices = mesh.vertices,
indices = mesh.indices,
material = m,
light_intensity = l,
uvs = mesh.uvs,
normals = mesh.normals,
uv_indices = mesh.uv_indices,
normal_indices = mesh.normal_indices))
return objects
else:
return material_map, mesh_list, light_map
def generate_geometry_image(size: int):
"""
Generate an spherical geometry image [Gu et al. 2002 and Praun and Hoppe 2003]
of size [2 * size + 1, 2 * size + 1]. This can be used for encoding a genus-0
surface into a regular image, so that it is more convienent for a CNN to process.
The topology is given by a tesselated octahedron. UV is given by the spherical mapping.
Duplicated vertex are mapped to the one with smaller index (so some vertices on the
geometry image is unused by the indices).
Args
====
size: int
size of the geometry image
Returns
=======
tf.Tensor
vertices of size [(2 * size + 1 * 2 * size + 1), 3]
tf.Tensor
indices of size [2 * (2 * size + 1 * 2 * size + 1), 3]
tf.Tensor
uvs of size [(2 * size + 1 * 2 * size + 1), 2]
"""
size *= 2
# Generate vertices and uv by going through each vertex.
left_top = np.array([0.0, 0.0, 1.0])
top = np.array([0.0, 1.0, 0.0])
right_top = np.array([0.0, 0.0, 1.0])
left = np.array([-1.0, 0.0, 0.0])
middle = np.array([0.0, 0.0, -1.0])
right = np.array([1.0, 0.0, 0.0])
left_bottom = np.array([0.0, 0.0, 1.0])
bottom = np.array([0.0, -1.0, 0.0])
right_bottom = np.array([0.0, 0.0, 1.0])
vertices = np.zeros([(size + 1) * (size + 1), 3])
uvs = np.zeros([(size + 1) * (size + 1), 2])
vertex_id = 0
half_size = size / 2.0
for i in range(size + 1): # height
for j in range(size + 1): # width
# Left Top
if i + j <= half_size:
org = left_top
i_axis = left - left_top
j_axis = top - left_top
i_ = float(i) / half_size
j_ = float(j) / half_size
elif (i + j >= half_size and i <= half_size and j <= half_size):
org = middle
i_axis = top - middle
j_axis = left - middle
i_ = 1.0 - float(i) / half_size
j_ = 1.0 - float(j) / half_size
# Right Top
elif ((half_size - i + j - half_size) <= half_size
and i <= half_size and j >= half_size):
org = middle
i_axis = top - middle
j_axis = right - middle
i_ = 1.0 - float(i) / half_size
j_ = float(j) / half_size - 1.0
elif ((i + size - j) <= half_size):
org = right_top
i_axis = right - right_top
j_axis = top - right_top
i_ = float(i) / half_size
j_ = 2.0 - float(j) / half_size
# Left Bottom
elif ((i - half_size + half_size - j) <= half_size
and i >= half_size and j <= half_size):
org = middle
i_axis = bottom - middle
j_axis = left - middle
i_ = float(i) / half_size - 1.0
j_ = 1.0 - float(j) / half_size
elif ((size - i + j) <= half_size):
org = left_bottom
i_axis = left - left_bottom
j_axis = bottom - left_bottom
i_ = 2.0 - float(i) / half_size
j_ = float(j) / half_size
# Right Bottom
elif ((i - half_size + j - half_size) <= half_size
and i >= half_size and j >= half_size):
org = middle
i_axis = bottom - middle
j_axis = right - middle
i_ = float(i) / half_size - 1.0
j_ = float(j) / half_size - 1.0
else:
org = right_bottom
i_axis = right - right_bottom
j_axis = bottom - right_bottom
i_ = 2.0 - float(i) / half_size
j_ = 2.0 - float(j) / half_size
p = org + i_ * i_axis + j_ * j_axis
vertices[vertex_id, :] = p / np.linalg.norm(p)
# Spherical UV mapping
u = 0.5 + math.atan2(float(p[2]), float(p[0])) / (2 * math.pi)
v = 0.5 - math.asin(float(p[1])) / math.pi
uvs[vertex_id, :] = np.array([u, v])
vertex_id += 1
# Generate indices by going through each triangle.
# Duplicated vertex are mapped to the one with smaller index.
indices = []
for i in range(size): # height
for j in range(size): # width
left_top = i * (size + 1) + j
right_top = i * (size + 1) + j + 1
left_bottom = (i + 1) * (size + 1) + j
right_bottom = (i + 1) * (size + 1) + j + 1
# Wrap rule for octahedron topology
if i == 0 and j >= half_size:
if j > half_size:
left_top = i * (size + 1) + size - j
right_top = i * (size + 1) + (size - (j + 1))
elif i == size - 1 and j >= half_size:
if j > half_size:
left_bottom = (i + 1) * (size + 1) + size - j
right_bottom = (i + 1) * (size + 1) + (size - (j + 1))
if j == size - 1:
right_bottom = 0
elif j == 0 and i >= half_size:
if i > half_size:
left_top = (size - i) * (size + 1) + j
left_bottom = (size - (i + 1)) * (size + 1) + j
elif j == size - 1 and i >= half_size:
if i > half_size:
right_top = (size - i) * (size + 1) + j + 1
right_bottom = (size - (i + 1)) * (size + 1) + j + 1
# Left Top
if i < half_size and j < half_size:
indices.append((left_top, left_bottom, right_top))
indices.append((right_top, left_bottom, right_bottom))
# Right Top
elif i < half_size and j >= half_size:
indices.append((left_top, left_bottom, right_bottom))
indices.append((left_top, right_bottom, right_top))
# Left Bottom
elif i >= half_size and j < half_size:
indices.append((left_top, right_bottom, right_top))
indices.append((left_top, left_bottom, right_bottom))
# Right Bottom
else:
indices.append((left_top, left_bottom, right_top))
indices.append((right_top, left_bottom, right_bottom))
with tf.device(pyredner.get_device_name()):
vertices = tf.constant(vertices, dtype=tf.float32)
uvs = tf.constant(uvs, dtype=tf.float32)
indices = tf.constant(indices, dtype=tf.int32)
return vertices, indices, uvs
def create_gradient_buffers(ctx):
scene = ctx.scene
options = ctx.options
camera = ctx.camera
buffers = Context()
with tf.device(pyredner.get_device_name()):
if camera.use_look_at:
buffers.d_position = tf.zeros(3, dtype=tf.float32)
buffers.d_look_at = tf.zeros(3, dtype=tf.float32)
buffers.d_up = tf.zeros(3, dtype=tf.float32)
buffers.d_cam_to_world = None
buffers.d_world_to_cam = None
else:
buffers.d_position = None
buffers.d_look_at = None
buffers.d_up = None
buffers.d_cam_to_world = tf.zeros([4, 4], dtype=tf.float32)
buffers.d_world_to_cam = tf.zeros([4, 4], dtype=tf.float32)
buffers.d_intrinsic_mat_inv = tf.zeros([3, 3], dtype=tf.float32)
buffers.d_intrinsic_mat = tf.zeros([3, 3], dtype=tf.float32)
if camera.use_look_at:
buffers.d_camera = redner.DCamera(\
redner.float_ptr(pyredner.data_ptr(buffers.d_position)),
redner.float_ptr(pyredner.data_ptr(buffers.d_look_at)),
redner.float_ptr(pyredner.data_ptr(buffers.d_up)),
redner.float_ptr(0), # cam_to_world
redner.float_ptr(0), # world_to_cam
redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat)))
else:
buffers.d_camera = redner.DCamera(\
redner.float_ptr(0),
redner.float_ptr(0),
redner.float_ptr(0),
redner.float_ptr(pyredner.data_ptr(buffers.d_cam_to_world)),
redner.float_ptr(pyredner.data_ptr(buffers.d_world_to_cam)),
redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat)))
buffers.d_vertices_list = []
buffers.d_uvs_list = []
buffers.d_normals_list = []
buffers.d_colors_list = []
buffers.d_shapes = []
with tf.device(pyredner.get_device_name()):
for i, shape in enumerate(ctx.shapes):
num_vertices = shape.num_vertices
d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32)
d_uvs = tf.zeros([num_vertices, 2],
dtype=tf.float32) if shape.has_uvs() else None
d_normals = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_normals() else None
d_colors = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_colors() else None
buffers.d_vertices_list.append(d_vertices)
buffers.d_uvs_list.append(d_uvs)
buffers.d_normals_list.append(d_normals)
buffers.d_colors_list.append(d_colors)
buffers.d_shapes.append(redner.DShape(\
redner.float_ptr(pyredner.data_ptr(d_vertices)),
redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_colors) if d_colors is not None else 0)))
buffers.d_diffuse_list = []
buffers.d_specular_list = []
buffers.d_roughness_list = []
buffers.d_normal_map_list = []
buffers.d_diffuse_uv_scale_list = []
buffers.d_specular_uv_scale_list = []
buffers.d_roughness_uv_scale_list = []
buffers.d_generic_list = []
buffers.d_generic_uv_scale_list = []
buffers.d_normal_map_uv_scale_list = []
buffers.d_materials = []
with tf.device(pyredner.get_device_name()):
for material in ctx.materials:
if material.get_diffuse_size(0)[0] == 0:
d_diffuse = [tf.zeros(3, dtype=tf.float32)]
else:
d_diffuse = []
for l in range(material.get_diffuse_levels()):
diffuse_size = material.get_diffuse_size(l)
d_diffuse.append(\
tf.zeros([diffuse_size[1],
diffuse_size[0],
3], dtype=tf.float32))
if material.get_specular_size(0)[0] == 0:
d_specular = [tf.zeros(3, dtype=tf.float32)]
else:
d_specular = []
for l in range(material.get_specular_levels()):
specular_size = material.get_specular_size(l)
d_specular.append(\
tf.zeros([specular_size[1],
specular_size[0],
3], dtype=tf.float32))
if material.get_roughness_size(0)[0] == 0:
d_roughness = [tf.zeros(1, dtype=tf.float32)]
else:
d_roughness = []
for l in range(material.get_roughness_levels()):
roughness_size = material.get_roughness_size(l)
d_roughness.append(\
tf.zeros([roughness_size[1],
roughness_size[0],
1], dtype=tf.float32))
# HACK: tensorflow's eager mode uses a cache to store scalar
# constants to avoid memory copy. If we pass scalar tensors
# into the C++ code and modify them, we would corrupt the
# cache, causing incorrect result in future scalar constant
# creations. Thus we force tensorflow to copy by plusing a zero.
# (also see https://github.com/tensorflow/tensorflow/issues/11186
# for more discussion regarding copying tensors)
if d_roughness[0].shape.num_elements() == 1:
d_roughness[0] = d_roughness[0] + 0
if material.get_generic_levels() == 0:
d_generic = None
else:
d_generic = []
for l in range(material.get_generic_levels()):
generic_size = material.get_generic_size(l)
d_generic.append(\
tf.zeros([generic_size[2],
generic_size[1],
generic_size[0]], dtype=tf.float32))
if material.get_normal_map_levels() == 0:
d_normal_map = None
else:
d_normal_map = []
for l in range(material.get_normal_map_levels()):
normal_map_size = material.get_normal_map_size(l)
d_normal_map.append(\
tf.zeros([normal_map_size[1],
normal_map_size[0],
3], dtype=tf.float32))
buffers.d_diffuse_list.append(d_diffuse)
buffers.d_specular_list.append(d_specular)
buffers.d_roughness_list.append(d_roughness)
buffers.d_generic_list.append(d_generic)
buffers.d_normal_map_list.append(d_normal_map)
d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32)
d_specular_uv_scale = tf.zeros([2], dtype=tf.float32)
d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32)
if d_generic is None:
d_generic_uv_scale = None
else:
d_generic_uv_scale = tf.zeros([2], dtype=tf.float32)
if d_normal_map is None:
d_normal_map_uv_scale = None
else:
d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32)
buffers.d_diffuse_uv_scale_list.append(d_diffuse_uv_scale)
buffers.d_specular_uv_scale_list.append(d_specular_uv_scale)
buffers.d_roughness_uv_scale_list.append(d_roughness_uv_scale)
buffers.d_generic_uv_scale_list.append(d_generic_uv_scale)
buffers.d_normal_map_uv_scale_list.append(d_normal_map_uv_scale)
if len(d_diffuse[0].shape) == 1:
d_diffuse_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(d_diffuse[0]))],
[0],
[0],
3,
redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale)))
else:
d_diffuse_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_diffuse],
[x.shape[1] for x in d_diffuse],
[x.shape[0] for x in d_diffuse],
3,
redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale)))
if len(d_specular[0].shape) == 1:
d_specular_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(d_specular[0]))],
[0],
[0],
3,
redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale)))
else:
d_specular_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_specular],
[x.shape[1] for x in d_specular],
[x.shape[0] for x in d_specular],
3,
redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale)))
if len(d_roughness[0].shape) == 1:
d_roughness_tex = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(d_roughness[0]))],
[0],
[0],
1,
redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale)))
else:
d_roughness_tex = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_roughness],
[x.shape[1] for x in d_roughness],
[x.shape[0] for x in d_roughness],
1,
redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale)))
if d_generic is None:
d_generic_tex = redner.TextureN(\
[], [], [], 0, redner.float_ptr(0))
else:
d_generic_tex = redner.TextureN(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_generic],
[x.shape[1] for x in d_generic],
[x.shape[0] for x in d_generic],
d_generic[0].shape[2],
redner.float_ptr(pyredner.data_ptr(d_generic_uv_scale)))
if d_normal_map is None:
d_normal_map = redner.Texture3(\
[], [], [], 0, redner.float_ptr(0))
else:
d_normal_map = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_normal_map],
[x.shape[1] for x in d_normal_map],
[x.shape[0] for x in d_normal_map],
3,
redner.float_ptr(pyredner.data_ptr(d_normal_map_uv_scale)))
buffers.d_materials.append(redner.DMaterial(\
d_diffuse_tex, d_specular_tex, d_roughness_tex,
d_generic_tex, d_normal_map))
buffers.d_intensity_list = []
buffers.d_area_lights = []
with tf.device(pyredner.get_device_name()):
for light in ctx.area_lights:
d_intensity = tf.zeros(3, dtype=tf.float32)
buffers.d_intensity_list.append(d_intensity)
buffers.d_area_lights.append(\
redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity))))
buffers.d_envmap = None
if ctx.envmap is not None:
envmap = ctx.envmap
with tf.device(pyredner.get_device_name()):
buffers.d_envmap_values = []
for l in range(envmap.get_levels()):
size = envmap.get_size(l)
buffers.d_envmap_values.append(\
tf.zeros([size[1],
size[0],
3], dtype=tf.float32))
buffers.d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32)
buffers.d_world_to_env = tf.zeros([4, 4], dtype=tf.float32)
d_envmap_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in buffers.d_envmap_values],
[x.shape[1] for x in buffers.d_envmap_values],
[x.shape[0] for x in buffers.d_envmap_values],
3,
redner.float_ptr(pyredner.data_ptr(buffers.d_envmap_uv_scale)))
buffers.d_envmap = redner.DEnvironmentMap(
d_envmap_tex,
redner.float_ptr(pyredner.data_ptr(buffers.d_world_to_env)))
buffers.d_scene = redner.DScene(buffers.d_camera, buffers.d_shapes,
buffers.d_materials, buffers.d_area_lights,
buffers.d_envmap, pyredner.get_use_gpu(),
pyredner.get_gpu_device_id())
return buffers
def backward(grad_img):
camera = ctx.camera
scene = ctx.scene
options = ctx.options
with tf.device(pyredner.get_device_name()):
if camera.use_look_at:
d_position = tf.zeros(3, dtype=tf.float32)
d_look_at = tf.zeros(3, dtype=tf.float32)
d_up = tf.zeros(3, dtype=tf.float32)
d_cam_to_world = None
d_world_to_cam = None
else:
d_position = None
d_look_at = None
d_up = None
d_cam_to_world = tf.zeros([4, 4], dtype=tf.float32)
d_world_to_cam = tf.zeros([4, 4], dtype=tf.float32)
d_intrinsic_mat_inv = tf.zeros([3, 3], dtype=tf.float32)
d_intrinsic_mat = tf.zeros([3, 3], dtype=tf.float32)
if camera.use_look_at:
d_camera = redner.DCamera(
redner.float_ptr(pyredner.data_ptr(d_position)),
redner.float_ptr(pyredner.data_ptr(d_look_at)),
redner.float_ptr(pyredner.data_ptr(d_up)),
redner.float_ptr(0), # cam_to_world
redner.float_ptr(0), # world_to_cam
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat)))
else:
d_camera = redner.DCamera(
redner.float_ptr(0), redner.float_ptr(0),
redner.float_ptr(0),
redner.float_ptr(pyredner.data_ptr(d_cam_to_world)),
redner.float_ptr(pyredner.data_ptr(d_world_to_cam)),
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat_inv)),
redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat)))
d_vertices_list = []
d_uvs_list = []
d_normals_list = []
d_colors_list = []
d_shapes = []
with tf.device(pyredner.get_device_name()):
for i, shape in enumerate(ctx.shapes):
num_vertices = shape.num_vertices
d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32)
d_uvs = tf.zeros([num_vertices, 2],
dtype=tf.float32) if shape.has_uvs() else None
d_normals = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_normals() else None
d_colors = tf.zeros(
[num_vertices, 3],
dtype=tf.float32) if shape.has_colors() else None
d_vertices_list.append(d_vertices)
d_uvs_list.append(d_uvs)
d_normals_list.append(d_normals)
d_colors_list.append(d_colors)
d_shapes.append(redner.DShape(\
redner.float_ptr(pyredner.data_ptr(d_vertices)),
redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0),
redner.float_ptr(pyredner.data_ptr(d_colors) if d_colors is not None else 0)))
d_diffuse_list = []
d_specular_list = []
d_roughness_list = []
d_normal_map_list = []
d_diffuse_uv_scale_list = []
d_specular_uv_scale_list = []
d_roughness_uv_scale_list = []
d_generic_list = []
d_generic_uv_scale_list = []
d_normal_map_uv_scale_list = []
d_materials = []
with tf.device(pyredner.get_device_name()):
for material in ctx.materials:
if material.get_diffuse_size(0)[0] == 0:
d_diffuse = [tf.zeros(3, dtype=tf.float32)]
else:
d_diffuse = []
for l in range(material.get_diffuse_levels()):
diffuse_size = material.get_diffuse_size(l)
d_diffuse.append(\
tf.zeros([diffuse_size[1],
diffuse_size[0],
3], dtype=tf.float32))
if material.get_specular_size(0)[0] == 0:
d_specular = [tf.zeros(3, dtype=tf.float32)]
else:
d_specular = []
for l in range(material.get_specular_levels()):
specular_size = material.get_specular_size(l)
d_specular.append(\
tf.zeros([specular_size[1],
specular_size[0],
3], dtype=tf.float32))
if material.get_roughness_size(0)[0] == 0:
d_roughness = [tf.zeros(1, dtype=tf.float32)]
else:
d_roughness = []
for l in range(material.get_roughness_levels()):
roughness_size = material.get_roughness_size(l)
d_roughness.append(\
tf.zeros([roughness_size[1],
roughness_size[0],
1], dtype=tf.float32))
# HACK: tensorflow's eager mode uses a cache to store scalar
# constants to avoid memory copy. If we pass scalar tensors
# into the C++ code and modify them, we would corrupt the
# cache, causing incorrect result in future scalar constant
# creations. Thus we force tensorflow to copy by plusing a zero.
# (also see https://github.com/tensorflow/tensorflow/issues/11186
# for more discussion regarding copying tensors)
if d_roughness[0].shape.num_elements() == 1:
d_roughness[0] = d_roughness[0] + 0
if material.get_generic_levels() == 0:
d_generic = None
else:
d_generic = []
for l in range(material.get_generic_levels()):
generic_size = material.get_generic_size(l)
d_generic.append(\
tf.zeros([generic_size[2],
generic_size[1],
generic_size[0]], dtype=tf.float32))
if material.get_normal_map_levels() == 0:
d_normal_map = None
else:
d_normal_map = []
for l in range(material.get_normal_map_levels()):
normal_map_size = material.get_normal_map_size(l)
d_normal_map.append(\
tf.zeros([normal_map_size[1],
normal_map_size[0],
3], dtype=tf.float32))
d_diffuse_list.append(d_diffuse)
d_specular_list.append(d_specular)
d_roughness_list.append(d_roughness)
d_generic_list.append(d_generic)
d_normal_map_list.append(d_normal_map)
d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32)
d_specular_uv_scale = tf.zeros([2], dtype=tf.float32)
d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32)
if d_generic is None:
d_generic_uv_scale = None
else:
d_generic_uv_scale = tf.zeros([2], dtype=tf.float32)
if d_normal_map is None:
d_normal_map_uv_scale = None
else:
d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32)
d_diffuse_uv_scale_list.append(d_diffuse_uv_scale)
d_specular_uv_scale_list.append(d_specular_uv_scale)
d_roughness_uv_scale_list.append(d_roughness_uv_scale)
d_generic_uv_scale_list.append(d_generic_uv_scale)
d_normal_map_uv_scale_list.append(d_normal_map_uv_scale)
if len(d_diffuse[0].shape) == 1:
d_diffuse_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(d_diffuse[0]))],
[0],
[0],
3,
redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale)))
else:
d_diffuse_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_diffuse],
[x.shape[1] for x in d_diffuse],
[x.shape[0] for x in d_diffuse],
3,
redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale)))
if len(d_specular[0].shape) == 1:
d_specular_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(d_specular[0]))],
[0],
[0],
3,
redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale)))
else:
d_specular_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_specular],
[x.shape[1] for x in d_specular],
[x.shape[0] for x in d_specular],
3,
redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale)))
if len(d_roughness[0].shape) == 1:
d_roughness_tex = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(d_roughness[0]))],
[0],
[0],
1,
redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale)))
else:
d_roughness_tex = redner.Texture1(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_roughness],
[x.shape[1] for x in d_roughness],
[x.shape[0] for x in d_roughness],
1,
redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale)))
if d_generic is None:
d_generic_tex = redner.TextureN(\
[], [], [], 0, redner.float_ptr(0))
else:
d_generic_tex = redner.TextureN(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_generic],
[x.shape[1] for x in d_generic],
[x.shape[0] for x in d_generic],
d_generic[0].shape[2],
redner.float_ptr(pyredner.data_ptr(d_generic_uv_scale)))
if d_normal_map is None:
d_normal_map = redner.Texture3(\
[], [], [], 0, redner.float_ptr(0))
else:
d_normal_map = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_normal_map],
[x.shape[1] for x in d_normal_map],
[x.shape[0] for x in d_normal_map],
3,
redner.float_ptr(pyredner.data_ptr(d_normal_map_uv_scale)))
d_materials.append(redner.DMaterial(\
d_diffuse_tex, d_specular_tex, d_roughness_tex,
d_generic_tex, d_normal_map))
d_intensity_list = []
d_area_lights = []
with tf.device(pyredner.get_device_name()):
for light in ctx.area_lights:
d_intensity = tf.zeros(3, dtype=tf.float32)
d_intensity_list.append(d_intensity)
d_area_lights.append(\
redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity))))
d_envmap = None
if ctx.envmap is not None:
envmap = ctx.envmap
with tf.device(pyredner.get_device_name()):
d_envmap_values = []
for l in range(envmap.get_levels()):
size = envmap.get_size(l)
d_envmap_values.append(\
tf.zeros([size[1],
size[0],
3], dtype=tf.float32))
d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32)
d_world_to_env = tf.zeros([4, 4], dtype=tf.float32)
d_envmap_tex = redner.Texture3(\
[redner.float_ptr(pyredner.data_ptr(x)) for x in d_envmap_values],
[x.shape[1] for x in d_envmap_values],
[x.shape[0] for x in d_envmap_values],
3,
redner.float_ptr(pyredner.data_ptr(d_envmap_uv_scale)))
d_envmap = redner.DEnvironmentMap(
d_envmap_tex,
redner.float_ptr(pyredner.data_ptr(d_world_to_env)))
d_scene = redner.DScene(d_camera, d_shapes, d_materials, d_area_lights,
d_envmap, pyredner.get_use_gpu(),
pyredner.get_gpu_device_id())
if not get_use_correlated_random_number():
# Decod_uple the forward/backward random numbers by adding a big prime number
options.seed += 1000003
start = time.time()
options.num_samples = ctx.num_samples[1]
with tf.device(pyredner.get_device_name()):
grad_img = tf.identity(grad_img)
redner.render(
scene,
options,
redner.float_ptr(0), # rendered_image
redner.float_ptr(pyredner.data_ptr(grad_img)),
d_scene,
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if print_timing:
print('Backward pass, time: %.5f s' % time_elapsed)
# # For debugging
# pyredner.imwrite(grad_img, 'grad_img.exr')
# grad_img = tf.ones([256, 256, 3], dtype=tf.float32)
# debug_img = tf.zeros([256, 256, 3], dtype=tf.float32)
# redner.render(scene, options,
# redner.float_ptr(0),
# redner.float_ptr(pyredner.data_ptr(grad_img)),
# d_scene,
# redner.float_ptr(pyredner.data_ptr(debug_img)))
# pyredner.imwrite(debug_img, 'debug.exr')
# pyredner.imwrite(-debug_img, 'debug_.exr')
# exit()
ret_list = []
ret_list.append(None) # seed
ret_list.append(None) # num_shapes
ret_list.append(None) # num_materials
ret_list.append(None) # num_lights
if camera.use_look_at:
ret_list.append(d_position)
ret_list.append(d_look_at)
ret_list.append(d_up)
ret_list.append(None) # cam_to_world
ret_list.append(None) # world_to_cam
else:
ret_list.append(None) # pos
ret_list.append(None) # look
ret_list.append(None) # up
ret_list.append(d_cam_to_world)
ret_list.append(d_world_to_cam)
ret_list.append(d_intrinsic_mat_inv)
ret_list.append(d_intrinsic_mat)
ret_list.append(None) # clip near
ret_list.append(None) # resolution
ret_list.append(None) # camera_type
num_shapes = len(ctx.shapes)
for i in range(num_shapes):
ret_list.append(d_vertices_list[i])
ret_list.append(None) # indices
ret_list.append(d_uvs_list[i])
ret_list.append(d_normals_list[i])
ret_list.append(None) # uv_indices
ret_list.append(None) # normal_indices
ret_list.append(d_colors_list[i])
ret_list.append(None) # material id
ret_list.append(None) # light id
num_materials = len(ctx.materials)
for i in range(num_materials):
ret_list.append(None) # num_levels
for d_diffuse in d_diffuse_list[i]:
ret_list.append(d_diffuse)
ret_list.append(d_diffuse_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_specular in d_specular_list[i]:
ret_list.append(d_specular)
ret_list.append(d_specular_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_roughness in d_roughness_list[i]:
ret_list.append(d_roughness)
ret_list.append(d_roughness_uv_scale_list[i])
if d_generic_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_generic in d_generic_list[i]:
ret_list.append(d_generic)
ret_list.append(d_generic_uv_scale_list[i])
if d_normal_map_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_normal_map in d_normal_map_list[i]:
ret_list.append(d_normal_map)
ret_list.append(d_normal_map_uv_scale_list[i])
ret_list.append(None) # compute_specular_lighting
ret_list.append(None) # two sided
ret_list.append(None) # use_vertex_color
num_area_lights = len(ctx.area_lights)
for i in range(num_area_lights):
ret_list.append(None) # shape id
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(d_intensity_list[i]))
ret_list.append(None) # two sided
if ctx.envmap is not None:
ret_list.append(None) # num_levels
for d_values in d_envmap_values:
ret_list.append(d_values)
ret_list.append(d_envmap_uv_scale)
ret_list.append(None) # env_to_world
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(d_world_to_env))
ret_list.append(None) # sample_cdf_ys
ret_list.append(None) # sample_cdf_xs
ret_list.append(None) # pdf_norm
else:
ret_list.append(None)
ret_list.append(None) # num samples
ret_list.append(None) # num bounces
ret_list.append(None) # num channels
for _ in range(ctx.num_channels):
ret_list.append(None) # channel
ret_list.append(None) # sampler type
ret_list.append(None) # use_primary_edge_sampling
ret_list.append(None) # use_secondary_edge_sampling
ret_list.append(None) # sample_pixel_center
return ret_list
def generate_sphere(theta_steps: int, phi_steps: int):
"""
Generate a triangle mesh representing a UV sphere,
center at (0, 0, 0) with radius 1.
Args
====
theta_steps: int
zenith subdivision
phi_steps: int
azimuth subdivision
Returns
=======
tf.Tensor
vertices
tf.Tensor
indices
tf.Tensor
uvs
tf.Tensor
normals
"""
d_theta = math.pi / (theta_steps - 1)
d_phi = (2 * math.pi) / (phi_steps - 1)
num_vertices = theta_steps * phi_steps - 2 * (phi_steps - 1)
vertices = np.zeros([num_vertices, 3], dtype=np.float32)
uvs = np.zeros([num_vertices, 2], dtype=np.float32)
vertices_index = 0
for theta_index in range(theta_steps):
sin_theta = math.sin(theta_index * d_theta)
cos_theta = math.cos(theta_index * d_theta)
if theta_index == 0:
# For the two polars of the sphere, only generate one vertex
vertices[vertices_index, :] = \
np.array([0.0, 1.0, 0.0], dtype = np.float32)
uvs[vertices_index, 0] = 0.0
uvs[vertices_index, 1] = 0.0
vertices_index += 1
elif theta_index == theta_steps - 1:
# For the two polars of the sphere, only generate one vertex
vertices[vertices_index, :] = \
np.array([0.0, -1.0, 0.0], dtype = np.float32)
uvs[vertices_index, 0] = 0.0
uvs[vertices_index, 1] = 1.0
vertices_index += 1
else:
for phi_index in range(phi_steps):
sin_phi = math.sin(phi_index * d_phi)
cos_phi = math.cos(phi_index * d_phi)
vertices[vertices_index, :] = \
np.array([sin_theta * cos_phi, cos_theta, sin_theta * sin_phi])
uvs[vertices_index, 0] = phi_index * d_phi / (2 * math.pi)
uvs[vertices_index, 1] = theta_index * d_theta / math.pi
vertices_index += 1
indices = []
for theta_index in range(1, theta_steps):
for phi_index in range(phi_steps - 1):
if theta_index < theta_steps - 1:
id0 = phi_steps * theta_index + phi_index - (phi_steps - 1)
id1 = phi_steps * theta_index + phi_index + 1 - (phi_steps - 1)
else:
# There is only one vertex at the pole
assert (theta_index == theta_steps - 1)
id0 = num_vertices - 1
id1 = num_vertices - 1
if theta_index > 1:
id2 = phi_steps * (theta_index - 1) + phi_index - (phi_steps -
1)
id3 = phi_steps * (theta_index - 1) + phi_index + 1 - (
phi_steps - 1)
else:
# There is only one vertex at the pole
assert (theta_index == 1)
id2 = 0
id3 = 0
if (theta_index < theta_steps - 1):
indices.append([id0, id2, id1])
if (theta_index > 1):
indices.append([id1, id2, id3])
with tf.device(pyredner.get_device_name()):
indices = tf.convert_to_tensor(indices, dtype=tf.int32)
vertices = tf.convert_to_tensor(vertices, dtype=tf.float32)
uvs = tf.convert_to_tensor(uvs, dtype=tf.float32)
normals = tf.identity(vertices)
return (vertices, indices, uvs, normals)
def backward(grad_img):
scene = ctx.scene
options = ctx.options
buffers = create_gradient_buffers(ctx)
if not get_use_correlated_random_number():
# Decod_uple the forward/backward random numbers by adding a big prime number
options.seed += 1000003
start = time.time()
options.num_samples = ctx.num_samples[1]
with tf.device(pyredner.get_device_name()):
grad_img = tf.identity(grad_img)
redner.render(
scene,
options,
redner.float_ptr(0), # rendered_image
redner.float_ptr(pyredner.data_ptr(grad_img)),
buffers.d_scene,
redner.float_ptr(0), # translational_gradient_image
redner.float_ptr(0)) # debug_image
time_elapsed = time.time() - start
if get_print_timing():
print('Backward pass, time: %.5f s' % time_elapsed)
ret_list = []
ret_list.append(None) # seed
ret_list.append(None) # num_shapes
ret_list.append(None) # num_materials
ret_list.append(None) # num_lights
if ctx.camera.use_look_at:
ret_list.append(buffers.d_position)
ret_list.append(buffers.d_look_at)
ret_list.append(buffers.d_up)
ret_list.append(None) # cam_to_world
ret_list.append(None) # world_to_cam
else:
ret_list.append(None) # pos
ret_list.append(None) # look
ret_list.append(None) # up
ret_list.append(buffers.d_cam_to_world)
ret_list.append(buffers.d_world_to_cam)
ret_list.append(buffers.d_intrinsic_mat_inv)
ret_list.append(buffers.d_intrinsic_mat)
ret_list.append(None) # clip near
ret_list.append(None) # resolution
ret_list.append(None) # viewport
ret_list.append(None) # camera_type
num_shapes = len(ctx.shapes)
for i in range(num_shapes):
ret_list.append(buffers.d_vertices_list[i])
ret_list.append(None) # indices
ret_list.append(buffers.d_uvs_list[i])
ret_list.append(buffers.d_normals_list[i])
ret_list.append(None) # uv_indices
ret_list.append(None) # normal_indices
ret_list.append(buffers.d_colors_list[i])
ret_list.append(None) # material id
ret_list.append(None) # light id
num_materials = len(ctx.materials)
for i in range(num_materials):
ret_list.append(None) # num_levels
for d_diffuse in buffers.d_diffuse_list[i]:
ret_list.append(d_diffuse)
ret_list.append(buffers.d_diffuse_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_specular in buffers.d_specular_list[i]:
ret_list.append(d_specular)
ret_list.append(buffers.d_specular_uv_scale_list[i])
ret_list.append(None) # num_levels
for d_roughness in buffers.d_roughness_list[i]:
ret_list.append(d_roughness)
ret_list.append(buffers.d_roughness_uv_scale_list[i])
if buffers.d_generic_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_generic in buffers.d_generic_list[i]:
ret_list.append(d_generic)
ret_list.append(buffers.d_generic_uv_scale_list[i])
if buffers.d_normal_map_list[i] is None:
ret_list.append(None) # num_levels
else:
ret_list.append(None) # num_levels
for d_normal_map in buffers.d_normal_map_list[i]:
ret_list.append(d_normal_map)
ret_list.append(buffers.d_normal_map_uv_scale_list[i])
ret_list.append(None) # compute_specular_lighting
ret_list.append(None) # two sided
ret_list.append(None) # use_vertex_color
num_area_lights = len(ctx.area_lights)
for i in range(num_area_lights):
ret_list.append(None) # shape id
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(buffers.d_intensity_list[i]))
ret_list.append(None) # two_sided
ret_list.append(None) # directly_visible
if ctx.envmap is not None:
ret_list.append(None) # num_levels
for d_values in buffers.d_envmap_values:
ret_list.append(d_values)
ret_list.append(buffers.d_envmap_uv_scale)
ret_list.append(None) # env_to_world
with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())):
ret_list.append(tf.identity(buffers.d_world_to_env))
ret_list.append(None) # sample_cdf_ys
ret_list.append(None) # sample_cdf_xs
ret_list.append(None) # pdf_norm
ret_list.append(None) # directly_visible
else:
ret_list.append(None)
ret_list.append(None) # num samples
ret_list.append(None) # num bounces
ret_list.append(None) # num channels
for _ in range(ctx.num_channel_args):
ret_list.append(None) # channel
ret_list.append(None) # sampler type
ret_list.append(None) # use_primary_edge_sampling
ret_list.append(None) # use_secondary_edge_sampling
ret_list.append(None) # sample_pixel_center
return ret_list
def __init__(self, texels, uv_scale=tf.constant([1.0, 1.0])):
assert (tf.executing_eagerly())
if pyredner.get_use_gpu():
texels = tf.identity(texels).gpu(pyredner.get_gpu_device_id())
uv_scale = tf.identity(uv_scale).gpu(pyredner.get_gpu_device_id())
else:
texels = tf.identity(texels).cpu()
uv_scale = tf.identity(uv_scale).cpu()
self.texels = texels
if len(texels.shape) >= 2:
with tf.device(pyredner.get_device_name()):
# Build a mipmap for texels
width = max(
texels.shape[0], texels.shape[1]
).value # Without value, it will have a type `Dimension`
num_levels = math.ceil(math.log(width, 2) + 1)
mipmap = tf.broadcast_to(texels, [num_levels, *texels.shape])
if len(mipmap.shape) == 3:
mipmap = tf.expand_dims(mipmap, axis=-1)
num_channels = mipmap.shape[-1]
"""NOTE: conv2d kernel axes
torch: (outchannels, in_channels / groups, kH, kW)
tf: [filter_height, filter_width, in_channels, out_channels]
"""
box_filter = tf.ones([2, 2, num_channels, 1],
dtype=tf.float32) / 4.0
# (TF) [batch, in_height, in_width, in_channels], i.e. NHWC
base_level = tf.transpose(tf.expand_dims(texels, axis=0),
perm=[0, 3, 1, 2])
mipmap = [base_level]
prev_lvl = base_level
for l in range(1, num_levels):
dilation_size = 2**(l - 1)
# Pad for circular boundary condition
# This is slow. The hope is at some point Tensorflow will support
# circular boundary condition for conv2d
desired_height = prev_lvl.shape[2] + dilation_size
while prev_lvl.shape[2] < desired_height:
prev_lvl = tf.concat([
prev_lvl, prev_lvl[:, :, 0:(desired_height -
prev_lvl.shape[2])]
], 2)
desired_width = prev_lvl.shape[3] + dilation_size
while prev_lvl.shape[3] < desired_width:
prev_lvl = tf.concat(
[prev_lvl, prev_lvl[:, :, :, 0:dilation_size]], 3)
"""NOTE: Torch conv data_format is NCHW. In Tensorflow, GPU supports
NCHW but CPU supports only NHWC. Hence, we need to convert between
NCHW and NHwC when we use CPU.
"""
"""NOTE: Current libxsmm and customized CPU implementations do
not yet support dilation rates larger than 1, i.e. we cannot use
TF Conv2DCustomBackpropInputOp
https://github.com/tensorflow/tensorflow/blob/7bc1c3c37ce4e591012f4325ab7a25ae387773c7/tensorflow/core/kernels/conv_grad_input_ops.cc#L300
"""
# if pyredner.use_gpu:
# current_lvl = tf.nn.depthwise_conv2d(
# prev_lvl,
# box_filter, # [filter_height, filter_width, in_channels, out_channels]
# dilations=[dilation_size,dilation_size],
# strides=[1,1,1,1],
# padding="VALID", # No padding
# data_format="NCHW"
# )
# else:
prev_lvl = tf.transpose(prev_lvl, perm=[0, 2, 3, 1])
current_lvl = tf.nn.depthwise_conv2d(
prev_lvl,
box_filter, # [filter_height, filter_width, in_channels, out_channels]
dilations=[dilation_size, dilation_size],
strides=[1, 1, 1, 1],
padding="VALID", # No padding
data_format="NHWC")
current_lvl = tf.transpose(current_lvl, [0, 3, 1, 2])
mipmap.append(current_lvl)
prev_lvl = current_lvl
mipmap = tf.concat(mipmap, 0)
# Convert from NCHW to NHWC
mipmap = tf.transpose(mipmap, perm=[0, 2, 3, 1])
texels = mipmap
self.mipmap = texels
self.uv_scale = uv_scale
