def generate_mipmap(self): texels = self._texels if len(texels.shape) >= 2: with tf.device(pyredner.get_device_name()): # Build a mipmap for texels width = max(texels.shape[0], texels.shape[1]) num_levels = math.ceil(math.log(width, 2) + 1) mipmap = tf.broadcast_to(texels, [num_levels, *texels.shape]) if len(mipmap.shape) == 3: mipmap = tf.expand_dims(mipmap, axis=-1) num_channels = mipmap.shape[-1] """NOTE: conv2d kernel axes torch: (outchannels, in_channels / groups, kH, kW) tf: [filter_height, filter_width, in_channels, out_channels] """ box_filter = tf.ones([2, 2, num_channels, 1], dtype=tf.float32) / 4.0 # (TF) [batch, in_height, in_width, in_channels], i.e. NHWC base_level = tf.transpose(tf.expand_dims(texels, axis=0), perm=[0, 3, 1, 2]) mipmap = [base_level] prev_lvl = base_level for l in range(1, num_levels): dilation_size = 2**(l - 1) # Pad for circular boundary condition # This is slow. The hope is at some point Tensorflow will support # circular boundary condition for conv2d desired_height = prev_lvl.shape[2] + dilation_size while prev_lvl.shape[2] < desired_height: prev_lvl = tf.concat([ prev_lvl, prev_lvl[:, :, 0:(desired_height - prev_lvl.shape[2])] ], 2) desired_width = prev_lvl.shape[3] + dilation_size while prev_lvl.shape[3] < desired_width: prev_lvl = tf.concat( [prev_lvl, prev_lvl[:, :, :, 0:dilation_size]], 3) prev_lvl = tf.transpose(prev_lvl, perm=[0, 2, 3, 1]) current_lvl = tf.nn.depthwise_conv2d( prev_lvl, box_filter, # [filter_height, filter_width, in_channels, out_channels] dilations=[dilation_size, dilation_size], strides=[1, 1, 1, 1], padding="VALID", # No padding data_format="NHWC") current_lvl = tf.transpose(current_lvl, [0, 3, 1, 2]) mipmap.append(current_lvl) prev_lvl = current_lvl mipmap = tf.concat(mipmap, 0) # Convert from NCHW to NHWC mipmap = tf.transpose(mipmap, perm=[0, 2, 3, 1]) texels = mipmap self.mipmap = texels
def generate_envmap_pdf(self): assert (tf.executing_eagerly()) values = self.values with tf.device(pyredner.get_device_name()): # Build sampling table luminance = 0.212671 * values.texels[:, :, 0] + \ 0.715160 * values.texels[:, :, 1] + \ 0.072169 * values.texels[:, :, 2] # For each y, compute CDF over x sample_cdf_xs_ = tf.cumsum(luminance, axis=1) y_weight = tf.sin( math.pi * (tf.cast(tf.range(luminance.shape[0]), tf.float32) + 0.5) / float(luminance.shape[0])) # Compute CDF for x sample_cdf_ys_ = tf.cumsum(sample_cdf_xs_[:, -1] * y_weight, axis=0) pdf_norm = (luminance.shape[0] * luminance.shape[1]) / \ (sample_cdf_ys_[-1] * (2 * math.pi * math.pi)) # Normalize to [0, 1) sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \ tf.math.maximum( sample_cdf_xs_[ :, (luminance.shape[1] - 1):luminance.shape[1]], 1e-8 * tf.convert_to_tensor(np.ones((sample_cdf_xs_.shape[0], 1)), dtype=tf.float32) ) sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \ tf.math.maximum(sample_cdf_ys_[-1], tf.constant([1e-8])) self.sample_cdf_ys = sample_cdf_ys self.sample_cdf_xs = sample_cdf_xs self.pdf_norm = pdf_norm
def serialize_texture(texture, args): if texture is None: args.append(tf.constant(0)) return args.append(tf.constant(len(texture.mipmap))) with tf.device(pyredner.get_device_name()): for mipmap in texture.mipmap: args.append(tf.identity(mipmap)) args.append(tf.identity(texture.uv_scale))
def forward(seed: int, *args): """ Forward rendering pass: given a serialized scene and output an image. """ args_ctx = unpack_args(seed, args) area_lights = args_ctx.area_lights camera = args_ctx.camera channels = args_ctx.channels envmap = args_ctx.envmap materials = args_ctx.materials num_samples = args_ctx.num_samples options = args_ctx.options resolution = args_ctx.resolution viewport = args_ctx.viewport scene = args_ctx.scene shapes = args_ctx.shapes num_channel_args = args_ctx.num_channel_args num_channels = redner.compute_num_channels( channels, scene.max_generic_texture_dimension) with tf.device(pyredner.get_device_name()): img_height = viewport[2] - viewport[0] img_width = viewport[3] - viewport[1] rendered_image = tf.zeros(shape=[img_height, img_width, num_channels], dtype=tf.float32) start = time.time() redner.render( scene, options, redner.float_ptr(pyredner.data_ptr(rendered_image)), redner.float_ptr(0), # d_rendered_image None, # d_scene redner.float_ptr(0), # translational_gradient_image redner.float_ptr(0)) # debug_image time_elapsed = time.time() - start if get_print_timing(): print('Forward pass, time: %.5f s' % time_elapsed) ctx = Context() ctx.camera = camera ctx.shapes = shapes ctx.materials = materials ctx.area_lights = area_lights ctx.envmap = envmap ctx.scene = scene ctx.options = options ctx.num_samples = num_samples ctx.num_channel_args = num_channel_args ctx.args = args # important to avoid GC on tf tensors return rendered_image, ctx
def compute_uvs(vertices, indices, print_progress = True): """ Compute UV coordinates of a given mesh using a charting algorithm with least square conformal mapping. This calls the `xatlas <https://github.com/jpcy/xatlas>`_ library. Args ==== vertices: tf.Tensor 3D position of vertices float32 tensor with size num_vertices x 3 indices: tf.Tensor vertex indices of triangle faces. int32 tensor with size num_triangles x 3 Returns ======= tf.Tensor uv vertices pool, float32 Tensor with size num_uv_vertices x 3 tf.Tensor uv indices, int32 Tensor with size num_triangles x 3 """ with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): vertices = tf.identity(vertices) indices = tf.identity(indices) uv_trimesh = redner.UVTriMesh(redner.float_ptr(pyredner.data_ptr(vertices)), redner.int_ptr(pyredner.data_ptr(indices)), redner.float_ptr(0), redner.int_ptr(0), int(vertices.shape[0]), 0, int(indices.shape[0])) atlas = redner.TextureAtlas() num_uv_vertices = redner.automatic_uv_map([uv_trimesh], atlas, print_progress)[0] uvs = tf.zeros([num_uv_vertices, 2], dtype=tf.float32) uv_indices = tf.zeros_like(indices) uv_trimesh.uvs = redner.float_ptr(pyredner.data_ptr(uvs)) uv_trimesh.uv_indices = redner.int_ptr(pyredner.data_ptr(uv_indices)) uv_trimesh.num_uv_vertices = num_uv_vertices redner.copy_texture_atlas(atlas, [uv_trimesh]) with tf.device(pyredner.get_device_name()): vertices = tf.identity(vertices) indices = tf.identity(indices) uvs = tf.identity(uvs) uv_indices = tf.identity(uv_indices) return uvs, uv_indices
def __init__(self, values, env_to_world=tf.eye(4, 4)): assert (tf.executing_eagerly()) # Convert to constant texture if necessary if tf.is_tensor(values): values = pyredner.Texture(values) # assert(values.texels.is_contiguous()) assert (values.texels.dtype == tf.float32) with tf.device(pyredner.get_device_name()): # Build sampling table luminance = 0.212671 * values.texels[:, :, 0] + \ 0.715160 * values.texels[:, :, 1] + \ 0.072169 * values.texels[:, :, 2] # For each y, compute CDF over x sample_cdf_xs_ = tf.cumsum(luminance, axis=1) y_weight = tf.sin( math.pi * (tf.cast(tf.range(luminance.shape[0].value), tf.float32) + 0.5) / float(luminance.shape[0].value)) # Compute CDF for x sample_cdf_ys_ = tf.cumsum(sample_cdf_xs_[:, -1] * y_weight, axis=0) pdf_norm = (luminance.shape[0].value * luminance.shape[1].value) / \ (sample_cdf_ys_[-1] * (2 * math.pi * math.pi)) # Normalize to [0, 1) sample_cdf_xs = (sample_cdf_xs_ - sample_cdf_xs_[:, 0:1]) / \ tf.math.maximum( sample_cdf_xs_[ :, (luminance.shape[1].value - 1):luminance.shape[1].value], 1e-8 * tf.convert_to_tensor(np.ones((sample_cdf_xs_.shape[0], 1)), dtype=tf.float32) ) sample_cdf_ys = (sample_cdf_ys_ - sample_cdf_ys_[0]) / \ tf.math.maximum(sample_cdf_ys_[-1], tf.constant([1e-8])) self.values = values self.sample_cdf_ys = sample_cdf_ys self.sample_cdf_xs = sample_cdf_xs with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): self.pdf_norm = pdf_norm.cpu() env_to_world = tf.identity(env_to_world).cpu() self.env_to_world = env_to_world self.world_to_env = tf.linalg.inv(env_to_world)
def generate_quad_light(position: tf.Tensor, look_at: tf.Tensor, size: tf.Tensor, intensity: tf.Tensor): """ Generate a pyredner.Object that is a quad light source. Args ==== position: tf.Tensor 1-d tensor of size 3 look_at: tf.Tensor 1-d tensor of size 3 size: tf.Tensor 1-d tensor of size 2 intensity: tf.Tensor 1-d tensor of size 3 Returns ======= pyredner.Object quad light source """ d = look_at - position d = d / tf.norm(d) # ONB -- generate two axes that are orthogonal to d a = 1 / (1 + d[2]) b = -d[0] * d[1] * a x = tf.where(d[2] < (-1 + 1e-6), tf.constant([0.0, -1.0, 0.0]), tf.stack([1 - d[0] * d[0] * a, b, -d[0]])) y = tf.where(d[2] < (-1 + 1e-6), tf.constant([-1.0, 0.0, 0.0]), tf.stack([b, 1 - d[1] * d[1] * a, -d[1]])) v0 = position - x * size[0] * 0.5 - y * size[1] * 0.5 v1 = position + x * size[0] * 0.5 - y * size[1] * 0.5 v2 = position - x * size[0] * 0.5 + y * size[1] * 0.5 v3 = position + x * size[0] * 0.5 + y * size[1] * 0.5 with tf.device(pyredner.get_device_name()): vertices = tf.stack((v0, v1, v2, v3), axis=0) indices = tf.constant([[0, 1, 2], [1, 3, 2]], dtype=tf.int32) m = pyredner.Material(diffuse_reflectance=tf.constant([0.0, 0.0, 0.0])) return pyredner.Object(vertices=vertices, indices=indices, material=m, light_intensity=intensity)
def generate_mipmap(self): texels = self._texels if len(texels.shape) >= 2: with tf.device(pyredner.get_device_name()): # Build a mipmap for texels width = max(texels.shape[0], texels.shape[1]) num_levels = min(math.ceil(math.log(width, 2) + 1), 8) num_channels = texels.shape[2] box_filter = tf.ones([2, 2, num_channels, 1], dtype=tf.float32) / 4.0 mipmap = [texels] # HWC -> NHWC base_level = tf.expand_dims(texels, axis=0) prev_lvl = base_level for l in range(1, num_levels): # Pad for circular boundary condition prev_lvl = tf.concat([prev_lvl, prev_lvl[:, 0:1, :, :]], 1) prev_lvl = tf.concat([prev_lvl, prev_lvl[:, :, 0:1, :]], 2) # Convolve with a box filter current_lvl = tf.nn.depthwise_conv2d( prev_lvl, box_filter, # [filter_height, filter_width, in_channels, out_channels] strides=[1, 1, 1, 1], padding='VALID', # No padding data_format='NHWC') # Downsample # tf.image.resize is too slow, so we use average pooling current_lvl = tf.nn.avg_pool2d(current_lvl, ksize=2, strides=2, padding='SAME') mipmap.append(tf.squeeze(current_lvl, axis=0)) prev_lvl = current_lvl else: mipmap = [texels] self.mipmap = mipmap
def unpack_args(seed, args, use_primary_edge_sampling=None, use_secondary_edge_sampling=None): """ Given a list of serialized scene arguments, unpack all information into a Context. """ # Unpack arguments current_index = 0 num_shapes = int(args[current_index]) current_index += 1 num_materials = int(args[current_index]) current_index += 1 num_lights = int(args[current_index]) current_index += 1 # Camera arguments cam_position = args[current_index] current_index += 1 cam_look_at = args[current_index] current_index += 1 cam_up = args[current_index] current_index += 1 cam_to_world = args[current_index] current_index += 1 world_to_cam = args[current_index] current_index += 1 intrinsic_mat_inv = args[current_index] current_index += 1 intrinsic_mat = args[current_index] current_index += 1 clip_near = float(args[current_index]) current_index += 1 resolution = args[current_index].numpy() # Tuple[int, int] current_index += 1 viewport = args[current_index].numpy() # Tuple[int, int, int, int] current_index += 1 camera_type = RednerCameraType.asCameraType( args[current_index]) # FIXME: Map to custom type current_index += 1 with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): if is_empty_tensor(cam_to_world): camera = redner.Camera( resolution[1], resolution[0], redner.float_ptr(pyredner.data_ptr(cam_position)), redner.float_ptr(pyredner.data_ptr(cam_look_at)), redner.float_ptr(pyredner.data_ptr(cam_up)), redner.float_ptr(0), # cam_to_world redner.float_ptr(0), # world_to_cam redner.float_ptr(pyredner.data_ptr(intrinsic_mat_inv)), redner.float_ptr(pyredner.data_ptr(intrinsic_mat)), clip_near, camera_type, redner.Vector2i(viewport[1], viewport[0]), redner.Vector2i(viewport[3], viewport[2])) else: camera = redner.Camera( resolution[1], resolution[0], redner.float_ptr(0), redner.float_ptr(0), redner.float_ptr(0), redner.float_ptr(pyredner.data_ptr(cam_to_world)), redner.float_ptr(pyredner.data_ptr(world_to_cam)), redner.float_ptr(pyredner.data_ptr(intrinsic_mat_inv)), redner.float_ptr(pyredner.data_ptr(intrinsic_mat)), clip_near, camera_type, redner.Vector2i(viewport[1], viewport[0]), redner.Vector2i(viewport[3], viewport[2])) with tf.device(pyredner.get_device_name()): shapes = [] for i in range(num_shapes): vertices = args[current_index] current_index += 1 indices = args[current_index] current_index += 1 uvs = args[current_index] current_index += 1 normals = args[current_index] current_index += 1 uv_indices = args[current_index] current_index += 1 normal_indices = args[current_index] current_index += 1 colors = args[current_index] current_index += 1 material_id = int(args[current_index]) current_index += 1 light_id = int(args[current_index]) current_index += 1 shapes.append(redner.Shape(\ redner.float_ptr(pyredner.data_ptr(vertices)), redner.int_ptr(pyredner.data_ptr(indices)), redner.float_ptr(pyredner.data_ptr(uvs) if not is_empty_tensor(uvs) else 0), redner.float_ptr(pyredner.data_ptr(normals) if not is_empty_tensor(normals) else 0), redner.int_ptr(pyredner.data_ptr(uv_indices) if not is_empty_tensor(uv_indices) else 0), redner.int_ptr(pyredner.data_ptr(normal_indices) if not is_empty_tensor(normal_indices) else 0), redner.float_ptr(pyredner.data_ptr(colors) if not is_empty_tensor(colors) else 0), int(vertices.shape[0]), int(uvs.shape[0]) if not is_empty_tensor(uvs) else 0, int(normals.shape[0]) if not is_empty_tensor(normals) else 0, int(indices.shape[0]), material_id, light_id)) materials = [] with tf.device(pyredner.get_device_name()): for i in range(num_materials): num_levels = int(args[current_index]) current_index += 1 diffuse_reflectance = [] for j in range(num_levels): diffuse_reflectance.append(args[current_index]) current_index += 1 diffuse_uv_scale = args[current_index] current_index += 1 num_levels = int(args[current_index]) current_index += 1 specular_reflectance = [] for j in range(num_levels): specular_reflectance.append(args[current_index]) current_index += 1 specular_uv_scale = args[current_index] current_index += 1 num_levels = int(args[current_index]) current_index += 1 roughness = [] for j in range(num_levels): roughness.append(args[current_index]) current_index += 1 roughness_uv_scale = args[current_index] current_index += 1 num_levels = int(args[current_index]) current_index += 1 generic_texture = [] if num_levels > 0: for j in range(num_levels): generic_texture.append(args[current_index]) current_index += 1 generic_uv_scale = args[current_index] current_index += 1 else: generic_uv_scale = None num_levels = int(args[current_index]) current_index += 1 normal_map = [] if num_levels > 0: for j in range(num_levels): normal_map.append(args[current_index]) current_index += 1 normal_map_uv_scale = args[current_index] current_index += 1 else: normal_map_uv_scale = None compute_specular_lighting = bool(args[current_index]) current_index += 1 two_sided = bool(args[current_index]) current_index += 1 use_vertex_color = bool(args[current_index]) current_index += 1 if get_tensor_dimension(diffuse_reflectance[0]) == 1: diffuse_reflectance = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(diffuse_reflectance[0]))], [0], [0], 3, redner.float_ptr(pyredner.data_ptr(diffuse_uv_scale))) else: assert (get_tensor_dimension(diffuse_reflectance[0]) == 3) diffuse_reflectance = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in diffuse_reflectance], [x.shape[1] for x in diffuse_reflectance], [x.shape[0] for x in diffuse_reflectance], 3, redner.float_ptr(pyredner.data_ptr(diffuse_uv_scale))) if get_tensor_dimension(specular_reflectance[0]) == 1: specular_reflectance = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(specular_reflectance[0]))], [0], [0], 3, redner.float_ptr(pyredner.data_ptr(specular_uv_scale))) else: assert (get_tensor_dimension(specular_reflectance[0]) == 3) specular_reflectance = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in specular_reflectance], [x.shape[1] for x in specular_reflectance], [x.shape[0] for x in specular_reflectance], 3, redner.float_ptr(pyredner.data_ptr(specular_uv_scale))) if get_tensor_dimension(roughness[0]) == 1: roughness = redner.Texture1(\ [redner.float_ptr(pyredner.data_ptr(roughness[0]))], [0], [0], 1, redner.float_ptr(pyredner.data_ptr(roughness_uv_scale))) else: assert (get_tensor_dimension(roughness[0]) == 3) roughness = redner.Texture1(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in roughness], [x.shape[1] for x in roughness], [x.shape[0] for x in roughness], 3, redner.float_ptr(pyredner.data_ptr(roughness_uv_scale))) if len(generic_texture) > 0: generic_texture = redner.TextureN(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in generic_texture], [x.shape[1] for x in generic_texture], [x.shape[0] for x in generic_texture], generic_texture[0].shape[2], redner.float_ptr(pyredner.data_ptr(generic_uv_scale))) else: generic_texture = redner.TextureN(\ [], [], [], 0, redner.float_ptr(0)) if len(normal_map) > 0: normal_map = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in normal_map], [x.shape[1] for x in normal_map], [x.shape[0] for x in normal_map], normal_map[0].shape[2], redner.float_ptr(pyredner.data_ptr(normal_map_uv_scale))) else: normal_map = redner.Texture3(\ [], [], [], 0, redner.float_ptr(0)) materials.append(redner.Material(\ diffuse_reflectance, specular_reflectance, roughness, generic_texture, normal_map, compute_specular_lighting, two_sided, use_vertex_color)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): area_lights = [] for i in range(num_lights): shape_id = int(args[current_index]) current_index += 1 intensity = args[current_index] current_index += 1 two_sided = bool(args[current_index]) current_index += 1 directly_visible = bool(args[current_index]) current_index += 1 area_lights.append( redner.AreaLight( shape_id, redner.float_ptr(pyredner.data_ptr(intensity)), two_sided, directly_visible)) envmap = None if not is_empty_tensor(args[current_index]): num_levels = args[current_index] current_index += 1 values = [] for j in range(num_levels): values.append(args[current_index]) current_index += 1 envmap_uv_scale = args[current_index] current_index += 1 env_to_world = args[current_index] current_index += 1 world_to_env = args[current_index] current_index += 1 sample_cdf_ys = args[current_index] current_index += 1 sample_cdf_xs = args[current_index] current_index += 1 pdf_norm = float(args[current_index]) current_index += 1 directly_visible = bool(args[current_index]) current_index += 1 assert isinstance(pdf_norm, float) with tf.device(pyredner.get_device_name()): sample_cdf_ys = redner.float_ptr(pyredner.data_ptr(sample_cdf_ys)) sample_cdf_xs = redner.float_ptr(pyredner.data_ptr(sample_cdf_xs)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): env_to_world = redner.float_ptr(pyredner.data_ptr(env_to_world)) world_to_env = redner.float_ptr(pyredner.data_ptr(world_to_env)) with tf.device(pyredner.get_device_name()): values = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in values], [x.shape[1] for x in values], # width [x.shape[0] for x in values], # height 3, # channels redner.float_ptr(pyredner.data_ptr(envmap_uv_scale))) envmap = redner.EnvironmentMap(\ values, env_to_world, world_to_env, sample_cdf_ys, sample_cdf_xs, pdf_norm, directly_visible) else: current_index += 1 # Options num_samples = args[current_index] current_index += 1 if len(num_samples.shape) == 0 or num_samples.shape[0] == 1: num_samples = int(num_samples) else: assert (num_samples.shape[0] == 2) num_samples = (int(num_samples[0]), int(num_samples[1])) max_bounces = int(args[current_index]) current_index += 1 num_channel_args = int(args[current_index]) current_index += 1 channels = [] for _ in range(num_channel_args): ch = args[current_index] ch = RednerChannels.asChannel(ch) channels.append(ch) current_index += 1 sampler_type = args[current_index] sampler_type = RednerSamplerType.asSamplerType(sampler_type) current_index += 1 use_primary_edge_sampling = args[current_index] current_index += 1 use_secondary_edge_sampling = args[current_index] current_index += 1 sample_pixel_center = args[current_index] current_index += 1 start = time.time() scene = redner.Scene(camera, shapes, materials, area_lights, envmap, pyredner.get_use_gpu(), pyredner.get_gpu_device_id(), use_primary_edge_sampling, use_secondary_edge_sampling) time_elapsed = time.time() - start if get_print_timing(): print('Scene construction, time: %.5f s' % time_elapsed) # check that num_samples is a tuple if isinstance(num_samples, int): num_samples = (num_samples, num_samples) options = redner.RenderOptions(seed, num_samples[0], max_bounces, channels, sampler_type, sample_pixel_center) ctx = Context() ctx.channels = channels ctx.options = options ctx.resolution = resolution ctx.viewport = viewport ctx.scene = scene ctx.camera = camera ctx.shapes = shapes ctx.materials = materials ctx.area_lights = area_lights ctx.envmap = envmap ctx.scene = scene ctx.options = options ctx.num_samples = num_samples ctx.num_channel_args = num_channel_args return ctx
def visualize_screen_gradient(grad_img: tf.Tensor, seed: int, scene: pyredner.Scene, num_samples: Union[int, Tuple[int, int]], max_bounces: int, channels: List = [redner.channels.radiance], sampler_type=redner.SamplerType.independent, use_primary_edge_sampling: bool = True, use_secondary_edge_sampling: bool = True, sample_pixel_center: bool = False): """ Given a serialized scene and output an 2-channel image, which visualizes the derivatives of pixel color with respect to the screen space coordinates. Args ==== grad_img: Optional[tf.Tensor] The "adjoint" of the backpropagation gradient. If you don't know what this means just give None seed: int seed for the Monte Carlo random samplers See serialize_scene for the explanation of the rest of the arguments. """ args = serialize_scene(\ scene = scene, num_samples = num_samples, max_bounces = max_bounces, sampler_type = sampler_type, channels = channels, sample_pixel_center = sample_pixel_center) args_ctx = unpack_args(\ seed, args, use_primary_edge_sampling, use_secondary_edge_sampling) channels = args_ctx.channels options = args_ctx.options resolution = args_ctx.resolution viewport = args_ctx.viewport scene = args_ctx.scene buffers = create_gradient_buffers(args_ctx) num_channels = redner.compute_num_channels( channels, scene.max_generic_texture_dimension) with tf.device(pyredner.get_device_name()): img_height = viewport[2] - viewport[0] img_width = viewport[3] - viewport[1] screen_gradient_image = tf.zeros(\ shape = [img_height, img_width, 2], dtype = tf.float32) if grad_img is not None: assert (grad_img.shape[0] == resolution[0]) assert (grad_img.shape[1] == resolution[1]) assert (grad_img.shape[2] == num_channels) else: grad_img = tf.ones(\ shape = [img_height, img_width, num_channels], dtype = tf.float32) start = time.time() redner.render( scene, options, redner.float_ptr(0), # rendered_image redner.float_ptr(pyredner.data_ptr(grad_img)), # d_rendered_image buffers.d_scene, redner.float_ptr(pyredner.data_ptr(screen_gradient_image)), redner.float_ptr(0)) # debug_image time_elapsed = time.time() - start if get_print_timing(): print('Visualize gradient, time: %.5f s' % time_elapsed) return screen_gradient_image
def backward(grad_img): global __ctx ctx = __ctx scene = ctx.scene options = ctx.options with tf.device(pyredner.get_device_name()): d_position = tf.zeros(3, dtype=tf.float32) d_look_at = tf.zeros(3, dtype=tf.float32) d_up = tf.zeros(3, dtype=tf.float32) d_ndc_to_cam = tf.zeros([3, 3], dtype=tf.float32) d_cam_to_ndc = tf.zeros([3, 3], dtype=tf.float32) d_camera = redner.DCamera( redner.float_ptr(pyredner.data_ptr(d_position)), redner.float_ptr(pyredner.data_ptr(d_look_at)), redner.float_ptr(pyredner.data_ptr(d_up)), redner.float_ptr(pyredner.data_ptr(d_ndc_to_cam)), redner.float_ptr(pyredner.data_ptr(d_cam_to_ndc))) d_vertices_list = [] d_uvs_list = [] d_normals_list = [] d_shapes = [] with tf.device(pyredner.get_device_name()): for i, shape in enumerate(ctx.shapes): num_vertices = shape.num_vertices d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32) d_uvs = tf.zeros([num_vertices, 2], dtype=tf.float32) if shape.has_uvs() else None d_normals = tf.zeros( [num_vertices, 3], dtype=tf.float32) if shape.has_normals() else None d_vertices_list.append(d_vertices) d_uvs_list.append(d_uvs) d_normals_list.append(d_normals) d_shapes.append(redner.DShape(\ redner.float_ptr(pyredner.data_ptr(d_vertices)), redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0), redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0))) d_diffuse_list = [] d_specular_list = [] d_roughness_list = [] d_normal_map_list = [] d_diffuse_uv_scale_list = [] d_specular_uv_scale_list = [] d_roughness_uv_scale_list = [] d_normal_map_uv_scale_list = [] d_materials = [] with tf.device(pyredner.get_device_name()): for material in ctx.materials: diffuse_size = material.get_diffuse_size() specular_size = material.get_specular_size() roughness_size = material.get_roughness_size() normal_map_size = material.get_normal_map_size() if diffuse_size[0] == 0: d_diffuse = tf.zeros(3, dtype=tf.float32) else: d_diffuse = tf.zeros( [diffuse_size[2], diffuse_size[1], diffuse_size[0], 3], dtype=tf.float32) if specular_size[0] == 0: d_specular = tf.zeros(3, dtype=tf.float32) else: d_specular = tf.zeros([ specular_size[2], specular_size[1], specular_size[0], 3 ], dtype=tf.float32) if roughness_size[0] == 0: d_roughness = tf.zeros(1, dtype=tf.float32) else: d_roughness = tf.zeros([ roughness_size[2], roughness_size[1], roughness_size[0], 1 ], dtype=tf.float32) # HACK: tensorflow's eager mode uses a cache to store scalar # constants to avoid memory copy. If we pass scalar tensors # into the C++ code and modify them, we would corrupt the # cache, causing incorrect result in future scalar constant # creations. Thus we force tensorflow to copy by plusing a zero # (also see https://github.com/tensorflow/tensorflow/issues/11186 # for more discussion regarding copying tensors) if d_roughness.shape.num_elements() == 1: d_roughness = d_roughness + 0 if normal_map_size[0] == 0: d_normal_map = None else: d_normal_map = tf.zeros([ normal_map_size[2], normal_map_size[1], normal_map_size[0], 3 ], dtype=tf.float32) d_diffuse_list.append(d_diffuse) d_specular_list.append(d_specular) d_roughness_list.append(d_roughness) d_normal_map_list.append(d_normal_map) d_diffuse = redner.float_ptr(pyredner.data_ptr(d_diffuse)) d_specular = redner.float_ptr(pyredner.data_ptr(d_specular)) d_roughness = redner.float_ptr(pyredner.data_ptr(d_roughness)) if normal_map_size[0] > 0: d_normal_map = redner.float_ptr( pyredner.data_ptr(d_normal_map)) d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32) d_specular_uv_scale = tf.zeros([2], dtype=tf.float32) d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32) if normal_map_size[0] > 0: d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32) else: d_normal_map_uv_scale = None d_diffuse_uv_scale_list.append(d_diffuse_uv_scale) d_specular_uv_scale_list.append(d_specular_uv_scale) d_roughness_uv_scale_list.append(d_roughness_uv_scale) d_normal_map_uv_scale_list.append(d_normal_map_uv_scale) d_diffuse_uv_scale = redner.float_ptr( pyredner.data_ptr(d_diffuse_uv_scale)) d_specular_uv_scale = redner.float_ptr( pyredner.data_ptr(d_specular_uv_scale)) d_roughness_uv_scale = redner.float_ptr( pyredner.data_ptr(d_roughness_uv_scale)) if normal_map_size[0] > 0: d_normal_map_uv_scale = redner.float_ptr( pyredner.data_ptr(d_normal_map_uv_scale)) d_diffuse_tex = redner.Texture3(\ d_diffuse, diffuse_size[0], diffuse_size[1], diffuse_size[2], d_diffuse_uv_scale) d_specular_tex = redner.Texture3(\ d_specular, specular_size[0], specular_size[1], specular_size[2], d_specular_uv_scale) d_roughness_tex = redner.Texture1(\ d_roughness, roughness_size[0], roughness_size[1], roughness_size[2], d_roughness_uv_scale) if normal_map_size[0] > 0: d_normal_map_tex = redner.Texture3(\ d_normal_map, normal_map_size[0], normal_map_size[1], normal_map_size[2], d_normal_map_uv_scale) else: d_normal_map_tex = redner.Texture3(\ redner.float_ptr(0), 0, 0, 0, redner.float_ptr(0)) d_materials.append( redner.DMaterial(d_diffuse_tex, d_specular_tex, d_roughness_tex, d_normal_map_tex)) d_intensity_list = [] d_area_lights = [] with tf.device(pyredner.get_device_name()): for light in ctx.area_lights: d_intensity = tf.zeros(3, dtype=tf.float32) d_intensity_list.append(d_intensity) d_area_lights.append(\ redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity)))) d_envmap = None if ctx.envmap is not None: envmap = ctx.envmap size = envmap.get_size() with tf.device(pyredner.get_device_name()): d_envmap_values = tf.zeros([size[2], size[1], size[0], 3], dtype=tf.float32) d_envmap_values_ptr = redner.float_ptr( pyredner.data_ptr(d_envmap_values)) d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32) d_envmap_uv_scale_ptr = redner.float_ptr( pyredner.data_ptr(d_envmap_uv_scale)) d_world_to_env = tf.zeros([4, 4], dtype=tf.float32) d_world_to_env_ptr = redner.float_ptr( pyredner.data_ptr(d_world_to_env)) d_envmap_tex = redner.Texture3(\ d_envmap_values_ptr, size[0], size[1], size[2], d_envmap_uv_scale_ptr) d_envmap = redner.DEnvironmentMap(d_envmap_tex, d_world_to_env_ptr) d_scene = redner.DScene(d_camera, d_shapes, d_materials, d_area_lights, d_envmap, pyredner.get_use_gpu(), -1) if not get_use_correlated_random_number(): # Decod_uple the forward/backward random numbers by adding a big prime number options.seed += 1000003 start = time.time() options.num_samples = ctx.num_samples[1] with tf.device(pyredner.get_device_name()): if pyredner.get_use_gpu(): grad_img = grad_img.gpu(pyredner.get_gpu_device_id()) else: grad_img = grad_img.cpu() redner.render( scene, options, redner.float_ptr(0), # rendered_image redner.float_ptr(pyredner.data_ptr(grad_img)), d_scene, redner.float_ptr(0)) # debug_image time_elapsed = time.time() - start if print_timing: print('Backward pass, time: %.5f s' % time_elapsed) # # For debugging # pyredner.imwrite(grad_img, 'grad_img.exr') # grad_img = tf.ones([256, 256, 3], dtype=tf.float32) # debug_img = tf.zeros([256, 256, 3], dtype=tf.float32) # redner.render(scene, options, # redner.float_ptr(0), # redner.float_ptr(pyredner.data_ptr(grad_img)), # d_scene, # redner.float_ptr(pyredner.data_ptr(debug_img))) # pyredner.imwrite(debug_img, 'debug.exr') # pyredner.imwrite(-debug_img, 'debug_.exr') # exit() ret_list = [] ret_list.append(None) # seed ret_list.append(None) # num_shapes ret_list.append(None) # num_materials ret_list.append(None) # num_lights ret_list.append(d_position) ret_list.append(d_look_at) ret_list.append(d_up) ret_list.append(d_ndc_to_cam) ret_list.append(d_cam_to_ndc) ret_list.append(None) # clip near ret_list.append(None) # resolution ret_list.append(None) # camera_type num_shapes = len(ctx.shapes) for i in range(num_shapes): ret_list.append(d_vertices_list[i]) ret_list.append(None) # indices ret_list.append(d_uvs_list[i]) ret_list.append(d_normals_list[i]) ret_list.append(None) # material id ret_list.append(None) # light id num_materials = len(ctx.materials) for i in range(num_materials): ret_list.append(d_diffuse_list[i]) ret_list.append(d_diffuse_uv_scale_list[i]) ret_list.append(d_specular_list[i]) ret_list.append(d_specular_uv_scale_list[i]) ret_list.append(d_roughness_list[i]) ret_list.append(d_roughness_uv_scale_list[i]) ret_list.append(d_normal_map_list[i]) ret_list.append(d_normal_map_uv_scale_list[i]) ret_list.append(None) # two sided num_area_lights = len(ctx.area_lights) for i in range(num_area_lights): ret_list.append(None) # shape id ret_list.append(d_intensity_list[i].cpu()) ret_list.append(None) # two sided if ctx.envmap is not None: ret_list.append(d_envmap_values) ret_list.append(d_envmap_uv_scale) ret_list.append(None) # env_to_world ret_list.append(d_world_to_env.cpu()) ret_list.append(None) # sample_cdf_ys ret_list.append(None) # sample_cdf_xs ret_list.append(None) # pdf_norm else: ret_list.append(None) ret_list.append(None) ret_list.append(None) ret_list.append(None) ret_list.append(None) ret_list.append(None) ret_list.append(None) ret_list.append(None) # num samples ret_list.append(None) # num bounces ret_list.append(None) # num channels for _ in range(ctx.num_channels): ret_list.append(None) # channel ret_list.append(None) # sampler type ret_list.append(None) # use_primary_edge_sampling ret_list.append(None) # use_secondary_edge_sampling # pdb.set_trace() return ret_list
def forward(seed: int, *args): """ Forward rendering pass: given a scene and output an image. """ global __ctx ctx = __ctx # Unpack arguments current_index = 0 num_shapes = int(args[current_index]) current_index += 1 num_materials = int(args[current_index]) current_index += 1 num_lights = int(args[current_index]) current_index += 1 # Camera arguments cam_position = args[current_index] current_index += 1 cam_look_at = args[current_index] current_index += 1 cam_up = args[current_index] current_index += 1 ndc_to_cam = args[current_index] current_index += 1 cam_to_ndc = args[current_index] current_index += 1 clip_near = float(args[current_index]) current_index += 1 resolution = args[current_index].numpy() # Tuple[int, int] current_index += 1 camera_type = pyredner.RednerCameraType.asCameraType( args[current_index]) # FIXME: Map to custom type current_index += 1 with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): camera = redner.Camera( resolution[1], resolution[0], redner.float_ptr(pyredner.data_ptr(cam_position)), redner.float_ptr(pyredner.data_ptr(cam_look_at)), redner.float_ptr(pyredner.data_ptr(cam_up)), redner.float_ptr(pyredner.data_ptr(ndc_to_cam)), redner.float_ptr(pyredner.data_ptr(cam_to_ndc)), clip_near, camera_type) with tf.device(pyredner.get_device_name()): shapes = [] for i in range(num_shapes): vertices = args[current_index] current_index += 1 indices = args[current_index] current_index += 1 uvs = args[current_index] current_index += 1 normals = args[current_index] current_index += 1 material_id = int(args[current_index]) current_index += 1 light_id = int(args[current_index]) current_index += 1 shapes.append(redner.Shape(\ redner.float_ptr(pyredner.data_ptr(vertices)), redner.int_ptr(pyredner.data_ptr(indices)), redner.float_ptr(pyredner.data_ptr(uvs) if uvs is not None else 0), redner.float_ptr(pyredner.data_ptr(normals) if normals is not None else 0), int(vertices.shape[0]), int(indices.shape[0]), material_id, light_id)) materials = [] with tf.device(pyredner.get_device_name()): for i in range(num_materials): diffuse_reflectance = args[current_index] current_index += 1 diffuse_uv_scale = args[current_index] current_index += 1 specular_reflectance = args[current_index] current_index += 1 specular_uv_scale = args[current_index] current_index += 1 roughness = args[current_index] current_index += 1 roughness_uv_scale = args[current_index] current_index += 1 normal_map = args[current_index] current_index += 1 normal_map_uv_scale = args[current_index] current_index += 1 two_sided = bool(args[current_index]) current_index += 1 diffuse_reflectance_ptr = redner.float_ptr( pyredner.data_ptr(diffuse_reflectance)) specular_reflectance_ptr = redner.float_ptr( pyredner.data_ptr(specular_reflectance)) roughness_ptr = redner.float_ptr(pyredner.data_ptr(roughness)) if normal_map.shape[0] > 0: normal_map_ptr = redner.float_ptr( pyredner.data_ptr(normal_map)) diffuse_uv_scale_ptr = redner.float_ptr( pyredner.data_ptr(diffuse_uv_scale)) specular_uv_scale_ptr = redner.float_ptr( pyredner.data_ptr(specular_uv_scale)) roughness_uv_scale_ptr = redner.float_ptr( pyredner.data_ptr(roughness_uv_scale)) if normal_map.shape[0] > 0: normal_map_uv_scale_ptr = redner.float_ptr( pyredner.data_ptr(normal_map_uv_scale)) if get_tensor_dimension(diffuse_reflectance) == 1: diffuse_reflectance = redner.Texture3(diffuse_reflectance_ptr, 0, 0, 0, diffuse_uv_scale_ptr) else: diffuse_reflectance = redner.Texture3(\ diffuse_reflectance_ptr, int(diffuse_reflectance.shape[2]), # width int(diffuse_reflectance.shape[1]), # height int(diffuse_reflectance.shape[0]), # num levels diffuse_uv_scale_ptr) if get_tensor_dimension(specular_reflectance) == 1: specular_reflectance = redner.Texture3( specular_reflectance_ptr, 0, 0, 0, specular_uv_scale_ptr) else: specular_reflectance = redner.Texture3(\ specular_reflectance_ptr, int(specular_reflectance.shape[2]), # width int(specular_reflectance.shape[1]), # height int(specular_reflectance.shape[0]), # num levels specular_uv_scale_ptr) if get_tensor_dimension(roughness) == 1: roughness = redner.Texture1(roughness_ptr, 0, 0, 0, roughness_uv_scale_ptr) else: assert (get_tensor_dimension(roughness) == 4) roughness = redner.Texture1(\ roughness_ptr, int(roughness.shape[2]), # width int(roughness.shape[1]), # height int(roughness.shape[0]), # num levels roughness_uv_scale_ptr) if normal_map.shape[0] > 0: normal_map = redner.Texture3(\ normal_map_ptr, int(normal_map.shape[2]), int(normal_map.shape[1]), int(normal_map.shape[0]), normal_map_uv_scale_ptr) else: normal_map = redner.Texture3(\ redner.float_ptr(0), 0, 0, 0, redner.float_ptr(0)) materials.append(redner.Material(\ diffuse_reflectance, specular_reflectance, roughness, normal_map, two_sided)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): area_lights = [] for i in range(num_lights): shape_id = int(args[current_index]) current_index += 1 intensity = args[current_index] current_index += 1 two_sided = bool(args[current_index]) current_index += 1 area_lights.append( redner.AreaLight( shape_id, redner.float_ptr(pyredner.data_ptr(intensity)), two_sided)) envmap = None if not is_empty_tensor(args[current_index]): values = args[current_index] current_index += 1 envmap_uv_scale = args[current_index] current_index += 1 env_to_world = args[current_index] current_index += 1 world_to_env = args[current_index] current_index += 1 sample_cdf_ys = args[current_index] current_index += 1 sample_cdf_xs = args[current_index] current_index += 1 pdf_norm = float(args[current_index]) current_index += 1 assert isinstance(pdf_norm, float) with tf.device(pyredner.get_device_name()): values_ptr = redner.float_ptr(pyredner.data_ptr(values)) sample_cdf_ys = redner.float_ptr(pyredner.data_ptr(sample_cdf_ys)) sample_cdf_xs = redner.float_ptr(pyredner.data_ptr(sample_cdf_xs)) envmap_uv_scale = redner.float_ptr( pyredner.data_ptr(envmap_uv_scale)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): env_to_world = redner.float_ptr(pyredner.data_ptr(env_to_world)) world_to_env = redner.float_ptr(pyredner.data_ptr(world_to_env)) values = redner.Texture3( values_ptr, int(values.shape[2]), # width int(values.shape[1]), # height int(values.shape[0]), # num levels envmap_uv_scale) envmap = redner.EnvironmentMap(\ values, env_to_world, world_to_env, sample_cdf_ys, sample_cdf_xs, pdf_norm) else: current_index += 7 # Options num_samples = int(args[current_index]) current_index += 1 max_bounces = int(args[current_index]) current_index += 1 __num_channels = int(args[current_index]) current_index += 1 channels = [] for _ in range(__num_channels): ch = args[current_index] ch = pyredner.RednerChannels.asChannel(ch) channels.append(ch) current_index += 1 sampler_type = args[current_index] sampler_type = pyredner.RednerSamplerType.asSamplerType(sampler_type) current_index += 1 use_primary_edge_sampling = args[current_index] current_index += 1 use_secondary_edge_sampling = args[current_index] current_index += 1 scene = redner.Scene(camera, shapes, materials, area_lights, envmap, pyredner.get_use_gpu(), pyredner.get_gpu_device_id(), use_primary_edge_sampling, use_secondary_edge_sampling) # check that num_samples is a tuple if isinstance(num_samples, int): num_samples = (num_samples, num_samples) options = redner.RenderOptions(seed, num_samples[0], max_bounces, channels, sampler_type) num_channels = redner.compute_num_channels(channels) with tf.device(pyredner.get_device_name()): rendered_image = tf.zeros( shape=[resolution[0], resolution[1], num_channels], dtype=tf.float32) start = time.time() # pdb.set_trace() redner.render(scene, options, redner.float_ptr(pyredner.data_ptr(rendered_image)), redner.float_ptr(0), None, redner.float_ptr(0)) time_elapsed = time.time() - start if print_timing: print('Forward pass, time: %.5f s' % time_elapsed) # # For debugging # debug_img = tf.zeros((256, 256, 3), dtype=tf.float32) # redner.render(scene, # options, # redner.float_ptr(pyredner.data_ptr(rendered_image)), # redner.float_ptr(0), # None, # redner.float_ptr(pyredner.data_ptr(debug_img))) # pyredner.imwrite(debug_img, 'debug.png') # exit() # import pdb; pdb.set_trace() ctx.shapes = shapes ctx.materials = materials ctx.area_lights = area_lights ctx.envmap = envmap ctx.scene = scene ctx.options = options ctx.num_samples = num_samples ctx.num_channels = __num_channels return rendered_image
look_at = tf.Variable([0.0, 0.0, 0.0], dtype=tf.float32) up = tf.Variable([0.0, 1.0, 0.0], dtype=tf.float32) fov = tf.Variable([45.0], dtype=tf.float32) clip_near = 1e-2 resolution = (256, 256) cam = pyredner.Camera(position=position, look_at=look_at, up=up, fov=fov, clip_near=clip_near, resolution=resolution) mat_perlin = pyredner.Material(diffuse_reflectance=diffuse, specular_reflectance=specular, roughness=roughness) with tf.device(pyredner.get_device_name()): mat_black = pyredner.Material( diffuse_reflectance=tf.Variable([0.0, 0.0, 0.0], dtype=tf.float32)) materials = [mat_perlin, mat_black] vertices = tf.Variable([[-1.5, -1.5, 0.0], [-1.5, 1.5, 0.0], [1.5, -1.5, 0.0], [1.5, 1.5, 0.0]], dtype=tf.float32) indices = tf.constant([[0, 1, 2], [1, 3, 2]], dtype=tf.int32) uvs = tf.Variable([[0.05, 0.05], [0.05, 0.95], [0.95, 0.05], [0.95, 0.95]], dtype=tf.float32) shape_plane = pyredner.Shape(vertices, indices, 0, uvs) light_vertices = tf.Variable([[-1.0, -1.0, -7.0], [1.0, -1.0, -7.0], [-1.0, 1.0, -7.0], [1.0, 1.0, -7.0]], dtype=tf.float32) light_indices = tf.constant([[0, 1, 2], [1, 3, 2]], dtype=tf.int32) shape_light = pyredner.Shape(light_vertices, light_indices, 1)
def serialize_scene(scene: pyredner.Scene, num_samples: Union[int, Tuple[int, int]], max_bounces: int, channels=[redner.channels.radiance], sampler_type=redner.SamplerType.independent, use_primary_edge_sampling=True, use_secondary_edge_sampling=True) -> List: """ Given a pyredner scene & rendering options, convert them to a linear list of argument, so that we can use it in PyTorch. Args ==== scene: pyredner.Scene num_samples: int number of samples per pixel for forward and backward passes can be an integer or a tuple of 2 integers if a single integer is provided, use the same number of samples for both max_bounces: int number of bounces for global illumination 1 means direct lighting only channels: List[redner.channels] | A list of channels that should present in the output image | following channels are supported\: | redner.channels.radiance, | redner.channels.alpha, | redner.channels.depth, | redner.channels.position, | redner.channels.geometry_normal, | redner.channels.shading_normal, | redner.channels.uv, | redner.channels.diffuse_reflectance, | redner.channels.specular_reflectance, | redner.channels.vertex_color, | redner.channels.roughness, | redner.channels.generic_texture, | redner.channels.shape_id, | redner.channels.material_id | all channels, except for shape id and material id, are differentiable sampler_type: redner.SamplerType | Which sampling pattern to use? | see `Chapter 7 of the PBRT book <http://www.pbr-book.org/3ed-2018/Sampling_and_Reconstruction.html>` for an explanation of the difference between different samplers. | Following samplers are supported: | redner.SamplerType.independent | redner.SamplerType.sobol use_primary_edge_sampling: bool use_secondary_edge_sampling: bool """ cam = scene.camera num_shapes = len(scene.shapes) num_materials = len(scene.materials) num_lights = len(scene.area_lights) num_channels = len(channels) for light_id, light in enumerate(scene.area_lights): scene.shapes[light.shape_id].light_id = light_id args = [] args.append(tf.constant(num_shapes)) args.append(tf.constant(num_materials)) args.append(tf.constant(num_lights)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): if cam.position is None: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) else: args.append(tf.identity(cam.position)) args.append(tf.identity(cam.look_at)) args.append(tf.identity(cam.up)) if cam.cam_to_world is None: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) else: args.append(tf.identity(cam.cam_to_world)) args.append(tf.identity(cam.world_to_cam)) args.append(tf.identity(cam.intrinsic_mat_inv)) args.append(tf.identity(cam.intrinsic_mat)) args.append(tf.constant(cam.clip_near)) args.append(tf.constant(cam.resolution)) args.append(RednerCameraType.asTensor(cam.camera_type)) for shape in scene.shapes: with tf.device(pyredner.get_device_name()): args.append(tf.identity(shape.vertices)) args.append(tf.identity(shape.indices)) if shape.uvs is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.uvs)) if shape.normals is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.normals)) if shape.uv_indices is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.uv_indices)) if shape.normal_indices is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.normal_indices)) if shape.colors is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.colors)) args.append(tf.constant(shape.material_id)) args.append(tf.constant(shape.light_id)) for material in scene.materials: with tf.device(pyredner.get_device_name()): args.append(tf.identity(material.diffuse_reflectance.mipmap)) args.append(tf.identity(material.diffuse_reflectance.uv_scale)) args.append(tf.identity(material.specular_reflectance.mipmap)) args.append(tf.identity(material.specular_reflectance.uv_scale)) args.append(tf.identity(material.roughness.mipmap)) args.append(tf.identity(material.roughness.uv_scale)) if material.generic_texture is not None: args.append(tf.identity(material.generic_texture.mipmap)) args.append(tf.identity(material.generic_texture.uv_scale)) else: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) if material.normal_map is not None: args.append(tf.identity(material.normal_map.mipmap)) args.append(tf.identity(material.normal_map.uv_scale)) else: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(tf.constant(material.compute_specular_lighting)) args.append(tf.constant(material.two_sided)) args.append(tf.constant(material.use_vertex_color)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): for light in scene.area_lights: args.append(tf.constant(light.shape_id)) args.append(tf.identity(light.intensity)) args.append(tf.constant(light.two_sided)) if scene.envmap is not None: with tf.device(pyredner.get_device_name()): args.append(tf.identity(scene.envmap.values.mipmap)) args.append(tf.identity(scene.envmap.values.uv_scale)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): args.append(tf.identity(scene.envmap.env_to_world)) args.append(tf.identity(scene.envmap.world_to_env)) with tf.device(pyredner.get_device_name()): args.append(tf.identity(scene.envmap.sample_cdf_ys)) args.append(tf.identity(scene.envmap.sample_cdf_xs)) args.append(scene.envmap.pdf_norm) else: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(tf.constant(num_samples)) args.append(tf.constant(max_bounces)) args.append(tf.constant(num_channels)) for ch in channels: args.append(RednerChannels.asTensor(ch)) args.append(RednerSamplerType.asTensor(sampler_type)) args.append(tf.constant(use_primary_edge_sampling)) args.append(tf.constant(use_secondary_edge_sampling)) return args
def serialize_scene(scene: pyredner.Scene, num_samples: Union[int, Tuple[int, int]], max_bounces: int, channels=[redner.channels.radiance], sampler_type=redner.SamplerType.independent, use_primary_edge_sampling=True, use_secondary_edge_sampling=True, sample_pixel_center: bool = False) -> List: """ Given a pyredner scene & rendering options, convert them to a linear list of argument, so that we can use it in TensorFlow. Args ==== scene: pyredner.Scene num_samples: int number of samples per pixel for forward and backward passes can be an integer or a tuple of 2 integers if a single integer is provided, use the same number of samples for both max_bounces: int number of bounces for global illumination 1 means direct lighting only channels: List[redner.channels] | A list of channels that should present in the output image | following channels are supported\: | redner.channels.radiance, | redner.channels.alpha, | redner.channels.depth, | redner.channels.position, | redner.channels.geometry_normal, | redner.channels.shading_normal, | redner.channels.uv, | redner.channels.diffuse_reflectance, | redner.channels.specular_reflectance, | redner.channels.vertex_color, | redner.channels.roughness, | redner.channels.generic_texture, | redner.channels.shape_id, | redner.channels.triangle_id, | redner.channels.material_id | all channels, except for shape id, triangle id and material id, are differentiable sampler_type: redner.SamplerType | Which sampling pattern to use? | see `Chapter 7 of the PBRT book <http://www.pbr-book.org/3ed-2018/Sampling_and_Reconstruction.html>` for an explanation of the difference between different samplers. | Following samplers are supported: | redner.SamplerType.independent | redner.SamplerType.sobol use_primary_edge_sampling: bool use_secondary_edge_sampling: bool sample_pixel_center: bool Always sample at the pixel center when rendering. This trades noise with aliasing. If this option is activated, the rendering becomes non-differentiable (since there is no antialiasing integral), and redner's edge sampling becomes an approximation to the gradients of the aliased rendering. """ # TODO: figure out a way to determine whether a TF tensor requires gradient or not cam = scene.camera num_shapes = len(scene.shapes) num_materials = len(scene.materials) num_lights = len(scene.area_lights) num_channels = len(channels) for light_id, light in enumerate(scene.area_lights): scene.shapes[light.shape_id].light_id = light_id if max_bounces == 0: use_secondary_edge_sampling = False args = [] args.append(tf.constant(num_shapes)) args.append(tf.constant(num_materials)) args.append(tf.constant(num_lights)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): if cam.position is None: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) else: args.append(tf.identity(cam.position)) args.append(tf.identity(cam.look_at)) args.append(tf.identity(cam.up)) if cam.cam_to_world is None: args.append(__EMPTY_TENSOR) args.append(__EMPTY_TENSOR) else: args.append(tf.identity(cam.cam_to_world)) args.append(tf.identity(cam.world_to_cam)) args.append(tf.identity(cam.intrinsic_mat_inv)) args.append(tf.identity(cam.intrinsic_mat)) args.append(tf.constant(cam.clip_near)) args.append(tf.constant(cam.resolution)) viewport = cam.viewport if viewport is None: viewport = (0, 0, cam.resolution[0], cam.resolution[1]) # Clamp the viewport if necessary viewport = (max(viewport[0], 0), max(viewport[1], 0), min(viewport[2], cam.resolution[0]), min(viewport[3], cam.resolution[1])) args.append(tf.constant(viewport)) args.append(RednerCameraType.asTensor(cam.camera_type)) for shape in scene.shapes: with tf.device(pyredner.get_device_name()): args.append(tf.identity(shape.vertices)) # HACK: tf.bitcast forces tensorflow to copy int32 to GPU memory. # tf.identity stopped working since TF 2.1 (if you print the device # it will say it's on GPU, but the address returned by data_ptr is wrong). # Hopefully TF people will fix this in the future. args.append(tf.bitcast(shape.indices, type=tf.int32)) if shape.uvs is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.uvs)) if shape.normals is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.normals)) if shape.uv_indices is None: args.append(__EMPTY_TENSOR) else: args.append(tf.bitcast(shape.uv_indices, type=tf.int32)) if shape.normal_indices is None: args.append(__EMPTY_TENSOR) else: args.append(tf.bitcast(shape.normal_indices, type=tf.int32)) if shape.colors is None: args.append(__EMPTY_TENSOR) else: args.append(tf.identity(shape.colors)) args.append(tf.constant(shape.material_id)) args.append(tf.constant(shape.light_id)) for material in scene.materials: serialize_texture(material.diffuse_reflectance, args) serialize_texture(material.specular_reflectance, args) serialize_texture(material.roughness, args) serialize_texture(material.generic_texture, args) serialize_texture(material.normal_map, args) args.append(tf.constant(material.compute_specular_lighting)) args.append(tf.constant(material.two_sided)) args.append(tf.constant(material.use_vertex_color)) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): for light in scene.area_lights: args.append(tf.constant(light.shape_id)) args.append(tf.identity(light.intensity)) args.append(tf.constant(light.two_sided)) args.append(tf.constant(light.directly_visible)) if scene.envmap is not None: serialize_texture(scene.envmap.values, args) with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): args.append(tf.identity(scene.envmap.env_to_world)) args.append(tf.identity(scene.envmap.world_to_env)) with tf.device(pyredner.get_device_name()): args.append(tf.identity(scene.envmap.sample_cdf_ys)) args.append(tf.identity(scene.envmap.sample_cdf_xs)) args.append(scene.envmap.pdf_norm) args.append(scene.envmap.directly_visible) else: args.append(__EMPTY_TENSOR) args.append(tf.constant(num_samples)) args.append(tf.constant(max_bounces)) args.append(tf.constant(num_channels)) for ch in channels: args.append(RednerChannels.asTensor(ch)) args.append(RednerSamplerType.asTensor(sampler_type)) args.append(tf.constant(use_primary_edge_sampling)) args.append(tf.constant(use_secondary_edge_sampling)) args.append(tf.constant(sample_pixel_center)) return args
def load_obj(filename: str, obj_group: bool = True, flip_tex_coords: bool = True, use_common_indices: bool = False, return_objects: bool = False): """ Load from a Wavefront obj file as PyTorch tensors. Args ==== obj_group: bool split the meshes based on materials flip_tex_coords: bool flip the v coordinate of uv by applying v' = 1 - v use_common_indices: bool Use the same indices for position, uvs, normals. Not recommended since texture seams in the objects sharing the same positions would cause the optimization to "tear" the object return_objects: bool Output list of Object instead. If there is no corresponding material for a shape, assign a grey material. Returns ======= if return_objects == True, return a list of Object if return_objects == False, return (material_map, mesh_list, light_map), material_map -> Map[mtl_name, WavefrontMaterial] mesh_list -> List[TriangleMesh] light_map -> Map[mtl_name, torch.Tensor] """ if device_name is None: device_name = pyredner.get_device_name() vertices_pool = [] uvs_pool = [] normals_pool = [] indices = [] uv_indices = [] normal_indices = [] vertices = [] uvs = [] normals = [] vertices_map = {} uvs_map = {} normals_map = {} material_map = {} current_mtllib = {} current_material_name = None def create_mesh(indices, uv_indices, normal_indices, vertices, uvs, normals): indices = tf.constant(indices, dtype=tf.int32) if len(uv_indices) == 0: uv_indices = None else: uv_indices = tf.constant(uv_indices, dtype=tf.int32) if len(normal_indices) == 0: normal_indices = None else: normal_indices = tf.constant(normal_indices, dtype=tf.int32) vertices = tf.constant(vertices) if len(uvs) == 0: uvs = None else: uvs = tf.constant(uvs) if len(normals) == 0: normals = None else: normals = tf.constant(normals) return TriangleMesh(indices, uv_indices, normal_indices, vertices, uvs, normals) mesh_list = [] light_map = {} with open(filename, 'r') as f: d = os.path.dirname(filename) cwd = os.getcwd() if d != '': os.chdir(d) for line in f: line = line.strip() splitted = re.split('\ +', line) if splitted[0] == 'mtllib': current_mtllib = load_mtl(splitted[1]) elif splitted[0] == 'usemtl': if len(indices) > 0 and obj_group is True: # Flush mesh_list.append( (current_material_name, create_mesh(indices, uv_indices, normal_indices, vertices, uvs, normals))) indices = [] uv_indices = [] normal_indices = [] vertices = [] normals = [] uvs = [] vertices_map = {} uvs_map = {} normals_map = {} mtl_name = splitted[1] current_material_name = mtl_name if mtl_name not in material_map: m = current_mtllib[mtl_name] if m.map_Kd is None: diffuse_reflectance = tf.constant(m.Kd, dtype=tf.float32) else: diffuse_reflectance = pyredner.imread(m.map_Kd) if m.map_Ks is None: specular_reflectance = tf.constant(m.Ks, dtype=tf.float32) else: specular_reflectance = pyredner.imread(m.map_Ks) if m.map_Ns is None: roughness = tf.constant([2.0 / (m.Ns + 2.0)], dtype=tf.float32) else: roughness = 2.0 / (pyredner.imread(m.map_Ks) + 2.0) if m.Ke != (0.0, 0.0, 0.0): light_map[mtl_name] = tf.constant(m.Ke, dtype=tf.float32) material_map[mtl_name] = pyredner.Material( diffuse_reflectance, specular_reflectance, roughness) elif splitted[0] == 'v': vertices_pool.append([ float(splitted[1]), float(splitted[2]), float(splitted[3]) ]) elif splitted[0] == 'vt': u = float(splitted[1]) v = float(splitted[2]) if flip_tex_coords: v = 1 - v uvs_pool.append([u, v]) elif splitted[0] == 'vn': normals_pool.append([ float(splitted[1]), float(splitted[2]), float(splitted[3]) ]) elif splitted[0] == 'f': def num_indices(x): return len(re.split('/', x)) def get_index(x, i): return int(re.split('/', x)[i]) def parse_face_index(x, i): f = get_index(x, i) if f > 0: f -= 1 return f assert (len(splitted) <= 5) def get_vertex_id(indices): pi = parse_face_index(indices, 0) uvi = None if (num_indices(indices) > 1 and re.split('/', indices)[1] != ''): uvi = parse_face_index(indices, 1) ni = None if (num_indices(indices) > 2 and re.split('/', indices)[2] != ''): ni = parse_face_index(indices, 2) if use_common_indices: # vertex, uv, normals share the same indexing key = (pi, uvi, ni) if key in vertices_map: vertex_id = vertices_map[key] return vertex_id, vertex_id, vertex_id vertex_id = len(vertices) vertices_map[key] = vertex_id vertices.append(vertices_pool[pi]) if uvi is not None: uvs.append(uvs_pool[uvi]) if ni is not None: normals.append(normals_pool[ni]) return vertex_id, vertex_id, vertex_id else: # vertex, uv, normals use separate indexing vertex_id = None uv_id = None normal_id = None if pi in vertices_map: vertex_id = vertices_map[pi] else: vertex_id = len(vertices) vertices.append(vertices_pool[pi]) vertices_map[pi] = vertex_id if uvi is not None: if uvi in uvs_map: uv_id = uvs_map[uvi] else: uv_id = len(uvs) uvs.append(uvs_pool[uvi]) uvs_map[uvi] = uv_id if ni is not None: if ni in normals_map: normal_id = normals_map[ni] else: normal_id = len(normals) normals.append(normals_pool[ni]) normals_map[ni] = normal_id return vertex_id, uv_id, normal_id vid0, uv_id0, n_id0 = get_vertex_id(splitted[1]) vid1, uv_id1, n_id1 = get_vertex_id(splitted[2]) vid2, uv_id2, n_id2 = get_vertex_id(splitted[3]) indices.append([vid0, vid1, vid2]) if uv_id0 is not None: assert (uv_id1 is not None and uv_id2 is not None) uv_indices.append([uv_id0, uv_id1, uv_id2]) if n_id0 is not None: assert (n_id1 is not None and n_id2 is not None) normal_indices.append([n_id0, n_id1, n_id2]) if (len(splitted) == 5): vid3, uv_id3, n_id3 = get_vertex_id(splitted[4]) indices.append([vid0, vid2, vid3]) if uv_id0 is not None: assert (uv_id3 is not None) uv_indices.append([uv_id0, uv_id2, uv_id3]) if n_id0 is not None: assert (n_id3 is not None) normal_indices.append([n_id0, n_id2, n_id3]) mesh_list.append((current_material_name, create_mesh(indices, uv_indices, normal_indices, vertices, uvs, normals))) if d != '': os.chdir(cwd) if return_objects: objects = [] for mtl_name, mesh in mesh_list: if mtl_name in material_map: m = material_map[mtl_name] else: m = pyredner.Material(diffuse_reflectance = \ tf.constant((0.5, 0.5, 0.5))) if mtl_name in light_map: l = light_map[mtl_name] else: l = None objects.append(pyredner.Object(\ vertices = mesh.vertices, indices = mesh.indices, material = m, light_intensity = l, uvs = mesh.uvs, normals = mesh.normals, uv_indices = mesh.uv_indices, normal_indices = mesh.normal_indices)) return objects else: return material_map, mesh_list, light_map
def generate_geometry_image(size: int): """ Generate an spherical geometry image [Gu et al. 2002 and Praun and Hoppe 2003] of size [2 * size + 1, 2 * size + 1]. This can be used for encoding a genus-0 surface into a regular image, so that it is more convienent for a CNN to process. The topology is given by a tesselated octahedron. UV is given by the spherical mapping. Duplicated vertex are mapped to the one with smaller index (so some vertices on the geometry image is unused by the indices). Args ==== size: int size of the geometry image Returns ======= tf.Tensor vertices of size [(2 * size + 1 * 2 * size + 1), 3] tf.Tensor indices of size [2 * (2 * size + 1 * 2 * size + 1), 3] tf.Tensor uvs of size [(2 * size + 1 * 2 * size + 1), 2] """ size *= 2 # Generate vertices and uv by going through each vertex. left_top = np.array([0.0, 0.0, 1.0]) top = np.array([0.0, 1.0, 0.0]) right_top = np.array([0.0, 0.0, 1.0]) left = np.array([-1.0, 0.0, 0.0]) middle = np.array([0.0, 0.0, -1.0]) right = np.array([1.0, 0.0, 0.0]) left_bottom = np.array([0.0, 0.0, 1.0]) bottom = np.array([0.0, -1.0, 0.0]) right_bottom = np.array([0.0, 0.0, 1.0]) vertices = np.zeros([(size + 1) * (size + 1), 3]) uvs = np.zeros([(size + 1) * (size + 1), 2]) vertex_id = 0 half_size = size / 2.0 for i in range(size + 1): # height for j in range(size + 1): # width # Left Top if i + j <= half_size: org = left_top i_axis = left - left_top j_axis = top - left_top i_ = float(i) / half_size j_ = float(j) / half_size elif (i + j >= half_size and i <= half_size and j <= half_size): org = middle i_axis = top - middle j_axis = left - middle i_ = 1.0 - float(i) / half_size j_ = 1.0 - float(j) / half_size # Right Top elif ((half_size - i + j - half_size) <= half_size and i <= half_size and j >= half_size): org = middle i_axis = top - middle j_axis = right - middle i_ = 1.0 - float(i) / half_size j_ = float(j) / half_size - 1.0 elif ((i + size - j) <= half_size): org = right_top i_axis = right - right_top j_axis = top - right_top i_ = float(i) / half_size j_ = 2.0 - float(j) / half_size # Left Bottom elif ((i - half_size + half_size - j) <= half_size and i >= half_size and j <= half_size): org = middle i_axis = bottom - middle j_axis = left - middle i_ = float(i) / half_size - 1.0 j_ = 1.0 - float(j) / half_size elif ((size - i + j) <= half_size): org = left_bottom i_axis = left - left_bottom j_axis = bottom - left_bottom i_ = 2.0 - float(i) / half_size j_ = float(j) / half_size # Right Bottom elif ((i - half_size + j - half_size) <= half_size and i >= half_size and j >= half_size): org = middle i_axis = bottom - middle j_axis = right - middle i_ = float(i) / half_size - 1.0 j_ = float(j) / half_size - 1.0 else: org = right_bottom i_axis = right - right_bottom j_axis = bottom - right_bottom i_ = 2.0 - float(i) / half_size j_ = 2.0 - float(j) / half_size p = org + i_ * i_axis + j_ * j_axis vertices[vertex_id, :] = p / np.linalg.norm(p) # Spherical UV mapping u = 0.5 + math.atan2(float(p[2]), float(p[0])) / (2 * math.pi) v = 0.5 - math.asin(float(p[1])) / math.pi uvs[vertex_id, :] = np.array([u, v]) vertex_id += 1 # Generate indices by going through each triangle. # Duplicated vertex are mapped to the one with smaller index. indices = [] for i in range(size): # height for j in range(size): # width left_top = i * (size + 1) + j right_top = i * (size + 1) + j + 1 left_bottom = (i + 1) * (size + 1) + j right_bottom = (i + 1) * (size + 1) + j + 1 # Wrap rule for octahedron topology if i == 0 and j >= half_size: if j > half_size: left_top = i * (size + 1) + size - j right_top = i * (size + 1) + (size - (j + 1)) elif i == size - 1 and j >= half_size: if j > half_size: left_bottom = (i + 1) * (size + 1) + size - j right_bottom = (i + 1) * (size + 1) + (size - (j + 1)) if j == size - 1: right_bottom = 0 elif j == 0 and i >= half_size: if i > half_size: left_top = (size - i) * (size + 1) + j left_bottom = (size - (i + 1)) * (size + 1) + j elif j == size - 1 and i >= half_size: if i > half_size: right_top = (size - i) * (size + 1) + j + 1 right_bottom = (size - (i + 1)) * (size + 1) + j + 1 # Left Top if i < half_size and j < half_size: indices.append((left_top, left_bottom, right_top)) indices.append((right_top, left_bottom, right_bottom)) # Right Top elif i < half_size and j >= half_size: indices.append((left_top, left_bottom, right_bottom)) indices.append((left_top, right_bottom, right_top)) # Left Bottom elif i >= half_size and j < half_size: indices.append((left_top, right_bottom, right_top)) indices.append((left_top, left_bottom, right_bottom)) # Right Bottom else: indices.append((left_top, left_bottom, right_top)) indices.append((right_top, left_bottom, right_bottom)) with tf.device(pyredner.get_device_name()): vertices = tf.constant(vertices, dtype=tf.float32) uvs = tf.constant(uvs, dtype=tf.float32) indices = tf.constant(indices, dtype=tf.int32) return vertices, indices, uvs
def create_gradient_buffers(ctx): scene = ctx.scene options = ctx.options camera = ctx.camera buffers = Context() with tf.device(pyredner.get_device_name()): if camera.use_look_at: buffers.d_position = tf.zeros(3, dtype=tf.float32) buffers.d_look_at = tf.zeros(3, dtype=tf.float32) buffers.d_up = tf.zeros(3, dtype=tf.float32) buffers.d_cam_to_world = None buffers.d_world_to_cam = None else: buffers.d_position = None buffers.d_look_at = None buffers.d_up = None buffers.d_cam_to_world = tf.zeros([4, 4], dtype=tf.float32) buffers.d_world_to_cam = tf.zeros([4, 4], dtype=tf.float32) buffers.d_intrinsic_mat_inv = tf.zeros([3, 3], dtype=tf.float32) buffers.d_intrinsic_mat = tf.zeros([3, 3], dtype=tf.float32) if camera.use_look_at: buffers.d_camera = redner.DCamera(\ redner.float_ptr(pyredner.data_ptr(buffers.d_position)), redner.float_ptr(pyredner.data_ptr(buffers.d_look_at)), redner.float_ptr(pyredner.data_ptr(buffers.d_up)), redner.float_ptr(0), # cam_to_world redner.float_ptr(0), # world_to_cam redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat_inv)), redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat))) else: buffers.d_camera = redner.DCamera(\ redner.float_ptr(0), redner.float_ptr(0), redner.float_ptr(0), redner.float_ptr(pyredner.data_ptr(buffers.d_cam_to_world)), redner.float_ptr(pyredner.data_ptr(buffers.d_world_to_cam)), redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat_inv)), redner.float_ptr(pyredner.data_ptr(buffers.d_intrinsic_mat))) buffers.d_vertices_list = [] buffers.d_uvs_list = [] buffers.d_normals_list = [] buffers.d_colors_list = [] buffers.d_shapes = [] with tf.device(pyredner.get_device_name()): for i, shape in enumerate(ctx.shapes): num_vertices = shape.num_vertices d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32) d_uvs = tf.zeros([num_vertices, 2], dtype=tf.float32) if shape.has_uvs() else None d_normals = tf.zeros( [num_vertices, 3], dtype=tf.float32) if shape.has_normals() else None d_colors = tf.zeros( [num_vertices, 3], dtype=tf.float32) if shape.has_colors() else None buffers.d_vertices_list.append(d_vertices) buffers.d_uvs_list.append(d_uvs) buffers.d_normals_list.append(d_normals) buffers.d_colors_list.append(d_colors) buffers.d_shapes.append(redner.DShape(\ redner.float_ptr(pyredner.data_ptr(d_vertices)), redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0), redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0), redner.float_ptr(pyredner.data_ptr(d_colors) if d_colors is not None else 0))) buffers.d_diffuse_list = [] buffers.d_specular_list = [] buffers.d_roughness_list = [] buffers.d_normal_map_list = [] buffers.d_diffuse_uv_scale_list = [] buffers.d_specular_uv_scale_list = [] buffers.d_roughness_uv_scale_list = [] buffers.d_generic_list = [] buffers.d_generic_uv_scale_list = [] buffers.d_normal_map_uv_scale_list = [] buffers.d_materials = [] with tf.device(pyredner.get_device_name()): for material in ctx.materials: if material.get_diffuse_size(0)[0] == 0: d_diffuse = [tf.zeros(3, dtype=tf.float32)] else: d_diffuse = [] for l in range(material.get_diffuse_levels()): diffuse_size = material.get_diffuse_size(l) d_diffuse.append(\ tf.zeros([diffuse_size[1], diffuse_size[0], 3], dtype=tf.float32)) if material.get_specular_size(0)[0] == 0: d_specular = [tf.zeros(3, dtype=tf.float32)] else: d_specular = [] for l in range(material.get_specular_levels()): specular_size = material.get_specular_size(l) d_specular.append(\ tf.zeros([specular_size[1], specular_size[0], 3], dtype=tf.float32)) if material.get_roughness_size(0)[0] == 0: d_roughness = [tf.zeros(1, dtype=tf.float32)] else: d_roughness = [] for l in range(material.get_roughness_levels()): roughness_size = material.get_roughness_size(l) d_roughness.append(\ tf.zeros([roughness_size[1], roughness_size[0], 1], dtype=tf.float32)) # HACK: tensorflow's eager mode uses a cache to store scalar # constants to avoid memory copy. If we pass scalar tensors # into the C++ code and modify them, we would corrupt the # cache, causing incorrect result in future scalar constant # creations. Thus we force tensorflow to copy by plusing a zero. # (also see https://github.com/tensorflow/tensorflow/issues/11186 # for more discussion regarding copying tensors) if d_roughness[0].shape.num_elements() == 1: d_roughness[0] = d_roughness[0] + 0 if material.get_generic_levels() == 0: d_generic = None else: d_generic = [] for l in range(material.get_generic_levels()): generic_size = material.get_generic_size(l) d_generic.append(\ tf.zeros([generic_size[2], generic_size[1], generic_size[0]], dtype=tf.float32)) if material.get_normal_map_levels() == 0: d_normal_map = None else: d_normal_map = [] for l in range(material.get_normal_map_levels()): normal_map_size = material.get_normal_map_size(l) d_normal_map.append(\ tf.zeros([normal_map_size[1], normal_map_size[0], 3], dtype=tf.float32)) buffers.d_diffuse_list.append(d_diffuse) buffers.d_specular_list.append(d_specular) buffers.d_roughness_list.append(d_roughness) buffers.d_generic_list.append(d_generic) buffers.d_normal_map_list.append(d_normal_map) d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32) d_specular_uv_scale = tf.zeros([2], dtype=tf.float32) d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32) if d_generic is None: d_generic_uv_scale = None else: d_generic_uv_scale = tf.zeros([2], dtype=tf.float32) if d_normal_map is None: d_normal_map_uv_scale = None else: d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32) buffers.d_diffuse_uv_scale_list.append(d_diffuse_uv_scale) buffers.d_specular_uv_scale_list.append(d_specular_uv_scale) buffers.d_roughness_uv_scale_list.append(d_roughness_uv_scale) buffers.d_generic_uv_scale_list.append(d_generic_uv_scale) buffers.d_normal_map_uv_scale_list.append(d_normal_map_uv_scale) if len(d_diffuse[0].shape) == 1: d_diffuse_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(d_diffuse[0]))], [0], [0], 3, redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale))) else: d_diffuse_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_diffuse], [x.shape[1] for x in d_diffuse], [x.shape[0] for x in d_diffuse], 3, redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale))) if len(d_specular[0].shape) == 1: d_specular_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(d_specular[0]))], [0], [0], 3, redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale))) else: d_specular_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_specular], [x.shape[1] for x in d_specular], [x.shape[0] for x in d_specular], 3, redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale))) if len(d_roughness[0].shape) == 1: d_roughness_tex = redner.Texture1(\ [redner.float_ptr(pyredner.data_ptr(d_roughness[0]))], [0], [0], 1, redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale))) else: d_roughness_tex = redner.Texture1(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_roughness], [x.shape[1] for x in d_roughness], [x.shape[0] for x in d_roughness], 1, redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale))) if d_generic is None: d_generic_tex = redner.TextureN(\ [], [], [], 0, redner.float_ptr(0)) else: d_generic_tex = redner.TextureN(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_generic], [x.shape[1] for x in d_generic], [x.shape[0] for x in d_generic], d_generic[0].shape[2], redner.float_ptr(pyredner.data_ptr(d_generic_uv_scale))) if d_normal_map is None: d_normal_map = redner.Texture3(\ [], [], [], 0, redner.float_ptr(0)) else: d_normal_map = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_normal_map], [x.shape[1] for x in d_normal_map], [x.shape[0] for x in d_normal_map], 3, redner.float_ptr(pyredner.data_ptr(d_normal_map_uv_scale))) buffers.d_materials.append(redner.DMaterial(\ d_diffuse_tex, d_specular_tex, d_roughness_tex, d_generic_tex, d_normal_map)) buffers.d_intensity_list = [] buffers.d_area_lights = [] with tf.device(pyredner.get_device_name()): for light in ctx.area_lights: d_intensity = tf.zeros(3, dtype=tf.float32) buffers.d_intensity_list.append(d_intensity) buffers.d_area_lights.append(\ redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity)))) buffers.d_envmap = None if ctx.envmap is not None: envmap = ctx.envmap with tf.device(pyredner.get_device_name()): buffers.d_envmap_values = [] for l in range(envmap.get_levels()): size = envmap.get_size(l) buffers.d_envmap_values.append(\ tf.zeros([size[1], size[0], 3], dtype=tf.float32)) buffers.d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32) buffers.d_world_to_env = tf.zeros([4, 4], dtype=tf.float32) d_envmap_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in buffers.d_envmap_values], [x.shape[1] for x in buffers.d_envmap_values], [x.shape[0] for x in buffers.d_envmap_values], 3, redner.float_ptr(pyredner.data_ptr(buffers.d_envmap_uv_scale))) buffers.d_envmap = redner.DEnvironmentMap( d_envmap_tex, redner.float_ptr(pyredner.data_ptr(buffers.d_world_to_env))) buffers.d_scene = redner.DScene(buffers.d_camera, buffers.d_shapes, buffers.d_materials, buffers.d_area_lights, buffers.d_envmap, pyredner.get_use_gpu(), pyredner.get_gpu_device_id()) return buffers
def backward(grad_img): camera = ctx.camera scene = ctx.scene options = ctx.options with tf.device(pyredner.get_device_name()): if camera.use_look_at: d_position = tf.zeros(3, dtype=tf.float32) d_look_at = tf.zeros(3, dtype=tf.float32) d_up = tf.zeros(3, dtype=tf.float32) d_cam_to_world = None d_world_to_cam = None else: d_position = None d_look_at = None d_up = None d_cam_to_world = tf.zeros([4, 4], dtype=tf.float32) d_world_to_cam = tf.zeros([4, 4], dtype=tf.float32) d_intrinsic_mat_inv = tf.zeros([3, 3], dtype=tf.float32) d_intrinsic_mat = tf.zeros([3, 3], dtype=tf.float32) if camera.use_look_at: d_camera = redner.DCamera( redner.float_ptr(pyredner.data_ptr(d_position)), redner.float_ptr(pyredner.data_ptr(d_look_at)), redner.float_ptr(pyredner.data_ptr(d_up)), redner.float_ptr(0), # cam_to_world redner.float_ptr(0), # world_to_cam redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat_inv)), redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat))) else: d_camera = redner.DCamera( redner.float_ptr(0), redner.float_ptr(0), redner.float_ptr(0), redner.float_ptr(pyredner.data_ptr(d_cam_to_world)), redner.float_ptr(pyredner.data_ptr(d_world_to_cam)), redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat_inv)), redner.float_ptr(pyredner.data_ptr(d_intrinsic_mat))) d_vertices_list = [] d_uvs_list = [] d_normals_list = [] d_colors_list = [] d_shapes = [] with tf.device(pyredner.get_device_name()): for i, shape in enumerate(ctx.shapes): num_vertices = shape.num_vertices d_vertices = tf.zeros([num_vertices, 3], dtype=tf.float32) d_uvs = tf.zeros([num_vertices, 2], dtype=tf.float32) if shape.has_uvs() else None d_normals = tf.zeros( [num_vertices, 3], dtype=tf.float32) if shape.has_normals() else None d_colors = tf.zeros( [num_vertices, 3], dtype=tf.float32) if shape.has_colors() else None d_vertices_list.append(d_vertices) d_uvs_list.append(d_uvs) d_normals_list.append(d_normals) d_colors_list.append(d_colors) d_shapes.append(redner.DShape(\ redner.float_ptr(pyredner.data_ptr(d_vertices)), redner.float_ptr(pyredner.data_ptr(d_uvs) if d_uvs is not None else 0), redner.float_ptr(pyredner.data_ptr(d_normals) if d_normals is not None else 0), redner.float_ptr(pyredner.data_ptr(d_colors) if d_colors is not None else 0))) d_diffuse_list = [] d_specular_list = [] d_roughness_list = [] d_normal_map_list = [] d_diffuse_uv_scale_list = [] d_specular_uv_scale_list = [] d_roughness_uv_scale_list = [] d_generic_list = [] d_generic_uv_scale_list = [] d_normal_map_uv_scale_list = [] d_materials = [] with tf.device(pyredner.get_device_name()): for material in ctx.materials: if material.get_diffuse_size(0)[0] == 0: d_diffuse = [tf.zeros(3, dtype=tf.float32)] else: d_diffuse = [] for l in range(material.get_diffuse_levels()): diffuse_size = material.get_diffuse_size(l) d_diffuse.append(\ tf.zeros([diffuse_size[1], diffuse_size[0], 3], dtype=tf.float32)) if material.get_specular_size(0)[0] == 0: d_specular = [tf.zeros(3, dtype=tf.float32)] else: d_specular = [] for l in range(material.get_specular_levels()): specular_size = material.get_specular_size(l) d_specular.append(\ tf.zeros([specular_size[1], specular_size[0], 3], dtype=tf.float32)) if material.get_roughness_size(0)[0] == 0: d_roughness = [tf.zeros(1, dtype=tf.float32)] else: d_roughness = [] for l in range(material.get_roughness_levels()): roughness_size = material.get_roughness_size(l) d_roughness.append(\ tf.zeros([roughness_size[1], roughness_size[0], 1], dtype=tf.float32)) # HACK: tensorflow's eager mode uses a cache to store scalar # constants to avoid memory copy. If we pass scalar tensors # into the C++ code and modify them, we would corrupt the # cache, causing incorrect result in future scalar constant # creations. Thus we force tensorflow to copy by plusing a zero. # (also see https://github.com/tensorflow/tensorflow/issues/11186 # for more discussion regarding copying tensors) if d_roughness[0].shape.num_elements() == 1: d_roughness[0] = d_roughness[0] + 0 if material.get_generic_levels() == 0: d_generic = None else: d_generic = [] for l in range(material.get_generic_levels()): generic_size = material.get_generic_size(l) d_generic.append(\ tf.zeros([generic_size[2], generic_size[1], generic_size[0]], dtype=tf.float32)) if material.get_normal_map_levels() == 0: d_normal_map = None else: d_normal_map = [] for l in range(material.get_normal_map_levels()): normal_map_size = material.get_normal_map_size(l) d_normal_map.append(\ tf.zeros([normal_map_size[1], normal_map_size[0], 3], dtype=tf.float32)) d_diffuse_list.append(d_diffuse) d_specular_list.append(d_specular) d_roughness_list.append(d_roughness) d_generic_list.append(d_generic) d_normal_map_list.append(d_normal_map) d_diffuse_uv_scale = tf.zeros([2], dtype=tf.float32) d_specular_uv_scale = tf.zeros([2], dtype=tf.float32) d_roughness_uv_scale = tf.zeros([2], dtype=tf.float32) if d_generic is None: d_generic_uv_scale = None else: d_generic_uv_scale = tf.zeros([2], dtype=tf.float32) if d_normal_map is None: d_normal_map_uv_scale = None else: d_normal_map_uv_scale = tf.zeros([2], dtype=tf.float32) d_diffuse_uv_scale_list.append(d_diffuse_uv_scale) d_specular_uv_scale_list.append(d_specular_uv_scale) d_roughness_uv_scale_list.append(d_roughness_uv_scale) d_generic_uv_scale_list.append(d_generic_uv_scale) d_normal_map_uv_scale_list.append(d_normal_map_uv_scale) if len(d_diffuse[0].shape) == 1: d_diffuse_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(d_diffuse[0]))], [0], [0], 3, redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale))) else: d_diffuse_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_diffuse], [x.shape[1] for x in d_diffuse], [x.shape[0] for x in d_diffuse], 3, redner.float_ptr(pyredner.data_ptr(d_diffuse_uv_scale))) if len(d_specular[0].shape) == 1: d_specular_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(d_specular[0]))], [0], [0], 3, redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale))) else: d_specular_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_specular], [x.shape[1] for x in d_specular], [x.shape[0] for x in d_specular], 3, redner.float_ptr(pyredner.data_ptr(d_specular_uv_scale))) if len(d_roughness[0].shape) == 1: d_roughness_tex = redner.Texture1(\ [redner.float_ptr(pyredner.data_ptr(d_roughness[0]))], [0], [0], 1, redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale))) else: d_roughness_tex = redner.Texture1(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_roughness], [x.shape[1] for x in d_roughness], [x.shape[0] for x in d_roughness], 1, redner.float_ptr(pyredner.data_ptr(d_roughness_uv_scale))) if d_generic is None: d_generic_tex = redner.TextureN(\ [], [], [], 0, redner.float_ptr(0)) else: d_generic_tex = redner.TextureN(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_generic], [x.shape[1] for x in d_generic], [x.shape[0] for x in d_generic], d_generic[0].shape[2], redner.float_ptr(pyredner.data_ptr(d_generic_uv_scale))) if d_normal_map is None: d_normal_map = redner.Texture3(\ [], [], [], 0, redner.float_ptr(0)) else: d_normal_map = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_normal_map], [x.shape[1] for x in d_normal_map], [x.shape[0] for x in d_normal_map], 3, redner.float_ptr(pyredner.data_ptr(d_normal_map_uv_scale))) d_materials.append(redner.DMaterial(\ d_diffuse_tex, d_specular_tex, d_roughness_tex, d_generic_tex, d_normal_map)) d_intensity_list = [] d_area_lights = [] with tf.device(pyredner.get_device_name()): for light in ctx.area_lights: d_intensity = tf.zeros(3, dtype=tf.float32) d_intensity_list.append(d_intensity) d_area_lights.append(\ redner.DAreaLight(redner.float_ptr(pyredner.data_ptr(d_intensity)))) d_envmap = None if ctx.envmap is not None: envmap = ctx.envmap with tf.device(pyredner.get_device_name()): d_envmap_values = [] for l in range(envmap.get_levels()): size = envmap.get_size(l) d_envmap_values.append(\ tf.zeros([size[1], size[0], 3], dtype=tf.float32)) d_envmap_uv_scale = tf.zeros([2], dtype=tf.float32) d_world_to_env = tf.zeros([4, 4], dtype=tf.float32) d_envmap_tex = redner.Texture3(\ [redner.float_ptr(pyredner.data_ptr(x)) for x in d_envmap_values], [x.shape[1] for x in d_envmap_values], [x.shape[0] for x in d_envmap_values], 3, redner.float_ptr(pyredner.data_ptr(d_envmap_uv_scale))) d_envmap = redner.DEnvironmentMap( d_envmap_tex, redner.float_ptr(pyredner.data_ptr(d_world_to_env))) d_scene = redner.DScene(d_camera, d_shapes, d_materials, d_area_lights, d_envmap, pyredner.get_use_gpu(), pyredner.get_gpu_device_id()) if not get_use_correlated_random_number(): # Decod_uple the forward/backward random numbers by adding a big prime number options.seed += 1000003 start = time.time() options.num_samples = ctx.num_samples[1] with tf.device(pyredner.get_device_name()): grad_img = tf.identity(grad_img) redner.render( scene, options, redner.float_ptr(0), # rendered_image redner.float_ptr(pyredner.data_ptr(grad_img)), d_scene, redner.float_ptr(0)) # debug_image time_elapsed = time.time() - start if print_timing: print('Backward pass, time: %.5f s' % time_elapsed) # # For debugging # pyredner.imwrite(grad_img, 'grad_img.exr') # grad_img = tf.ones([256, 256, 3], dtype=tf.float32) # debug_img = tf.zeros([256, 256, 3], dtype=tf.float32) # redner.render(scene, options, # redner.float_ptr(0), # redner.float_ptr(pyredner.data_ptr(grad_img)), # d_scene, # redner.float_ptr(pyredner.data_ptr(debug_img))) # pyredner.imwrite(debug_img, 'debug.exr') # pyredner.imwrite(-debug_img, 'debug_.exr') # exit() ret_list = [] ret_list.append(None) # seed ret_list.append(None) # num_shapes ret_list.append(None) # num_materials ret_list.append(None) # num_lights if camera.use_look_at: ret_list.append(d_position) ret_list.append(d_look_at) ret_list.append(d_up) ret_list.append(None) # cam_to_world ret_list.append(None) # world_to_cam else: ret_list.append(None) # pos ret_list.append(None) # look ret_list.append(None) # up ret_list.append(d_cam_to_world) ret_list.append(d_world_to_cam) ret_list.append(d_intrinsic_mat_inv) ret_list.append(d_intrinsic_mat) ret_list.append(None) # clip near ret_list.append(None) # resolution ret_list.append(None) # camera_type num_shapes = len(ctx.shapes) for i in range(num_shapes): ret_list.append(d_vertices_list[i]) ret_list.append(None) # indices ret_list.append(d_uvs_list[i]) ret_list.append(d_normals_list[i]) ret_list.append(None) # uv_indices ret_list.append(None) # normal_indices ret_list.append(d_colors_list[i]) ret_list.append(None) # material id ret_list.append(None) # light id num_materials = len(ctx.materials) for i in range(num_materials): ret_list.append(None) # num_levels for d_diffuse in d_diffuse_list[i]: ret_list.append(d_diffuse) ret_list.append(d_diffuse_uv_scale_list[i]) ret_list.append(None) # num_levels for d_specular in d_specular_list[i]: ret_list.append(d_specular) ret_list.append(d_specular_uv_scale_list[i]) ret_list.append(None) # num_levels for d_roughness in d_roughness_list[i]: ret_list.append(d_roughness) ret_list.append(d_roughness_uv_scale_list[i]) if d_generic_list[i] is None: ret_list.append(None) # num_levels else: ret_list.append(None) # num_levels for d_generic in d_generic_list[i]: ret_list.append(d_generic) ret_list.append(d_generic_uv_scale_list[i]) if d_normal_map_list[i] is None: ret_list.append(None) # num_levels else: ret_list.append(None) # num_levels for d_normal_map in d_normal_map_list[i]: ret_list.append(d_normal_map) ret_list.append(d_normal_map_uv_scale_list[i]) ret_list.append(None) # compute_specular_lighting ret_list.append(None) # two sided ret_list.append(None) # use_vertex_color num_area_lights = len(ctx.area_lights) for i in range(num_area_lights): ret_list.append(None) # shape id with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): ret_list.append(tf.identity(d_intensity_list[i])) ret_list.append(None) # two sided if ctx.envmap is not None: ret_list.append(None) # num_levels for d_values in d_envmap_values: ret_list.append(d_values) ret_list.append(d_envmap_uv_scale) ret_list.append(None) # env_to_world with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): ret_list.append(tf.identity(d_world_to_env)) ret_list.append(None) # sample_cdf_ys ret_list.append(None) # sample_cdf_xs ret_list.append(None) # pdf_norm else: ret_list.append(None) ret_list.append(None) # num samples ret_list.append(None) # num bounces ret_list.append(None) # num channels for _ in range(ctx.num_channels): ret_list.append(None) # channel ret_list.append(None) # sampler type ret_list.append(None) # use_primary_edge_sampling ret_list.append(None) # use_secondary_edge_sampling ret_list.append(None) # sample_pixel_center return ret_list
def generate_sphere(theta_steps: int, phi_steps: int): """ Generate a triangle mesh representing a UV sphere, center at (0, 0, 0) with radius 1. Args ==== theta_steps: int zenith subdivision phi_steps: int azimuth subdivision Returns ======= tf.Tensor vertices tf.Tensor indices tf.Tensor uvs tf.Tensor normals """ d_theta = math.pi / (theta_steps - 1) d_phi = (2 * math.pi) / (phi_steps - 1) num_vertices = theta_steps * phi_steps - 2 * (phi_steps - 1) vertices = np.zeros([num_vertices, 3], dtype=np.float32) uvs = np.zeros([num_vertices, 2], dtype=np.float32) vertices_index = 0 for theta_index in range(theta_steps): sin_theta = math.sin(theta_index * d_theta) cos_theta = math.cos(theta_index * d_theta) if theta_index == 0: # For the two polars of the sphere, only generate one vertex vertices[vertices_index, :] = \ np.array([0.0, 1.0, 0.0], dtype = np.float32) uvs[vertices_index, 0] = 0.0 uvs[vertices_index, 1] = 0.0 vertices_index += 1 elif theta_index == theta_steps - 1: # For the two polars of the sphere, only generate one vertex vertices[vertices_index, :] = \ np.array([0.0, -1.0, 0.0], dtype = np.float32) uvs[vertices_index, 0] = 0.0 uvs[vertices_index, 1] = 1.0 vertices_index += 1 else: for phi_index in range(phi_steps): sin_phi = math.sin(phi_index * d_phi) cos_phi = math.cos(phi_index * d_phi) vertices[vertices_index, :] = \ np.array([sin_theta * cos_phi, cos_theta, sin_theta * sin_phi]) uvs[vertices_index, 0] = phi_index * d_phi / (2 * math.pi) uvs[vertices_index, 1] = theta_index * d_theta / math.pi vertices_index += 1 indices = [] for theta_index in range(1, theta_steps): for phi_index in range(phi_steps - 1): if theta_index < theta_steps - 1: id0 = phi_steps * theta_index + phi_index - (phi_steps - 1) id1 = phi_steps * theta_index + phi_index + 1 - (phi_steps - 1) else: # There is only one vertex at the pole assert (theta_index == theta_steps - 1) id0 = num_vertices - 1 id1 = num_vertices - 1 if theta_index > 1: id2 = phi_steps * (theta_index - 1) + phi_index - (phi_steps - 1) id3 = phi_steps * (theta_index - 1) + phi_index + 1 - ( phi_steps - 1) else: # There is only one vertex at the pole assert (theta_index == 1) id2 = 0 id3 = 0 if (theta_index < theta_steps - 1): indices.append([id0, id2, id1]) if (theta_index > 1): indices.append([id1, id2, id3]) with tf.device(pyredner.get_device_name()): indices = tf.convert_to_tensor(indices, dtype=tf.int32) vertices = tf.convert_to_tensor(vertices, dtype=tf.float32) uvs = tf.convert_to_tensor(uvs, dtype=tf.float32) normals = tf.identity(vertices) return (vertices, indices, uvs, normals)
def backward(grad_img): scene = ctx.scene options = ctx.options buffers = create_gradient_buffers(ctx) if not get_use_correlated_random_number(): # Decod_uple the forward/backward random numbers by adding a big prime number options.seed += 1000003 start = time.time() options.num_samples = ctx.num_samples[1] with tf.device(pyredner.get_device_name()): grad_img = tf.identity(grad_img) redner.render( scene, options, redner.float_ptr(0), # rendered_image redner.float_ptr(pyredner.data_ptr(grad_img)), buffers.d_scene, redner.float_ptr(0), # translational_gradient_image redner.float_ptr(0)) # debug_image time_elapsed = time.time() - start if get_print_timing(): print('Backward pass, time: %.5f s' % time_elapsed) ret_list = [] ret_list.append(None) # seed ret_list.append(None) # num_shapes ret_list.append(None) # num_materials ret_list.append(None) # num_lights if ctx.camera.use_look_at: ret_list.append(buffers.d_position) ret_list.append(buffers.d_look_at) ret_list.append(buffers.d_up) ret_list.append(None) # cam_to_world ret_list.append(None) # world_to_cam else: ret_list.append(None) # pos ret_list.append(None) # look ret_list.append(None) # up ret_list.append(buffers.d_cam_to_world) ret_list.append(buffers.d_world_to_cam) ret_list.append(buffers.d_intrinsic_mat_inv) ret_list.append(buffers.d_intrinsic_mat) ret_list.append(None) # clip near ret_list.append(None) # resolution ret_list.append(None) # viewport ret_list.append(None) # camera_type num_shapes = len(ctx.shapes) for i in range(num_shapes): ret_list.append(buffers.d_vertices_list[i]) ret_list.append(None) # indices ret_list.append(buffers.d_uvs_list[i]) ret_list.append(buffers.d_normals_list[i]) ret_list.append(None) # uv_indices ret_list.append(None) # normal_indices ret_list.append(buffers.d_colors_list[i]) ret_list.append(None) # material id ret_list.append(None) # light id num_materials = len(ctx.materials) for i in range(num_materials): ret_list.append(None) # num_levels for d_diffuse in buffers.d_diffuse_list[i]: ret_list.append(d_diffuse) ret_list.append(buffers.d_diffuse_uv_scale_list[i]) ret_list.append(None) # num_levels for d_specular in buffers.d_specular_list[i]: ret_list.append(d_specular) ret_list.append(buffers.d_specular_uv_scale_list[i]) ret_list.append(None) # num_levels for d_roughness in buffers.d_roughness_list[i]: ret_list.append(d_roughness) ret_list.append(buffers.d_roughness_uv_scale_list[i]) if buffers.d_generic_list[i] is None: ret_list.append(None) # num_levels else: ret_list.append(None) # num_levels for d_generic in buffers.d_generic_list[i]: ret_list.append(d_generic) ret_list.append(buffers.d_generic_uv_scale_list[i]) if buffers.d_normal_map_list[i] is None: ret_list.append(None) # num_levels else: ret_list.append(None) # num_levels for d_normal_map in buffers.d_normal_map_list[i]: ret_list.append(d_normal_map) ret_list.append(buffers.d_normal_map_uv_scale_list[i]) ret_list.append(None) # compute_specular_lighting ret_list.append(None) # two sided ret_list.append(None) # use_vertex_color num_area_lights = len(ctx.area_lights) for i in range(num_area_lights): ret_list.append(None) # shape id with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): ret_list.append(tf.identity(buffers.d_intensity_list[i])) ret_list.append(None) # two_sided ret_list.append(None) # directly_visible if ctx.envmap is not None: ret_list.append(None) # num_levels for d_values in buffers.d_envmap_values: ret_list.append(d_values) ret_list.append(buffers.d_envmap_uv_scale) ret_list.append(None) # env_to_world with tf.device('/device:cpu:' + str(pyredner.get_cpu_device_id())): ret_list.append(tf.identity(buffers.d_world_to_env)) ret_list.append(None) # sample_cdf_ys ret_list.append(None) # sample_cdf_xs ret_list.append(None) # pdf_norm ret_list.append(None) # directly_visible else: ret_list.append(None) ret_list.append(None) # num samples ret_list.append(None) # num bounces ret_list.append(None) # num channels for _ in range(ctx.num_channel_args): ret_list.append(None) # channel ret_list.append(None) # sampler type ret_list.append(None) # use_primary_edge_sampling ret_list.append(None) # use_secondary_edge_sampling ret_list.append(None) # sample_pixel_center return ret_list
def __init__(self, texels, uv_scale=tf.constant([1.0, 1.0])): assert (tf.executing_eagerly()) if pyredner.get_use_gpu(): texels = tf.identity(texels).gpu(pyredner.get_gpu_device_id()) uv_scale = tf.identity(uv_scale).gpu(pyredner.get_gpu_device_id()) else: texels = tf.identity(texels).cpu() uv_scale = tf.identity(uv_scale).cpu() self.texels = texels if len(texels.shape) >= 2: with tf.device(pyredner.get_device_name()): # Build a mipmap for texels width = max( texels.shape[0], texels.shape[1] ).value # Without value, it will have a type `Dimension` num_levels = math.ceil(math.log(width, 2) + 1) mipmap = tf.broadcast_to(texels, [num_levels, *texels.shape]) if len(mipmap.shape) == 3: mipmap = tf.expand_dims(mipmap, axis=-1) num_channels = mipmap.shape[-1] """NOTE: conv2d kernel axes torch: (outchannels, in_channels / groups, kH, kW) tf: [filter_height, filter_width, in_channels, out_channels] """ box_filter = tf.ones([2, 2, num_channels, 1], dtype=tf.float32) / 4.0 # (TF) [batch, in_height, in_width, in_channels], i.e. NHWC base_level = tf.transpose(tf.expand_dims(texels, axis=0), perm=[0, 3, 1, 2]) mipmap = [base_level] prev_lvl = base_level for l in range(1, num_levels): dilation_size = 2**(l - 1) # Pad for circular boundary condition # This is slow. The hope is at some point Tensorflow will support # circular boundary condition for conv2d desired_height = prev_lvl.shape[2] + dilation_size while prev_lvl.shape[2] < desired_height: prev_lvl = tf.concat([ prev_lvl, prev_lvl[:, :, 0:(desired_height - prev_lvl.shape[2])] ], 2) desired_width = prev_lvl.shape[3] + dilation_size while prev_lvl.shape[3] < desired_width: prev_lvl = tf.concat( [prev_lvl, prev_lvl[:, :, :, 0:dilation_size]], 3) """NOTE: Torch conv data_format is NCHW. In Tensorflow, GPU supports NCHW but CPU supports only NHWC. Hence, we need to convert between NCHW and NHwC when we use CPU. """ """NOTE: Current libxsmm and customized CPU implementations do not yet support dilation rates larger than 1, i.e. we cannot use TF Conv2DCustomBackpropInputOp https://github.com/tensorflow/tensorflow/blob/7bc1c3c37ce4e591012f4325ab7a25ae387773c7/tensorflow/core/kernels/conv_grad_input_ops.cc#L300 """ # if pyredner.use_gpu: # current_lvl = tf.nn.depthwise_conv2d( # prev_lvl, # box_filter, # [filter_height, filter_width, in_channels, out_channels] # dilations=[dilation_size,dilation_size], # strides=[1,1,1,1], # padding="VALID", # No padding # data_format="NCHW" # ) # else: prev_lvl = tf.transpose(prev_lvl, perm=[0, 2, 3, 1]) current_lvl = tf.nn.depthwise_conv2d( prev_lvl, box_filter, # [filter_height, filter_width, in_channels, out_channels] dilations=[dilation_size, dilation_size], strides=[1, 1, 1, 1], padding="VALID", # No padding data_format="NHWC") current_lvl = tf.transpose(current_lvl, [0, 3, 1, 2]) mipmap.append(current_lvl) prev_lvl = current_lvl mipmap = tf.concat(mipmap, 0) # Convert from NCHW to NHWC mipmap = tf.transpose(mipmap, perm=[0, 2, 3, 1]) texels = mipmap self.mipmap = texels self.uv_scale = uv_scale