def get_pixels_deferred_v1(transformed_vertices, faces, vertex_normals, vertex_colours, light_intensity, background): # This is a naive implementation of deferred shading, that gives incorrect gradients. See # get_pixels_deferred_v2 below for a correct implementation! gbuffer_mask = dirt.rasterise( vertices=transformed_vertices, faces=faces, vertex_colors=tf.ones_like(transformed_vertices[:, :1]), background=tf.zeros([canvas_height, canvas_width, 1]), width=canvas_width, height=canvas_height, channels=1)[..., 0] background_value = -1.e4 gbuffer_vertex_colours_world = dirt.rasterise( vertices=transformed_vertices, faces=faces, vertex_colors=vertex_colours, background=tf.ones([canvas_height, canvas_width, 3]) * background, width=canvas_width, height=canvas_height, channels=3) gbuffer_vertex_normals_world = dirt.rasterise( vertices=transformed_vertices, faces=faces, vertex_colors=vertex_normals, background=tf.ones([canvas_height, canvas_width, 3]) * background_value, width=canvas_width, height=canvas_height, channels=3) # Dilate the normals to ensure correct gradients on the silhouette gbuffer_mask = gbuffer_mask[:, :, None] gbuffer_vertex_normals_world_dilated = tf.nn.max_pool( gbuffer_vertex_normals_world[None, ...], ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='SAME')[0] gbuffer_vertex_normals_world = gbuffer_vertex_normals_world * gbuffer_mask + gbuffer_vertex_normals_world_dilated * ( 1. - gbuffer_mask) pixels = gbuffer_mask * calculate_shading( gbuffer_vertex_colours_world, gbuffer_vertex_normals_world, light_intensity) + (1. - gbuffer_mask) * background return pixels
def call(self, x): #N: num. of vertices, F: num of faces vertices = x[0] #(1xNx3) #same for each batch faces = tf.cast(x[1], tf.int32) #1xFx3, same for each batch poses = x[2] #1 x 4x4, same for each batch # Transform vertices from camera to clip space vert_obj, vert_3d, vert_clip, normals = self.transform_vertices( vertices[0], poses[0], faces[0]) gbuffer_temp = dirt.rasterise( background=tf.zeros([self.img_h, self.img_w, 11]), vertices=vert_clip, vertex_colors=tf.concat( [ tf.ones_like(vert_obj[:, :1]), #1 mask vert_3d, normals, vert_obj ], axis=1), faces=faces[0], height=self.img_h, width=self.img_w, channels=11) return tf.expand_dims(gbuffer_temp, axis=0)
def render_colored(m_v, m_f, m_vc, width, height, camera_f, camera_c, bgcolor=np.zeros(3, dtype=np.float32), num_channels=3, camera_t=np.zeros(3, dtype=np.float32), camera_rt=np.zeros(3, dtype=np.float32), name=None): with ops.name_scope(name, "render", [m_v]) as name: assert (num_channels == m_vc.shape[-1] == bgcolor.shape[0]) projection_matrix = perspective_projection( camera_f, camera_c, width, height, .1, 10) # projection_matrix = matrices.perspective_projection(near=0.1, far=20., right=0.1, aspect=1.) view_matrix = matrices.compose( matrices.rodrigues(camera_rt.astype(np.float32)), matrices.translation(camera_t.astype(np.float32)), ) bg = tf.tile(bgcolor.astype(np.float32)[ np.newaxis, np.newaxis, :], (height, width, 1)) m_v = tf.cast(m_v, tf.float32) m_v = tf.concat([m_v, tf.ones_like(m_v[:, -1:])], axis=1) m_v = tf.matmul(m_v, view_matrix) m_v = tf.matmul(m_v, projection_matrix) return dirt.rasterise(bg, m_v, tf.cast(m_vc, tf.float32), tf.cast(m_f, tf.int32), name=name)
def main(): # Build the scene geometry, which is just an axis-aligned cube centred at the origin in world space # We replicate vertices that are shared, so normals are effectively per-face instead of smoothed cube_vertices_object, cube_faces = build_cube() cube_vertices_object = tf.constant(cube_vertices_object, dtype=tf.float32) cube_vertices_object, cube_faces = lighting.split_vertices_by_face( cube_vertices_object, cube_faces) cube_vertex_colors = tf.ones_like(cube_vertices_object) # Convert vertices to homogeneous coordinates cube_vertices_object = tf.concat( [cube_vertices_object, tf.ones_like(cube_vertices_object[:, -1:])], axis=1) # Transform vertices from object to world space, by rotating around the vertical axis cube_vertices_world = tf.matmul(cube_vertices_object, matrices.rodrigues([0., 0.5, 0.])) # Calculate face normals; pre_split implies that no faces share a vertex cube_normals_world = lighting.vertex_normals_pre_split( cube_vertices_world, cube_faces) # Transform vertices from world to camera space; note that the camera points along the negative-z axis in camera space view_matrix = matrices.compose( matrices.translation([0., -1.5, -3.5]), # translate it away from the camera matrices.rodrigues([-0.3, 0., 0.]) # tilt the view downwards ) cube_vertices_camera = tf.matmul(cube_vertices_world, view_matrix) # Transform vertices from camera to clip space projection_matrix = matrices.perspective_projection( near=0.1, far=20., right=0.1, aspect=float(frame_height) / frame_width) cube_vertices_clip = tf.matmul(cube_vertices_camera, projection_matrix) # Calculate lighting, as combination of diffuse and ambient vertex_colors_lit = lighting.diffuse_directional( cube_normals_world, cube_vertex_colors, light_direction=[1., 0., 0.], light_color=[1., 1., 1.]) * 0.8 + cube_vertex_colors * 0.2 pixels = dirt.rasterise(vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=vertex_colors_lit, background=tf.zeros([frame_height, frame_width, 3]), width=frame_width, height=frame_height, channels=3) session = tf.Session() with session.as_default(): pixels_eval = pixels.eval() cv2.imshow('simple.py', pixels_eval[:, :, (2, 1, 0)]) cv2.waitKey(0)
def get_pixels_direct(transformed_vertices, faces, vertex_normals, vertex_colours, light_intensity, background): return dirt.rasterise( vertices=transformed_vertices, faces=faces, vertex_colors=calculate_shading(vertex_colours, vertex_normals, light_intensity), background=tf.ones([canvas_height, canvas_width, 3]) * background)
def get_dirt_pixels(): # Build square in screen space square_vertices = tf.constant([[0, 0], [0, 1], [1, 1], [1, 0]], dtype=tf.float32) * square_size - square_size / 2. square_vertices += [centre_x, centre_y] # Transform to homogeneous coordinates in clip space square_vertices = square_vertices * 2. / [canvas_width, canvas_height] - 1. square_vertices = tf.concat([square_vertices, tf.zeros([4, 1]), tf.ones([4, 1])], axis=1) return dirt.rasterise( vertices=square_vertices, faces=[[0, 1, 2], [0, 2, 3]], vertex_colors=tf.ones([4, 1]), background=tf.zeros([canvas_height, canvas_width, 1]), height=canvas_height, width=canvas_width, channels=1 )[:, :, 0]
def get_dirt_pixels(width=canvas_width, height=canvas_height): square_vertices = tf.constant([[-1, -1, 0, 1], [-1, 1, 0, 1], [1, 1, 0, 1], [1, -1, 0, 1]], dtype=tf.float32) #background = skimage.io.imread('/n/fs/shaderml/datas_oceanic/test_img/test_middle_ground00000.png') #background = tf.constant(skimage.img_as_float(background), dtype=tf.float32) background = tf.zeros([height, width, 3], dtype=tf.float32) camera_pos = tf.placeholder(tf.float32, 8) return dirt.rasterise( vertices=square_vertices, faces=[[0, 1, 2], [0, 2, 3]], vertex_colors=tf.ones([4, 3]), background=background, camera_pos = camera_pos, height=height, width=width, channels=3 ), camera_pos
def main(): # Build the scene geometry, which is just an axis-aligned cube centred at the origin in world space cube_vertices_object = [] cube_uvs = [] cube_faces = [] def add_quad(vertices, uvs): index = len(cube_vertices_object) cube_faces.extend([[index + 2, index + 1, index], [index, index + 3, index + 2]]) cube_vertices_object.extend(vertices) cube_uvs.extend(uvs) add_quad(vertices=[[-1, -1, 1], [1, -1, 1], [1, 1, 1], [-1, 1, 1]], uvs=[[0.1, 0.9], [0.9, 0.9], [0.9, 0.1], [0.1, 0.1]]) # front add_quad(vertices=[[-1, -1, -1], [1, -1, -1], [1, 1, -1], [-1, 1, -1]], uvs=[[1, 1], [0, 1], [0, 0], [1, 0]]) # back add_quad(vertices=[[1, 1, 1], [1, 1, -1], [1, -1, -1], [1, -1, 1]], uvs=[[0.4, 0.35], [0.5, 0.35], [0.5, 0.45], [0.4, 0.45]]) # right add_quad(vertices=[[-1, 1, 1], [-1, 1, -1], [-1, -1, -1], [-1, -1, 1]], uvs=[[0.4, 0.4], [0.5, 0.4], [0.5, 0.5], [0.4, 0.5]]) # left add_quad(vertices=[[-1, 1, -1], [1, 1, -1], [1, 1, 1], [-1, 1, 1]], uvs=[[0, 0], [2, 0], [2, 2], [0, 2]]) # top add_quad(vertices=[[-1, -1, -1], [1, -1, -1], [1, -1, 1], [-1, -1, 1]], uvs=[[0, 0], [2, 0], [2, 2], [0, 2]]) # bottom cube_vertices_object = np.asarray(cube_vertices_object, np.float32) cube_uvs = np.asarray(cube_uvs, np.float32) # Load the texture image texture = tf.constant( imageio.imread(os.path.dirname(__file__) + '/cat.jpg'), dtype=tf.float32) / 255. # Convert vertices to homogeneous coordinates cube_vertices_object = tf.concat( [cube_vertices_object, tf.ones_like(cube_vertices_object[:, -1:])], axis=1) # Transform vertices from object to world space, by rotating around the vertical axis cube_vertices_world = tf.matmul(cube_vertices_object, matrices.rodrigues([0., 0.6, 0.])) # Calculate face normals cube_normals_world = lighting.vertex_normals(cube_vertices_world, cube_faces) # Transform vertices from world to camera space; note that the camera points along the negative-z axis in camera space view_matrix = matrices.compose( matrices.translation([0., -2., -3.2]), # translate it away from the camera matrices.rodrigues([-0.5, 0., 0.]) # tilt the view downwards ) cube_vertices_camera = tf.matmul(cube_vertices_world, view_matrix) # Transform vertices from camera to clip space projection_matrix = matrices.perspective_projection( near=0.1, far=20., right=0.1, aspect=float(frame_height) / frame_width) cube_vertices_clip = tf.matmul(cube_vertices_camera, projection_matrix) # Render the G-buffer channels (mask, UVs, and normals at each pixel) needed for deferred shading gbuffer_mask = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=tf.ones_like(cube_vertices_object[:, :1]), background=tf.zeros([frame_height, frame_width, 1]), width=frame_width, height=frame_height, channels=1)[..., 0] background_value = -1.e4 gbuffer_vertex_uvs = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=tf.concat( [cube_uvs, tf.zeros_like(cube_uvs[:, :1])], axis=1), background=tf.ones([frame_height, frame_width, 3]) * background_value, width=frame_width, height=frame_height, channels=3)[..., :2] gbuffer_vertex_normals_world = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=cube_normals_world, background=tf.ones([frame_height, frame_width, 3]) * background_value, width=frame_width, height=frame_height, channels=3) # Dilate the normals and UVs to ensure correct gradients on the silhouette gbuffer_mask = gbuffer_mask[:, :, None] gbuffer_vertex_normals_world_dilated = tf.nn.max_pool( gbuffer_vertex_normals_world[None, ...], ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='SAME')[0] gbuffer_vertex_normals_world = gbuffer_vertex_normals_world * gbuffer_mask + gbuffer_vertex_normals_world_dilated * ( 1. - gbuffer_mask) gbuffer_vertex_uvs_dilated = tf.nn.max_pool(gbuffer_vertex_uvs[None, ...], ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='SAME')[0] gbuffer_vertex_uvs = gbuffer_vertex_uvs * gbuffer_mask + gbuffer_vertex_uvs_dilated * ( 1. - gbuffer_mask) # Calculate the colour buffer, by sampling the texture according to the rasterised UVs gbuffer_colours = gbuffer_mask * sample_texture( texture, uvs_to_pixel_indices(gbuffer_vertex_uvs, tf.shape(texture)[:2])) # Calculate a simple grey ambient lighting component ambient_contribution = gbuffer_colours * [0.4, 0.4, 0.4] # Calculate a diffuse (Lambertian) lighting component light_direction = unit([1., -0.3, -0.5]) diffuse_contribution = lighting.diffuse_directional( tf.reshape(gbuffer_vertex_normals_world, [-1, 3]), tf.reshape(gbuffer_colours, [-1, 3]), light_direction, light_color=[0.6, 0.6, 0.6], double_sided=True) diffuse_contribution = tf.reshape(diffuse_contribution, [frame_height, frame_width, 3]) # Final pixels are given by combining the ambient and diffuse components pixels = diffuse_contribution + ambient_contribution session = tf.Session(config=tf.ConfigProto(gpu_options=tf.GPUOptions( allow_growth=True))) with session.as_default(): pixels_eval = pixels.eval() imageio.imsave('textured.jpg', (pixels_eval * 255).astype(np.uint8))
def main(): # Build the scene geometry, which is just an axis-aligned cube centred at the origin in world space # We replicate vertices that are shared, so normals are effectively per-face instead of smoothed cube_vertices_object, cube_faces = build_cube() cube_vertices_object = tf.constant(cube_vertices_object, dtype=tf.float32) cube_vertices_object, cube_faces = lighting.split_vertices_by_face( cube_vertices_object, cube_faces) cube_vertex_colors = tf.ones_like(cube_vertices_object) # Convert vertices to homogeneous coordinates cube_vertices_object = tf.concat( [cube_vertices_object, tf.ones_like(cube_vertices_object[:, -1:])], axis=1) # Transform vertices from object to world space, by rotating around the vertical axis cube_vertices_world = tf.matmul(cube_vertices_object, matrices.rodrigues([0., 0.5, 0.])) # Calculate face normals; pre_split implies that no faces share a vertex cube_normals_world = lighting.vertex_normals_pre_split( cube_vertices_world, cube_faces) # Transform vertices from world to camera space; note that the camera points along the negative-z axis in camera space view_matrix = matrices.compose( matrices.translation([0., -1.5, -3.5]), # translate it away from the camera matrices.rodrigues([-0.3, 0., 0.]) # tilt the view downwards ) cube_vertices_camera = tf.matmul(cube_vertices_world, view_matrix) # Transform vertices from camera to clip space projection_matrix = matrices.perspective_projection( near=0.1, far=20., right=0.1, aspect=float(frame_height) / frame_width) cube_vertices_clip = tf.matmul(cube_vertices_camera, projection_matrix) # Render the G-buffer channels (vertex position, colour and normal at each pixel) needed for deferred shading gbuffer_vertex_positions_world = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=cube_vertices_world[:, :3], background=tf.ones([frame_height, frame_width, 3]) * float('-inf'), width=frame_width, height=frame_height, channels=3) gbuffer_vertex_colours_world = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=cube_vertex_colors, background=tf.zeros([frame_height, frame_width, 3]), width=frame_width, height=frame_height, channels=3) gbuffer_vertex_normals_world = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces, vertex_colors=cube_normals_world, background=tf.ones([frame_height, frame_width, 3]) * float('-inf'), width=frame_width, height=frame_height, channels=3) # Dilate the position and normal channels at the silhouette boundary; this doesn't affect the image, but # ensures correct gradients for pixels just outside the silhouette background_mask = tf.cast( tf.equal(gbuffer_vertex_positions_world, float('-inf')), tf.float32) gbuffer_vertex_positions_world_dilated = tf.nn.max_pool( gbuffer_vertex_positions_world[None, ...], ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='SAME')[0] gbuffer_vertex_positions_world = gbuffer_vertex_positions_world * ( 1. - background_mask ) + gbuffer_vertex_positions_world_dilated * background_mask gbuffer_vertex_normals_world_dilated = tf.nn.max_pool( gbuffer_vertex_normals_world[None, ...], ksize=[1, 3, 3, 1], strides=[1, 1, 1, 1], padding='SAME')[0] gbuffer_vertex_normals_world = gbuffer_vertex_normals_world * ( 1. - background_mask ) + gbuffer_vertex_normals_world_dilated * background_mask # Calculate a simple grey ambient lighting component ambient_contribution = gbuffer_vertex_colours_world * [0.2, 0.2, 0.2] # Calculate a red diffuse (Lambertian) lighting component light_direction = unit([1., -0.3, -0.5]) diffuse_contribution = lighting.diffuse_directional( tf.reshape(gbuffer_vertex_normals_world, [-1, 3]), tf.reshape(gbuffer_vertex_colours_world, [-1, 3]), light_direction, light_color=[1., 0., 0.], double_sided=False) diffuse_contribution = tf.reshape(diffuse_contribution, [frame_height, frame_width, 3]) # Calculate a white specular (Phong) lighting component camera_position_world = tf.matrix_inverse(view_matrix)[3, :3] specular_contribution = lighting.specular_directional( tf.reshape(gbuffer_vertex_positions_world, [-1, 3]), tf.reshape(gbuffer_vertex_normals_world, [-1, 3]), tf.reshape(gbuffer_vertex_colours_world, [-1, 3]), light_direction, light_color=[1., 1., 1.], camera_position=camera_position_world, shininess=6., double_sided=False) specular_contribution = tf.reshape(specular_contribution, [frame_height, frame_width, 3]) # Final pixels are given by combining ambient, diffuse, and specular components pixels = diffuse_contribution + specular_contribution + ambient_contribution session = tf.Session() with session.as_default(): pixels_eval = pixels.eval() cv2.imshow('deferred.py', pixels_eval[:, :, (2, 1, 0)]) cv2.waitKey(0)
def __call__(self, verts, trans, cam=None, img=None, do_alpha=False, far=None, near=None, color_id=0, img_size=None, seg_parts=13774): """ cam is 3D [f, px, py] """ '''if img is not None: h, w = img.shape[:2] elif img_size is not None: h = img_size[0] w = img_size[1] else: h = self.h w = self.w if cam is None: cam = [self.flength, w / 2., h / 2.] use_cam = ProjectPoints( f=cam[0] * np.ones(2), rt=np.zeros(3), t=np.zeros(3), k=np.zeros(5), c=cam[1:3]) if near is None: near = np.maximum(np.min(verts[:, 2]) - 25, 0.1) if far is None: far = np.maximum(np.max(verts[:, 2]) + 25, 25) imtmp = render_model( verts, self.faces, w, h, use_cam, do_alpha=do_alpha, img=img, far=far, near=near, color_id=color_id)''' #print (self.textura.shape) frame_width, frame_height = self.w, self.h cube_vertices_object = verts[0,:,:] cube_faces = tf.constant(self.faces,dtype=tf.int64) #cube_vertices_object = tf.constant(cube_vertices_object, dtype=tf.float32) cube_vertices_object, cube_faces = lighting.split_vertices_by_face(cube_vertices_object, cube_faces) #cube_vertex_colors = tf.ones_like(cube_vertices_object) cube_vertex_colors = tf.constant(self.textura,dtype=tf.float32) # Convert vertices to homogeneous coordinates cube_vertices_object = tf.concat([cube_vertices_object,tf.ones_like(cube_vertices_object[:, -1:])], axis=1) # Transform vertices from object to world space, by rotating around the vertical axis # Calculate face normals; pre_split implies that no faces share a vertex cube_normals_world = lighting.vertex_normals_pre_split(cube_vertices_object, cube_faces) # Transform vertices from world to camera space; note that the camera points along the negative-z axis in camera space view_matrix = matrices.translation(trans) cube_vertices_world = tf.matmul(cube_vertices_object, view_matrix) cube_vertices_camera = tf.matmul(cube_vertices_world, matrices.rodrigues([np.pi, 0.0, 0.])) # Transform vertices from camera to clip space projection_matrix = matrices.perspective_projection(near=self.near, far=self.far, right=self.right, aspect=float(frame_height) / frame_width) cube_vertices_clip = tf.matmul(cube_vertices_camera, projection_matrix) # Calculate lighting, as combination of diffuse and ambient #vertex_colors_lit = lighting.diffuse_directional( # cube_normals_world, cube_vertex_colors, # light_direction=[1., 0., 0.], light_color=[1., 1., 1.] #) * 0.8 + cube_vertex_colors * 0.2 #2987 seg_parts pixels = dirt.rasterise( vertices=cube_vertices_clip, faces=cube_faces[:seg_parts,:], vertex_colors=cube_vertex_colors, background=tf.zeros([frame_height, frame_width, 3]), width=frame_width, height=frame_height, channels=3 ) return pixels #,cube_vertices_object, cube_faces# use this to dum color