def render(cls, mesh: TriangleMesh, dist, elev, azim, uv=None, texture=None, img_size=128): renderer = DIBRenderer(img_size, img_size, mode='Phong') cls.check_mesh_parameters(mesh, uv, texture) dist, elev, azim = cls.check_camera_parameters(dist, elev, azim) renderer.set_look_at_parameters([azim], [elev], [dist]) vertices = mesh.vertices.clone().to(DEVICE)[None] faces = mesh.faces.clone().to(DEVICE) if uv is None and texture is None: rand_color = cls.get_random_color(vertices.size(1)) uv, texture = rand_color[0], rand_color[1] render_imgs, render_alphas, face_norms = renderer.forward( points=[vertices, faces], uv_bxpx2=uv, texture_bx3xthxtw=texture, lightdirect_bx3=light, material_bx3x3=material, shininess_bx1=shininess) return render_imgs, render_alphas, face_norms
def main(): args = parse_arguments() ########################### # Load mesh ########################### mesh = TriangleMesh.from_obj(args.mesh) vertices = mesh.vertices faces = mesh.faces.int() # Expand such that batch size = 1 vertices = vertices[None, :, :].cuda() faces = faces[None, :, :].cuda() ########################### # Normalize mesh position ########################### vertices_max = vertices.max() vertices_min = vertices.min() vertices_middle = (vertices_max + vertices_min) / 2. vertices = (vertices - vertices_middle) * MESH_SIZE ########################### # Generate vertex color ########################### vert_min = torch.min(vertices, dim=1, keepdims=True)[0] vert_max = torch.max(vertices, dim=1, keepdims=True)[0] colors = (vertices - vert_min) / (vert_max - vert_min) ########################### # Render ########################### renderer = Renderer(HEIGHT, WIDTH, mode='VertexColor') loop = tqdm.tqdm(list(range(0, 360, 4))) loop.set_description('Drawing') os.makedirs(args.output_path, exist_ok=True) writer = imageio.get_writer(os.path.join(args.output_path, 'example.gif'), mode='I') for azimuth in loop: renderer.set_look_at_parameters([90 - azimuth], [CAMERA_ELEVATION], [CAMERA_DISTANCE]) predictions, _, _ = renderer(points=[vertices, faces[0].long()], colors=[colors]) image = predictions.detach().cpu().numpy()[0] writer.append_data((image * 255).astype(np.uint8)) writer.close()
def __init__(self, N=6, f_dim=256, point_num=1024): super(Renderer, self).__init__() ''' 初始化参数: ''' self.N = N self.f_dim = f_dim self.point_num = point_num # DIB渲染器 # self.renderer = DIBRenderer(height=640, width=400, mode='Lambertian',camera_fov_y=66.96 * np.pi / 180.0) self.renderer = DIBRenderer(height=256, width=256, mode='Lambertian', camera_fov_y=66.96 * np.pi / 180.0) self.renderer.set_look_at_parameters([0], [0], [0], fovx=57.77316 * np.pi / 180.0, fovy=44.95887 * np.pi / 180.0, near=0.01, far=10.0) # 设置相机参数 # self.renderer.set_camera_parameters() # 预定义相机外参 # 真实参数 # 相机全部往中心移动 # x = 0.597105 # y = 1.062068 # z = 1.111316 # x = -0.3 # y = 1.2 # z = 0.1 x = 0 y = 0 z = 0 self.cam_mat = np.array( [[[0.57432211, 0.77105488, 0.27500633], [-0.56542476, 0.13069975, 0.81437854], [0.59198729, -0.62321099, 0.51103728]], [[0.45602357, 0.72700674, 0.51332611], [0.14605884, -0.63010819, 0.76264702], [0.87790052, -0.2728092, -0.39352995]], [[0.60918383, 0.52822546, 0.59150057], [0.73834933, -0.64995523, -0.17999572], [0.28937056, 0.54638454, -0.78595714]], [[7.71746128e-01, 4.78767298e-01, 4.18556793e-01], [4.76878378e-01, -2.72559500e-04, -8.78969248e-01], [-4.20707651e-01, 8.77941797e-01, -2.28524119e-01]], [[0.78888283, 0.55521065, 0.2634483], [-0.15905217, 0.5985437, -0.78514193], [-0.59360448, 0.57748297, 0.56048831]], [[0.71232121, 0.68900052, 0.13370407], [-0.69422699, 0.71968522, -0.01010355], [-0.10318619, -0.085624, 0.99096979]]]) for i in range(6): self.cam_mat[i] = self.cam_mat[i] @ R self.cam_mat = torch.FloatTensor(self.cam_mat) self.cameras_coordinate = [ [2.50436065 + x, -3.75589484 + y, 1.88800446 + z], [4.02581981 + x, -2.56894275 + y, -3.29281609 + z], [1.01348544 + x, 1.88043939 + y, -5.4273143 + z], [-2.45261002 + x, 3.5962286 + y, -1.87506165 + z], [-3.12155638 + x, 2.09254542 + y, 2.21770186 + z], [-1.07692383 + x, -1.37631717 + y, 4.3081322 + z] ] xx = -0.3 yy = 1.2 zz = 0.1 for i in range(6): self.cameras_coordinate[i] = np.array( self.cameras_coordinate[i]) @ R self.cameras_coordinate[i][0] += xx self.cameras_coordinate[i][1] += yy self.cameras_coordinate[i][2] += zz self.cameras_coordinate = torch.FloatTensor(self.cameras_coordinate) if torch.cuda.is_available(): self.cam_mat = torch.FloatTensor(self.cam_mat).cuda() self.cameras_coordinate = torch.FloatTensor( self.cameras_coordinate).cuda() # self.gif_writer = imageio.get_writer('example.gif', mode='I') pass
class Renderer(nn.Module): ''' 上纹理+渲染 ''' def __init__(self, N=6, f_dim=256, point_num=1024): super(Renderer, self).__init__() ''' 初始化参数: ''' self.N = N self.f_dim = f_dim self.point_num = point_num # DIB渲染器 # self.renderer = DIBRenderer(height=640, width=400, mode='Lambertian',camera_fov_y=66.96 * np.pi / 180.0) self.renderer = DIBRenderer(height=256, width=256, mode='Lambertian', camera_fov_y=66.96 * np.pi / 180.0) self.renderer.set_look_at_parameters([0], [0], [0], fovx=57.77316 * np.pi / 180.0, fovy=44.95887 * np.pi / 180.0, near=0.01, far=10.0) # 设置相机参数 # self.renderer.set_camera_parameters() # 预定义相机外参 # 真实参数 # 相机全部往中心移动 # x = 0.597105 # y = 1.062068 # z = 1.111316 # x = -0.3 # y = 1.2 # z = 0.1 x = 0 y = 0 z = 0 self.cam_mat = np.array( [[[0.57432211, 0.77105488, 0.27500633], [-0.56542476, 0.13069975, 0.81437854], [0.59198729, -0.62321099, 0.51103728]], [[0.45602357, 0.72700674, 0.51332611], [0.14605884, -0.63010819, 0.76264702], [0.87790052, -0.2728092, -0.39352995]], [[0.60918383, 0.52822546, 0.59150057], [0.73834933, -0.64995523, -0.17999572], [0.28937056, 0.54638454, -0.78595714]], [[7.71746128e-01, 4.78767298e-01, 4.18556793e-01], [4.76878378e-01, -2.72559500e-04, -8.78969248e-01], [-4.20707651e-01, 8.77941797e-01, -2.28524119e-01]], [[0.78888283, 0.55521065, 0.2634483], [-0.15905217, 0.5985437, -0.78514193], [-0.59360448, 0.57748297, 0.56048831]], [[0.71232121, 0.68900052, 0.13370407], [-0.69422699, 0.71968522, -0.01010355], [-0.10318619, -0.085624, 0.99096979]]]) for i in range(6): self.cam_mat[i] = self.cam_mat[i] @ R self.cam_mat = torch.FloatTensor(self.cam_mat) self.cameras_coordinate = [ [2.50436065 + x, -3.75589484 + y, 1.88800446 + z], [4.02581981 + x, -2.56894275 + y, -3.29281609 + z], [1.01348544 + x, 1.88043939 + y, -5.4273143 + z], [-2.45261002 + x, 3.5962286 + y, -1.87506165 + z], [-3.12155638 + x, 2.09254542 + y, 2.21770186 + z], [-1.07692383 + x, -1.37631717 + y, 4.3081322 + z] ] xx = -0.3 yy = 1.2 zz = 0.1 for i in range(6): self.cameras_coordinate[i] = np.array( self.cameras_coordinate[i]) @ R self.cameras_coordinate[i][0] += xx self.cameras_coordinate[i][1] += yy self.cameras_coordinate[i][2] += zz self.cameras_coordinate = torch.FloatTensor(self.cameras_coordinate) if torch.cuda.is_available(): self.cam_mat = torch.FloatTensor(self.cam_mat).cuda() self.cameras_coordinate = torch.FloatTensor( self.cameras_coordinate).cuda() # self.gif_writer = imageio.get_writer('example.gif', mode='I') pass def forward(self, vertex_positions, mesh_faces, input_uvs, input_texture, mesh_face_textures): ''' 输入: vertices : 顶点数*3 faces : 面数*3 uv : uv坐标值 : 顶点数x2 texture : NxCxHxW 图像 输出: N个视角下的渲染图像:NxCxHxW ''' images = [] img_probs = [] for i in range(self.N): # N个视角下渲染,每次渲染一个视角下的batchsize张图片 # 设置相机参数 camera_view_mtx = self.cam_mat[i].repeat(vertex_positions.shape[0], 1, 1) camera_view_shift = self.cameras_coordinate[i].repeat( vertex_positions.shape[0], 1) self.renderer.camera_params = [ camera_view_mtx, camera_view_shift, self.renderer.camera_params[2] ] predictions, img_prob, _ = self.renderer( points=[vertex_positions, mesh_faces], uv_bxpx2=input_uvs, texture_bx3xthxtw=input_texture, ft_fx3=mesh_face_textures) predictions = torch.cat((predictions, img_prob), dim=3).permute(0, 3, 1, 2) temp = predictions.detach().cpu().numpy()[0] # self.gif_writer.append_data((temp * 255).astype(np.uint8)) images.append(predictions) img_probs.append(img_prob) return images, img_probs def get_camera_info(self): return self.cam_mat, self.cameras_coordinate
import torch from kaolin.rep import TriangleMesh from kaolin.graphics import DIBRenderer from config import DEVICE renderer = DIBRenderer(128, 128) class VertexRenderer: def __init__(self): pass @classmethod def render(cls, mesh, dist, elev, azim, colors=None): isinstance(mesh, TriangleMesh) dist, elev, azim = cls.check_camera_parameters(dist, elev, azim) renderer.set_look_at_parameters([azim], [elev], [dist]) vertices = mesh.vertices.clone().to(DEVICE)[None] faces = mesh.faces.clone().to(DEVICE) colors = torch.ones_like(vertices).to( DEVICE) if colors is None else colors render_rgbs, render_alphas, face_norms = renderer.forward( points=[vertices, faces], colors_bxpx3=colors) return render_rgbs, render_alphas, face_norms @staticmethod def check_camera_parameters(dist, elev, azim): if isinstance(dist, torch.Tensor):
def main(): args = parse_arguments() ########################### # Load mesh ########################### mesh = TriangleMesh.from_obj(args.mesh) vertices = mesh.vertices.cuda() faces = mesh.faces.int().cuda() # Expand such that batch size = 1 vertices = vertices.unsqueeze(0) ########################### # Normalize mesh position ########################### vertices_max = vertices.max() vertices_min = vertices.min() vertices_middle = (vertices_max + vertices_min) / 2. vertices = (vertices - vertices_middle) * MESH_SIZE ########################### # Generate vertex color ########################### if not args.use_texture: vert_min = torch.min(vertices, dim=1, keepdims=True)[0] vert_max = torch.max(vertices, dim=1, keepdims=True)[0] colors = (vertices - vert_min) / (vert_max - vert_min) ########################### # Generate texture mapping ########################### if args.use_texture: uv = get_spherical_coords_x(vertices[0].cpu().numpy()) uv = torch.from_numpy(uv).cuda() # Expand such that batch size = 1 uv = uv.unsqueeze(0) ########################### # Load texture ########################### if args.use_texture: # Load image as numpy array texture = np.array(Image.open(args.texture)) # Convert numpy array to PyTorch tensor texture = torch.from_numpy(texture).cuda() # Convert from [0, 255] to [0, 1] texture = texture.float() / 255.0 # Convert to NxCxHxW layout texture = texture.permute(2, 0, 1).unsqueeze(0) ########################### # Render ########################### if args.use_texture: renderer_mode = 'Lambertian' else: renderer_mode = 'VertexColor' renderer = Renderer(HEIGHT, WIDTH, mode=renderer_mode) loop = tqdm.tqdm(list(range(0, 360, 4))) loop.set_description('Drawing') os.makedirs(args.output_path, exist_ok=True) writer = imageio.get_writer(os.path.join(args.output_path, 'example.gif'), mode='I') for azimuth in loop: renderer.set_look_at_parameters([90 - azimuth], [CAMERA_ELEVATION], [CAMERA_DISTANCE]) if args.use_texture: predictions, _, _ = renderer(points=[vertices, faces.long()], uv_bxpx2=uv, texture_bx3xthxtw=texture) else: predictions, _, _ = renderer(points=[vertices, faces.long()], colors_bxpx3=colors) image = predictions.detach().cpu().numpy()[0] writer.append_data((image * 255).astype(np.uint8)) writer.close()
batch_size=args.batchsize, shuffle=False, collate_fn=collate_fn, num_workers=0) # Model mesh = kal.rep.TriangleMesh.from_obj('386.obj', enable_adjacency=True) mesh.cuda() normalize_adj(mesh) initial_verts = mesh.vertices.clone() camera_fov_y = 49.13434207744484 * np.pi / 180.0 cam_proj = perspectiveprojectionnp(camera_fov_y, 1.0) cam_proj = torch.FloatTensor(cam_proj).cuda() model = Encoder(4, 5, args.batchsize, 137, mesh.vertices.shape[0]).cuda() renderer = Dib_Renderer(137, 137, mode='VertexColor') model.load_state_dict(torch.load('log/{0}/best.pth'.format(args.expid))) loss_epoch = 0. f_epoch = 0. num_batches = 0 num_items = 0 loss_fn = kal.metrics.point.chamfer_distance model.eval() with torch.no_grad(): for data in (tqdm(dataloader_val)): # data creation
def visualisemask(vertices, faces, PdroneToCamera, b, isPredicted=True): """Set up renderer for visualising drone based on the predicted pose PdroneToCamera: relative pose predicted by network or GT """ if not __KAOLIN_LOADED__: return device = vertices.get_device() vertices = vertices.expand(b, vertices.size(1), vertices.size(2)) colors = torch.zeros(vertices.size()).to(device) #setup color vert_min = torch.min(vertices, dim=1, keepdims=True)[0] vert_max = torch.max(vertices, dim=1, keepdims=True)[0] if (isPredicted): #colors[:,:,:2] = 0 colors[:, :, :3] = 1 else: colors[:, :, 2:3] = 0 colors[:, :, 1] = 1 #get homogeneous coordinates for vertices vertices = torch.torch.nn.functional.pad(vertices[:, :, ], [0, 1], "constant", 1.0) vertices = vertices.transpose(2, 1) vertices = torch.matmul(PdroneToCamera, vertices) #set up renderer renderer = Renderer(320 * 1, 240 * 1) vertices = vertices.transpose(2, 1) b, _, _ = PdroneToCamera.size() #set camera parameters cameras = [] camera_rot_bx3x3 = torch.zeros((b, 3, 3), dtype=torch.float32).to(device) camera_rot_bx3x3[:, 0, 0] = 1 camera_rot_bx3x3[:, 1, 1] = 1 camera_rot_bx3x3[:, 2, 2] = 1 cameras.append(camera_rot_bx3x3) camera_pos_bx3 = torch.zeros((b, 3), dtype=torch.float32).to(device) cameras.append(camera_pos_bx3) camera_proj_3x1 = torch.zeros((3, 1), dtype=torch.float32).to(device) camera_proj_3x1[:, :] = torch.from_numpy( geometry.perspectiveprojectionnp(radians(50.8), ratio=4. / 3., near=0.3, far=750.0)) cameras.append(camera_proj_3x1) renderer.set_camera_parameters(cameras) #convert points from homogeneous z_vec = vertices[..., -1:] scale = torch.tensor(1.) / torch.clamp(z_vec, 0.000000001) vertices = vertices[..., :-1] #forward pass predictions, mask, _ = renderer(points=[vertices, faces.long()], colors_bxpx3=colors) return predictions, mask
def main(): args = parse_arguments() ########################### # Load mesh ########################### filename = os.path.join(current_dir, args.filename_input) #print(filename) os.makedirs(args.output, exist_ok=True) data_dir_1 = os.path.join(args.output, 'rendered_1.5_600', filename.split('/')[6], filename.split('/')[7]) mesh = TriangleMesh.from_obj(filename, with_vt=True, texture_res=5) vertices = mesh.vertices faces = mesh.faces.long() face_textures = mesh.face_textures textures = mesh.textures uvs = mesh.uvs ## trying to corect for pytorch coordinates #uvs[:, 1] = 1 - uvs[:, 1] #uvs = uvs * 2 - 1 pfmtx = face2pfmtx(faces.numpy()) # Expand such that batch size = 1 vertices = vertices[None, :, :].cuda() faces = faces[None, :, :].cuda() face_textures = face_textures[None, :, :].cuda() textures = textures[None, :, :, :].cuda() uvs = uvs[None, :, :].cuda() textures = textures.permute([0, 3, 1, 2]) #vertices = vertices.unsqueeze(0) #print(vertices.shape, faces.shape, textures.shape, uvs.shape) ########################### # Normalize mesh position ########################### vertices_max = vertices.max() vertices_min = vertices.min() vertices_middle = (vertices_max + vertices_min) / 2. vertices = (vertices - vertices_middle) * MESH_SIZE ########################### # Generate vertex color ########################### #vert_min = torch.min(vertices, dim=1, keepdims=True)[0] #vert_max = torch.max(vertices, dim=1, keepdims=True)[0] #colors = (vertices - vert_min) / (vert_max - vert_min) ########################### # Mat, Light, and Shine # ########################### bs = 1 material = np.array([[0.1, 0.1, 0.1], [1.0, 1.0, 1.0], [0.4, 0.4, 0.4]], dtype=np.float32).reshape(-1, 3, 3) shininess = np.array([100], dtype=np.float32).reshape(-1, 1) # this was 100 tfmat = torch.from_numpy(material).repeat(bs, 1, 1).cuda() tfshi = torch.from_numpy(shininess).repeat(bs, 1).cuda() lightdirect = np.array([0, 1, 0], dtype=np.float32).reshape( (bs, 3)) #2 * np.random.rand(bs, 3).astype(np.float32) - 1 #lightdirect[:, 2] += 2 tflight = torch.from_numpy(lightdirect) tflight_bx3 = tflight.cuda() ########################### # Render ########################### renderer = Renderer(HEIGHT, WIDTH, mode='Phong') #renderer.renderer.set_smooth(pfmtx) #loop = tqdm.tqdm(list(range(0, 180, 4))) #loop.set_description('Drawing') #writer = imageio.get_writer(os.path.join(args.output_path, 'example.gif'), mode='I') #writer_sil = imageio.get_writer(os.path.join(args.output_path, 'example_sil.gif'), mode='I') theta1 = np.round(np.random.uniform(0, 360, 20), 2) theta2 = tqdm.tqdm(list(np.round(np.random.uniform(0, 360, 20), 2))) os.makedirs(data_dir_1, exist_ok=True) #net = resnet18(pretrained=True).cuda() for num, azimuth in enumerate(theta2): renderer.set_look_at_parameters([90 - azimuth], [theta1[num]], [CAMERA_DISTANCE]) predictions, silhouette, _ = renderer( points=[vertices, faces[0].long()], uv_bxpx2=uvs, ft_fx3=face_textures[0], texture_bx3xthxtw=textures, lightdirect_bx3=tflight_bx3, material_bx3x3=tfmat, shininess_bx1=tfshi) image = predictions.detach().cpu().numpy()[0] #imageio.imwrite('results/example.png', (255*image[:,:,:]).astype(np.uint8)) #writer.append_data((image * 255).astype(np.uint8)) mask = silhouette.detach().cpu().numpy()[0] #print(mask.shape, image.shape) whole_image = np.concatenate((image, mask), axis=2) #print(whole_image.shape) imageio.imwrite( ('%s/t1_%s_t2_%s.png' % (data_dir_1, theta1[num], azimuth)), (255 * whole_image[:, :, :]).astype(np.uint8)) #imageio.imwrite('results/example_sil.png', (255*gray[:,:,:]).astype(np.uint8)) #writer_sil.append_data((gray * 255).astype(np.uint8)) #writer.close() '''os.makedirs(data_dir_1,exist_ok=True)