class VisRenderer(object): """ Utility to render meshes using pytorch NMR faces are F x 3 or 1 x F x 3 numpy """ def __init__(self, img_size, faces, t_size=3): self.renderer = NeuralRenderer(img_size) self.faces = Variable(torch.IntTensor(faces).cuda(), requires_grad=False) if self.faces.dim() == 2: self.faces = torch.unsqueeze(self.faces, 0) default_tex = np.ones( (1, self.faces.shape[1], t_size, t_size, t_size, 3)) blue = np.array([156, 199, 234.]) / 255. default_tex = default_tex * blue # Could make each triangle different color self.default_tex = Variable(torch.FloatTensor(default_tex).cuda(), requires_grad=False) # rot = transformations.quaternion_about_axis(np.pi/8, [1, 0, 0]) # This is median quaternion from sfm_pose # rot = np.array([ 0.66553962, 0.31033762, -0.02249813, 0.01267084]) # This is the side view: import cv2 R0 = cv2.Rodrigues(np.array([np.pi / 3, 0, 0]))[0] R1 = cv2.Rodrigues(np.array([0, np.pi / 2, 0]))[0] R = R1.dot(R0) R = np.vstack((np.hstack((R, np.zeros((3, 1)))), np.array([0, 0, 0, 1]))) rot = transformations.quaternion_from_matrix(R, isprecise=True) cam = np.hstack([0.75, 0, 0, rot]) self.default_cam = Variable(torch.FloatTensor(cam).cuda(), requires_grad=False) self.default_cam = torch.unsqueeze(self.default_cam, 0) def __call__(self, verts, cams=None, texture=None, rend_mask=False): """ verts is |V| x 3 cuda torch Variable cams is 7, cuda torch Variable Returns N x N x 3 numpy """ #print("visrender call") if texture is None: texture = self.default_tex elif texture.dim() == 5: # Here input it F x T x T x T x 3 (instead of F x T x T x 3) # So add batch dim. texture = torch.unsqueeze(texture, 0) if cams is None: cams = self.default_cam elif cams.dim() == 1: cams = torch.unsqueeze(cams, 0) if verts.dim() == 2: verts = torch.unsqueeze(verts, 0) #------------------------------ edited by parker #------------------------------this is edited bird mesh---- f = open("edited_bird_mesh.off", "w") f.write("OFF\n") line = str(len(verts[0])) + " " + str(len(self.faces[0])) + " 0\n" f.write(line) mesh_x = np.empty(len(verts[0])) mesh_y = np.empty(len(verts[0])) mesh_z = np.empty(len(verts[0])) # print("bird_vis verts:",verts[0]) for i in range(len(verts[0])): line = str(float(verts[0][i][0])) + " " + str(float( verts[0][i][1])) + " " + str(float(verts[0][i][2])) + "\n" f.write(line) for j in range(3): if (j == 0): mesh_x[i] = verts[0][i][j] elif (j == 1): mesh_y[i] = verts[0][i][j] else: mesh_z[i] = verts[0][i][j] tri_i = np.empty(len(self.faces[0])) tri_j = np.empty(len(self.faces[0])) tri_k = np.empty(len(self.faces[0])) for i in range(len(self.faces[0])): line = str(3) + " " + str(int(self.faces[0][i][0])) + " " + str( int(self.faces[0][i][1])) + " " + str(int( self.faces[0][i][2])) + "\n" f.write(line) for j in range(3): if (j == 0): tri_i[i] = self.faces[0][i][j] elif (j == 1): tri_j[i] = self.faces[0][i][j] else: tri_k[i] = self.faces[0][i][j] # fig = go.Figure( # data=[go.Mesh3d(x=mesh_x, y=mesh_y, z=mesh_z, color='lightblue', opacity=0.5, i=tri_i, j=tri_j, k=tri_k)]) # fig.show() f.close() # ----------------------edited by parker----------------- verts = asVariable(verts) cams = asVariable(cams) texture = asVariable(texture) if rend_mask: rend = self.renderer.forward(verts, self.faces, cams) rend = rend.repeat(3, 1, 1) rend = rend.unsqueeze(0) else: rend = self.renderer.forward(verts, self.faces, cams, texture) rend = rend.data.cpu().numpy()[0].transpose((1, 2, 0)) rend = np.clip(rend, 0, 1) * 255.0 return rend.astype(np.uint8) def rotated(self, vert, deg, axis=[0, 1, 0], cam=None, texture=None): """ vert is N x 3, torch FloatTensor (or Variable) """ import cv2 new_rot = cv2.Rodrigues(np.deg2rad(deg) * np.array(axis))[0] new_rot = convert_as(torch.FloatTensor(new_rot), vert) center = vert.mean(0) new_vert = torch.t(torch.matmul(new_rot, torch.t(vert - center))) + center # new_vert = torch.matmul(vert - center, new_rot) + center return self.__call__(new_vert, cams=cam, texture=texture) def diff_vp(self, verts, cam=None, angle=90, axis=[1, 0, 0], texture=None, kp_verts=None, new_ext=None, extra_elev=False): if cam is None: cam = self.default_cam[0] if new_ext is None: new_ext = [0.6, 0, 0] # Cam is 7D: [s, tx, ty, rot] import cv2 cam = asVariable(cam) quat = cam[-4:].view(1, 1, -1) R = transformations.quaternion_matrix( quat.squeeze().data.cpu().numpy())[:3, :3] rad_angle = np.deg2rad(angle) rotate_by = cv2.Rodrigues(rad_angle * np.array(axis))[0] # new_R = R.dot(rotate_by) new_R = rotate_by.dot(R) if extra_elev: # Left multiply the camera by 30deg on X. R_elev = cv2.Rodrigues(np.array([np.pi / 9, 0, 0]))[0] new_R = R_elev.dot(new_R) # Make homogeneous new_R = np.vstack( [np.hstack((new_R, np.zeros((3, 1)))), np.array([0, 0, 0, 1])]) new_quat = transformations.quaternion_from_matrix(new_R, isprecise=True) new_quat = Variable(torch.Tensor(new_quat).cuda(), requires_grad=False) # new_cam = torch.cat([cam[:-4], new_quat], 0) new_ext = Variable(torch.Tensor(new_ext).cuda(), requires_grad=False) new_cam = torch.cat([new_ext, new_quat], 0) rend_img = self.__call__(verts, cams=new_cam, texture=texture) if kp_verts is None: return rend_img else: kps = self.renderer.project_points(kp_verts.unsqueeze(0), new_cam.unsqueeze(0)) kps = kps[0].data.cpu().numpy() return kp2im(kps, rend_img, radius=1) def set_bgcolor(self, color): self.renderer.set_bgcolor(color) def set_light_dir(self, direction, int_dir=0.8, int_amb=0.8): renderer = self.renderer.renderer renderer.light_direction = direction renderer.light_intensity_directional = int_dir renderer.light_intensity_ambient = int_amb
class VisRenderer(object): """ Utility to render meshes using pytorch NMR faces are F x 3 or 1 x F x 3 numpy """ def __init__(self, img_size, faces, opts, t_size=3, uv_sampler=None): self.opts = opts # TODO junzhe option of renderer if self.opts.renderer_opt == 'nmr': from nnutils.nmr import NeuralRenderer elif self.opts.renderer_opt == 'nmr_kaolin': from nnutils.nmr_kaolin import NeuralRenderer elif self.opts.renderer_opt == 'dibr_kaolin': from nnutils.dibr_kaolin import NeuralRenderer else: raise NotImplementedError self.renderer = NeuralRenderer(img_size, uv_sampler=uv_sampler) self.faces = Variable(torch.IntTensor(faces).cuda(), requires_grad=False) if self.faces.dim() == 2: self.faces = torch.unsqueeze(self.faces, 0) default_tex = np.ones( (1, self.faces.shape[1], t_size, t_size, t_size, 3)) blue = np.array([156, 199, 234.]) / 255. default_tex = default_tex * blue # Could make each triangle different color self.default_tex = Variable(torch.FloatTensor(default_tex).cuda(), requires_grad=False) # rot = transformations.quaternion_about_axis(np.pi/8, [1, 0, 0]) # This is median quaternion from sfm_pose # rot = np.array([ 0.66553962, 0.31033762, -0.02249813, 0.01267084]) # This is the side view: import cv2 R0 = cv2.Rodrigues(np.array([np.pi / 3, 0, 0]))[0] R1 = cv2.Rodrigues(np.array([0, np.pi / 2, 0]))[0] R = R1.dot(R0) R = np.vstack((np.hstack((R, np.zeros((3, 1)))), np.array([0, 0, 0, 1]))) rot = transformations.quaternion_from_matrix(R, isprecise=True) cam = np.hstack([0.75, 0, 0, rot]) self.default_cam = Variable(torch.FloatTensor(cam).cuda(), requires_grad=False) self.default_cam = torch.unsqueeze(self.default_cam, 0) def __call__(self, verts, cams=None, texture=None, rend_mask=False): """ verts is |V| x 3 cuda torch Variable cams is 7, cuda torch Variable Returns N x N x 3 numpy """ if texture is None: texture = self.default_tex elif texture.dim() == 5: # Here input it F x T x T x T x 3 (instead of F x T x T x 3) # So add batch dim. texture = torch.unsqueeze(texture, 0) if cams is None: cams = self.default_cam elif cams.dim() == 1: cams = torch.unsqueeze(cams, 0) if verts.dim() == 2: verts = torch.unsqueeze(verts, 0) verts = asVariable(verts) cams = asVariable(cams) texture = asVariable(texture) if rend_mask: rend = self.renderer.forward(verts, self.faces, cams) rend = rend.repeat(3, 1, 1) rend = rend.unsqueeze(0) else: rend = self.renderer.forward(verts, self.faces, cams, texture) rend = rend.data.cpu().numpy()[0].transpose((1, 2, 0)) rend = np.clip(rend, 0, 1) * 255.0 return rend.astype(np.uint8) def rotated(self, vert, deg, axis=[0, 1, 0], cam=None, texture=None): """ vert is N x 3, torch FloatTensor (or Variable) """ import cv2 new_rot = cv2.Rodrigues(np.deg2rad(deg) * np.array(axis))[0] new_rot = convert_as(torch.FloatTensor(new_rot), vert) center = vert.mean(0) new_vert = torch.t(torch.matmul(new_rot, torch.t(vert - center))) + center # new_vert = torch.matmul(vert - center, new_rot) + center return self.__call__(new_vert, cams=cam, texture=texture) def diff_vp(self, verts, cam=None, angle=90, axis=[1, 0, 0], texture=None, kp_verts=None, new_ext=None, extra_elev=False): if cam is None: cam = self.default_cam[0] if new_ext is None: new_ext = [0.6, 0, 0] # Cam is 7D: [s, tx, ty, rot] import cv2 cam = asVariable(cam) quat = cam[-4:].view(1, 1, -1) R = transformations.quaternion_matrix( quat.squeeze().data.cpu().numpy())[:3, :3] rad_angle = np.deg2rad(angle) rotate_by = cv2.Rodrigues(rad_angle * np.array(axis))[0] # new_R = R.dot(rotate_by) new_R = rotate_by.dot(R) if extra_elev: # Left multiply the camera by 30deg on X. R_elev = cv2.Rodrigues(np.array([np.pi / 9, 0, 0]))[0] new_R = R_elev.dot(new_R) # Make homogeneous new_R = np.vstack( [np.hstack((new_R, np.zeros((3, 1)))), np.array([0, 0, 0, 1])]) new_quat = transformations.quaternion_from_matrix(new_R, isprecise=True) new_quat = Variable(torch.Tensor(new_quat).cuda(), requires_grad=False) # new_cam = torch.cat([cam[:-4], new_quat], 0) new_ext = Variable(torch.Tensor(new_ext).cuda(), requires_grad=False) new_cam = torch.cat([new_ext, new_quat], 0) rend_img = self.__call__(verts, cams=new_cam, texture=texture) if kp_verts is None: return rend_img else: kps = self.renderer.project_points(kp_verts.unsqueeze(0), new_cam.unsqueeze(0)) kps = kps[0].data.cpu().numpy() return kp2im(kps, rend_img, radius=1) def set_bgcolor(self, color): self.renderer.set_bgcolor(color) def set_light_dir(self, direction, int_dir=0.8, int_amb=0.8): renderer = self.renderer.renderer renderer.light_direction = direction renderer.light_intensity_directional = int_dir renderer.light_intensity_ambient = int_amb