def _create_renderer_mesh( # pylint: disable=too-many-arguments w=640, h=480, rt=np.zeros(3), t=np.zeros(3), f=None, c=None, k=None, near=1., far=10., texture=None): """Create a renderer for the specified parameters.""" f = np.array([w, w]) / 2. if f is None else f c = np.array([w, h]) / 2. if c is None else c k = np.zeros(5) if k is None else k if texture is not None: rn = TexturedRenderer() else: rn = ColoredRenderer() rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k) rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w} if texture is not None: rn.texture_image = np.asarray(texture, np.float64) / 255. rn.texture_image = rn.texture_image[:, :, ::-1] #print rn.texture_image.shape return rn
def renderBody(m): from opendr.camera import ProjectPoints from opendr.renderer import ColoredRenderer from opendr.lighting import LambertianPointLight # Create OpenDR renderer rn = ColoredRenderer() # Assign attributes to renderer w, h = (640, 480) rn.camera = ProjectPoints(v=m, rt=np.zeros(3), t=np.array([0, 0, 2.]), f=np.array([w, w]) / 2., c=np.array([w, h]) / 2., k=np.zeros(5)) rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h} rn.set(v=m, f=m.f, bgcolor=np.zeros(3)) # Construct point light source rn.vc = LambertianPointLight(f=m.f, v=rn.v, num_verts=len(m), light_pos=np.array([-1000, -1000, -2000]), vc=np.ones_like(m) * .9, light_color=np.array([1., 1., 1.])) plt.ion() plt.imshow(np.fliplr(rn.r)) # FLIPPED! plt.show() plt.xticks([]) plt.yticks([])
def initialize_camera(fx, fy, cx, cy): """Initialize camera translation and body orientation :param model: SMPL model :param j2d: 14x2 array of CNN joints :param img: h x w x 3 image :param init_pose: 72D vector of pose parameters used for initialization :param flength: camera focal length (kept fixed) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the estimated camera, a boolean deciding if both the optimized body orientation and its flip should be considered, 3D vector for the body orientation """ rt = ch.zeros(3) t = ch.zeros(3) # check how close the shoulder joints are # initialize the camera cam = ProjectPoints(f=np.array([fx, fy]), rt=rt, t=t, k=np.zeros(5), c=[cx, cy]) return cam
def main(mesh_list, out_list, scale=1.0, move_scale=True): assert len(mesh_list) == len(out_list) for mesh_file, out_file in zip(mesh_list, out_list): mesh = load_obj_data_binary(mesh_file) if move_scale: # move to center and scale to unit bounding box mesh['v'] = (mesh['v'] - np.array([128, -192, 128]) + 0.5) * voxel_size if not ('vn' in mesh and mesh['vn'] is not None): mesh['vn'] = np.array(VertNormals(f=mesh['f'], v=mesh['v'])) V = ch.array(mesh['v']) * scale V -= trans C = np.ones_like(mesh['v']) C *= np.array([186, 212, 255], dtype=np.float32) / 255.0 # C *= np.array([158, 180, 216], dtype=np.float32) / 250.0 C = np.minimum(C, 1.0) A = np.zeros_like(mesh['v']) A += LambertianPointLight(f=mesh['f'], v=V, vn=-mesh['vn'], num_verts=len(V), light_pos=np.array([0, -50, -50]), light_color=np.array([1.0, 1.0, 1.0]), vc=C) cam_t, cam_r = ch.array((0, 0, 0)), ch.array((3.14, 0, 0)) U = ProjectPoints(v=V, f=[flength, flength], c=[w / 2., h / 2.], k=ch.zeros(5), t=cam_t, rt=cam_r) rn = ColoredRenderer(camera=U, v=V, f=mesh['f'], vc=A, bgcolor=np.array([1.0, 0.0, 0.0]), frustum={ 'width': w, 'height': h, 'near': 0.1, 'far': 20 }) img = np.asarray(rn)[:, :, (2, 1, 0)] msk = np.sum(np.abs(img - np.array([[[0, 0, 1.0]]], dtype=np.float32)), axis=-1, keepdims=True) msk[msk > 0] = 1 img = cv.resize(img, (img.shape[1] // 2, img.shape[0] // 2)) msk = cv.resize(msk, (msk.shape[1] // 2, msk.shape[0] // 2), interpolation=cv.INTER_AREA) msk[msk < 1] = 0 msk = msk[:, :, np.newaxis] img = np.concatenate([img, msk], axis=-1) cv.imshow('render3', img) cv.waitKey(3) cv.imwrite(out_file, np.uint8(img * 255))
def create_renderer(w=640, h=480, rt=np.zeros(3), t=np.zeros(3), f=None, c=None, k=None, near=.5, far=10., mesh=None): f = np.array([w, w]) / 2. if f is None else f c = np.array([w, h]) / 2. if c is None else c k = np.zeros(5) if k is None else k if mesh is not None and hasattr(mesh, 'texture_image'): from opendr.renderer import TexturedRenderer rn = TexturedRenderer() rn.texture_image = mesh.texture_image if rn.texture_image.max() > 1: rn.texture_image[:] = rn.texture_image[:].r / 255. rn.ft = mesh.ft rn.vt = mesh.vt else: from opendr.renderer import ColoredRenderer rn = ColoredRenderer() rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k) rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w} return rn
def main(vt_ft_path, gar_type, mesh_location, texture_location, inpaint_mask, side, cam_file, save_tex_location): mesh = Mesh(filename=mesh_location) img = cv2.imread(texture_location) / 255. img = cv2.resize(img, (1000, 1000)) cam_data = pkl.load(open(cam_file, 'r')) cam_y, cam_z = cam_data[gar_type]['cam_y'], cam_data[gar_type]['cam_z'] camera = ProjectPoints(v=mesh.v, f=np.array([1000, 1000]), c=np.array([1000, 1000]) / 2., t=np.array([0, cam_y, cam_z]), rt=np.zeros(3), k=np.zeros(5)) data = pkl.load(open(vt_ft_path, 'r'))[gar_type] iso = Isomapper(data['vt'], data['ft'], 2000) tex = iso.render(img, camera, mesh.f) if side == "front": tex_save = tex[500:1500, 0:1000] else: tex_save = tex[500:1500, 1000:2000] inpaint_mask = cv2.imread(inpaint_mask, cv2.IMREAD_UNCHANGED) if inpaint_mask is not None: tex_save = cv2.inpaint(np.uint8(tex_save * 255), inpaint_mask, 3, cv2.INPAINT_TELEA) cv2.imwrite(save_tex_location, tex_save)
def Render(): verts = np.load('../../resault/verts.npy') faces = np.load('../../resault/faces.npy') rn = ColoredRenderer() w, h = (640, 480) rn.camera = ProjectPoints(v=verts, rt=np.zeros(3), t=np.array([0, 0, 2.]), f=np.array([w, w]) / 2., c=np.array([w, h]) / 2., k=np.zeros(5)) rn.frustum = {'near': 0.8, 'far': 16., 'width': w, 'height': h} rn.set(v=verts, f=faces, bgcolor=np.array([255, 255, 255])) rn.vc = LambertianPointLight(f=faces, v=rn.v, num_verts=len(verts), light_pos=np.array([-1000, -1000, -2000]), vc=np.ones_like(verts) * .9, light_color=np.array([1., 1., 1.])) # import cv2 # # cv2.imshow('render_SMPL', rn.r) # cv2.waitKey(0) import matplotlib.pyplot as plt plt.ion() plt.axis('off') plt.imshow(rn.r) raw_input() plt.show()
def main(mesh, sv_file, side, size, fmap_location, bmap_location, cam_file, gar_type): from opendr.camera import ProjectPoints from psbody.mesh import Mesh print(fmap_location) cam_data = pkl.load(open(cam_file, 'r')) cam_z, cam_y = cam_data[gar_type]['cam_z'], cam_data[gar_type]['cam_y'] mesh = Mesh(filename=mesh) fmap = np.load(fmap_location) bmap = np.load(bmap_location) cam = ProjectPoints(v=mesh.v, t=np.array([0, cam_y, cam_z]), rt=np.zeros(3), f=[1000, 1000], c=[1000 / 2., 1000 / 2.], k=np.zeros(5)) points = uv_to_xyz_and_normals(mesh, fmap, bmap) cam.v = points projection = cam.r.astype(np.int32) projection[projection > 999] = 999 projection = np.fliplr(np.around(projection.squeeze())).astype(np.int32) pixels_to_set = np.array(np.where(fmap != -1)).T x_to_set = pixels_to_set[:, 0] y_to_set = pixels_to_set[:, 1] cords_ret = -999 * np.ones((fmap.shape[0], fmap.shape[1], 2)) cords_ret[x_to_set, y_to_set, :] = projection cords_ret = cords_ret.astype('float64') cords_ret = 2 * ((cords_ret) / 999) - 1 cords_ret = np.flip(cords_ret, 2) if side == 'front': cords_ret = cords_ret[500:1500, 0:1000] else: cords_ret = cords_ret[500:1500, 1000:2000] cords_ret = cv2.resize(cords_ret, (size, size)) np.save(sv_file, cords_ret)
def render_orth(sv, w, h, cam, img=None, deg=None, margin=None, color_key='blue', use_face=None, vc=None): # Use large focal length to remove perspective effects. # Get projected points using current f. orig_trans = sv.trans.r.copy() # import ipdb; ipdb.set_trace() use_f = 5000. # We have f * X/Z = x = f' * X / Z' where f' is the use_f, solve for Z' new_Z = use_f / cam.f[0].r * orig_trans[2] sv.trans[2] = new_Z # Now project this # dist = np.mean(sv.r, axis=0)[2] min_z = np.maximum(np.min(sv.r, axis=0)[2] - 5, 0.5) max_z = np.max(sv.r, axis=0)[2] cam_f = ProjectPoints(rt=cam.rt, t=cam.t, f=np.array([use_f, use_f]), c=cam.c, k=cam.k) if use_face is None: #use_face = sv.model.f use_face = sv.f if vc is None: img_here = render_mesh(Mesh(sv.r, use_face), w, h, cam_f, near=min_z, far=max_z + 5, img=img, deg=deg, margin=margin, color_key=color_key) else: img_here = render_mesh(Mesh(sv.r, use_face, vc=vc), w, h, cam_f, near=min_z, far=max_z + 5, img=img, deg=deg, margin=margin, color_key=color_key) # Model and sv don't have ties but for safety put it back. sv.trans[:] = orig_trans return img_here
def fit_silhouettes_pyramid_multi_model(objs, sv, silhs, cams, #weights=1., mv=None, fix_shape=False, imgs=None, fix_rot=False, fix_trans=False, s2m_weights=1., m2s_weights=1., max_iter=100, alpha=None, vc=None, symIdx=None, mv2=None, FIX_POSE=False): ''' if alpha is not None, then replace that with sv.betas''' from opendr.camera import ProjectPoints silhs = [np.uint8(silh[:, :, 0] > 0) for silh in silhs] # Setup silhouet term camera. cam_copy = [ProjectPoints( rt=cam.rt, t=cam.t, f=cam.f, c=cam.c, k=cam.k, v=cam.v) for cam in cams] if imgs[0].shape[1] < 900: scales = 1. / (2 * np.array([3, 2, 1, 0.5])) else: scales = 1. / (2 * np.array([6, 4, 3, 2, 1])) res_silh = [] for sc in scales: if 'shape_prior' in objs.keys(): objs['shape_prior'] = 0.4 * objs['shape_prior'] silh_here = [cv2.resize(silh, (int(silh.shape[1] * sc),int(silh.shape[0] * sc))) for silh in silhs] cam_here = [scalecam(cam, sc) for cam in cam_copy] for i,cam in enumerate(cam_copy): cam_here[i].v = cam.v print('Scale %g' % (1 / sc)) w_s2m = s2m_weights w_m2s = m2s_weights R, s_objs = fit_silhouettes_multi_model(objs, sv, silh_here, cam_here, w_s2m, w_m2s, max_iter, mv, fix_shape, cams, imgs, alpha=alpha, fix_trans=fix_trans, pyr_scale=sc, vc=vc, symIdx=symIdx, mv2=mv2, FIX_POSE=FIX_POSE) # For scales < 1 we optimize f on the kp_camera (cams) and then we update cam_copy for i in range(len(cams)): cam_copy[i].f[:] = cam_here[i].f.r/sc res_silh.append(R) # Compute energy E = 0 for term in s_objs.values(): E = E + np.mean(term.r) return res_silh, E
def proj_smpl_onto_img(img, smpl, pose, shape, cam_f, cam_t): if isinstance(cam_f, float): cam_f = np.array([cam_f, cam_f]) smpl.pose[:] = pose smpl.betas[:] = shape center = np.array([img.shape[1] / 2, img.shape[0] / 2]) cam = ProjectPoints( f=cam_f, rt=ch.zeros(3), t=cam_t, k=np.zeros(5), c=center) cam.v = smpl.r for v in cam.r: r = int(round(v[1])) c = int(round(v[0])) if 0 <= r < img.shape[0] and 0 <= c < img.shape[1]: img[r, c, :] = np.asarray([255, 255, 255]) return img
def __call__(self, verts, faces, vert_colors=None, cam=None, img=None, do_alpha=False, far=None, near=None, color_id=0, img_size=None): """ cam is 3D [f, px, py] """ if img is not None: h, w = img.shape[:2] elif img_size is not None: h = img_size[0] w = img_size[1] else: h = self.h w = self.w if cam is None: if self.cam is not None: cam = self.cam else: cam = [self.flength, w / 2., h / 2.] use_cam = ProjectPoints( f=cam[0] * np.ones(2), rt=np.zeros(3), t=np.zeros(3), k=np.zeros(5), c=cam[1:3]) if near is None: near = np.maximum(np.min(verts[:, 2]) - 25, 0.1) if far is None: far = np.maximum(np.max(verts[:, 2]) + 25, 25) imtmp = render_model( verts, faces, w, h, use_cam, vert_colors = vert_colors, do_alpha = do_alpha, img = img, far = far, near = near, color_id = color_id) return (imtmp * 255).astype('uint8')
def project_vertices(v, w, h, cam_r, cam_t): """projects vertices onto image plane""" V = ch.array(v) U = ProjectPoints(v=V, f=[w, w], c=[w / 2., h / 2.], k=ch.zeros(5), t=cam_t, rt=cam_r) return U
def scalecam(cam, imres): if imres == 1: return cam # Returns a camera which shares camera parameters by reference, # and whose scale is resized according to imres return ProjectPoints(rt=cam.rt, t=cam.t, f=array(cam.f.r * imres), c=array(cam.c.r * imres), k=cam.k)
def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model_file, first_frame, last_frame, nohands, naked, display): # load data with open(model_file, 'rb') as fp: model_data = pkl.load(fp) with open(camera_file, 'rb') as fp: camera_data = pkl.load(fp) pose_data = h5py.File(pose_file, 'r') poses = pose_data['pose'][first_frame:last_frame] trans = pose_data['trans'][first_frame:last_frame] masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame] num_frames = masks.shape[0] indices_consensus = np.ceil(np.arange(num) * num_frames * 1. / num).astype(np.int) # init base_smpl = Smpl(model_data) base_smpl.betas[:] = np.array(pose_data['betas'], dtype=np.float32) camera = ProjectPoints(t=np.zeros(3), rt=np.zeros(3), c=camera_data['camera_c'], f=camera_data['camera_f'], k=camera_data['camera_k'], v=base_smpl) camera_t = camera_data['camera_t'] camera_rt = camera_data['camera_rt'] frustum = {'near': 0.1, 'far': 1000., 'width': int(camera_data['width']), 'height': int(camera_data['height'])} frames = [] for i in indices_consensus: log.info('Set up frame {}...'.format(i)) mask = np.array(masks[i] * 255, dtype=np.uint8) pose_i = np.array(poses[i], dtype=np.float32) trans_i = np.array(trans[i], dtype=np.float32) frames.append(setup_frame_rays(base_smpl, camera, camera_t, camera_rt, pose_i, trans_i, mask)) log.info('Set up complete.') log.info('Begin consensus fit...') fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display) with open(out, 'wb') as fp: pkl.dump({ 'v_personal': base_smpl.v_personal.r, 'betas': base_smpl.betas.r, }, fp, protocol=2) if obj_out is not None: base_smpl.pose[:] = 0 vt = np.load('assets/basicModel_vt.npy') ft = np.load('assets/basicModel_ft.npy') mesh.write(obj_out, base_smpl.r, base_smpl.f, vt=vt, ft=ft) log.info('Done.')
def get_cam_rend(verts, faces, cam_y, cam_z): frustum = {'near': 0.1, 'far': 1000., 'width': 1000, 'height': 1000} camera = ProjectPoints(v=verts, t=np.array([0, cam_y, cam_z]), rt=np.zeros(3), f=[1000, 1000], c=[1000 / 2., 1000 / 2.], k=np.zeros(5)) rn_m = ColoredRenderer(camera=camera, v=verts, f=faces, vc=np.ones_like(verts), frustum=frustum, bgcolor=0, num_channels=1) return camera, rn_m
def scalecam(cam, imres): # Returns a camera which shares camera parameters by reference, # and whose scale is resized according to imres from chumpy import array from opendr.camera import ProjectPoints return ProjectPoints( rt=cam.rt, t=cam.t, f=cam.f * imres, c=array(cam.c.r * imres), k=cam.k)
def _project_vertices(v, w, h, cam_r, cam_t): """projects vertices onto image plane""" V = ch.array(v) U = ProjectPoints( v=V, f=[w * 2, w * 2], c=[w / 2., h / 2.], # camera intrinsics k=ch.zeros(5), t=cam_t, rt=cam_r) return U
def __call__(self, verts, cam=None, img=None, do_alpha=False, far=None, near=None, color_id=0, img_size=None): """ cam is 3D [f, px, py] """ if img is not None: h, w = img.shape[:2] elif img_size is not None: h = img_size[0] w = img_size[1] else: h = self.h w = self.w if cam is None: cam = [self.flength, w / 2., h / 2.] use_cam = ProjectPoints(f=cam[0] * np.ones(2), rt=np.zeros(3), t=np.zeros(3), k=np.zeros(5), c=cam[1:3]) if near is None: near = np.maximum(np.min(verts[:, 2]) - 25, 0.1) if far is None: far = np.maximum(np.max(verts[:, 2]) + 25, 25) np_verts = np.array(verts) #np_verts = np.stack((np_verts[:,0],np.max(np_verts[:,1])-np_verts[:,1],np_verts[:,2]),axis=1) self.trimesh = trimesh.Trimesh(vertices=np.asarray(np_verts), faces=self.faces) imtmp = render_model(verts, self.faces, w, h, use_cam, do_alpha=do_alpha, img=img, far=far, near=near, color_id=color_id) return (imtmp * 255).astype('uint8')
def test_earth(): m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3)) # Create V, A, U, f: geometry, brightness, camera, renderer V = ch.array(m.v) A = SphericalHarmonics(vn=VertNormals(v=V, f=m.f), components=[3., 2., 0., 0., 0., 0., 0., 0., 0.], light_color=ch.ones(3)) # camera U = ProjectPoints(v=V, f=[w, w], c=[w / 2., h / 2.], k=ch.zeros(5), t=ch.zeros(3), rt=ch.zeros(3)) f = TexturedRenderer(vc=A, camera=U, f=m.f, bgcolor=[0., 0., 0.], texture_image=m.texture_image, vt=m.vt, ft=m.ft, frustum={ 'width': w, 'height': h, 'near': 1, 'far': 20 }) # Parameterize the vertices translation, rotation = ch.array([0, 0, 8]), ch.zeros(3) f.v = translation + V.dot(Rodrigues(rotation)) observed = f.r np.random.seed(1) # this is reactive # in the sense that changes to values will affect function which depend on them. translation[:] = translation.r + np.random.rand(3) rotation[:] = rotation.r + np.random.rand(3) * .2 # Create the energy E_raw = f - observed E_pyr = gaussian_pyramid(E_raw, n_levels=6, normalization='size') Image.fromarray((observed * 255).astype(np.uint8)).save( os.path.join(save_dir, "reference.png")) step = 0 Image.fromarray((f.r * 255).astype(np.uint8)).save( os.path.join(save_dir, "step_{:05d}.png".format(step))) print('OPTIMIZING TRANSLATION, ROTATION, AND LIGHT PARMS') free_variables = [translation, rotation] ch.minimize({'pyr': E_pyr}, x0=free_variables, callback=create_callback(f)) ch.minimize({'raw': E_raw}, x0=free_variables, callback=create_callback(f))
def setup_camera(w, h, flength=2000, rt=np.zeros(3), t=np.zeros(3), k=np.zeros(5)): '''Width is image.shape[0]!!''' from opendr.camera import ProjectPoints center = np.array([h / 2, w / 2]) f = np.array([flength, flength]) cam = ProjectPoints(f=f, rt=rt, t=t, k=k, c=center) return cam
def get_3DSV(mesh): from opendr.camera import ProjectPoints from opendr.renderer import DepthRenderer WIDTH, HEIGHT = 250, 250 camera = ProjectPoints(v=mesh.vertices, f=np.array([WIDTH, WIDTH]), c=np.array([WIDTH, HEIGHT]) / 2., t=np.array([0, 0, 2.5]), rt=np.array([np.pi, 0, 0]), k=np.zeros(5)) frustum = {'near': 1., 'far': 10., 'width': WIDTH, 'height': HEIGHT} rn = DepthRenderer(camera=camera, frustum=frustum, f=mesh.faces, overdraw=False) points3d = camera.unproject_depth_image(rn.r) points3d = points3d[points3d[:, :, 2] > np.min(points3d[:, :, 2]) + 0.01] # print('sampled {} points.'.format(points3d.shape[0])) return points3d
def setupCamera(v, cameraParams): chDistMat = geometry.Translate(x=0, y=cameraParams['Zshift'], z=cameraParams['chCamHeight']) chRotElMat = geometry.RotateX(a=-cameraParams['chCamEl']) chCamModelWorld = ch.dot(chDistMat, chRotElMat) flipZYRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0, 1.0, 0.0], [0.0, -1.0, 0, 0.0], [0.0, 0.0, 0.0, 1.0]]) chMVMat = ch.dot(chCamModelWorld, flipZYRotation) chInvCam = ch.inv(chMVMat) modelRotation = chInvCam[0:3, 0:3] chRod = opendr.geometry.Rodrigues(rt=modelRotation).reshape(3) chTranslation = chInvCam[0:3, 3] translation, rotation = (chTranslation, chRod) camera = ProjectPoints(v=v, rt=rotation, t=translation, f=1000 * cameraParams['chCamFocalLength'] * cameraParams['a'], c=cameraParams['c'], k=ch.zeros(5)) flipXRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0., 0.0], [0.0, 0., -1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) camera.openglMat = flipXRotation #Needed to match OpenGL flipped axis. return camera, modelRotation, chMVMat
def mesh2Image(vertices, faces, batch, path, name, height, width, vertices_num=6890): # Create OpenDR renderer rn = ColoredRenderer() rt_1 = np.zeros(3) rn.camera = ProjectPoints( v=vertices, # vertices # v=m, rt=rt_1, # x, y, z translation of the camera, z>=0 0 0 2 t=np.array([0, 0, 0]), # f=np.array([w,w])/2, # focus length? just scaling the picture # c=np.array([w,h])/2, # just move the picture along top-left axis? not sure f=np.array([1, 1]), c=np.array([0, 0]), k=np.zeros(5)) rn.frustum = {'near': 1, 'far': 15, 'width': width, 'height': height} rn.set(v=vertices, f=faces, bgcolor=np.zeros(3)) # Construct point light source rn.vc = LambertianPointLight( f=faces, # face v=vertices, # v=rn.v, #vertex? num_verts=len(vertices), light_pos=np.array([-1000, -1000, -2000]), # point light position vc=np.ones_like(vertices) * .9, # albedo per vertex light_color=np.array([1., 1., 1.])) # Blue, Green, Red; light intensity # make the image binary(black and white); these are actually magic steps rn.change_col(np.ones((vertices_num, 3))) #mask = rn.r.copy() # takes lots of time mask = rn.r * 255 import cv2 if batch == 1: cv2.imwrite('%s/%s.png' % (path, name), mask) else: cv2.imwrite('%s/%s_%d.png' % (path, name, i), mask) '''
def get_3DSV(mesh): from opendr.camera import ProjectPoints from opendr.renderer import DepthRenderer WIDTH, HEIGHT = 250, 250 rt = R.from_euler('xyz', [np.pi, 0, 0]).as_rotvec() rt_mat = R.from_euler('xyz', [np.pi, 0, 0]).as_matrix() camera = ProjectPoints(v=mesh.vertices, f=np.array([WIDTH, WIDTH]), c=np.array([WIDTH, HEIGHT]) / 2., t=np.array([0, 0, 3.0]), rt=rt, k=np.zeros(5)) frustum = {'near': 1., 'far': 10., 'width': WIDTH, 'height': HEIGHT} rn = DepthRenderer(camera=camera, frustum=frustum, f=mesh.faces, overdraw=False) # import cv2 depth_image = rn.depth_image.copy() mask = depth_image < depth_image.max() - 0.01 depth_image[~mask] = 0 depth_image[mask] = 255 - (depth_image[mask] - depth_image[mask].min()) / (depth_image[mask].max() - depth_image[mask].min()) * 255 points3d = camera.unproject_depth_image(rn.r) mask = points3d[:, :, 2] > np.min(points3d[:, :, 2]) + 0.01 points3d = points3d[mask] return points3d, depth_image
def initialize_camera(fx, fy, cx, cy): """ @param: fx,fy,cx,cy as pinhole camera model parameter @return: a differentiable camera instance """ rt = ch.zeros(3) t = ch.zeros(3) cam = ProjectPoints(f=np.array([fx, fy]), rt=rt, t=t, k=np.zeros(5), c=[cx, cy]) return cam
def render_bound(mesh, require_id = False, img_size=(448, 448), f=1000): rn = DepthRenderer() rn.camera = ProjectPoints(rt = np.zeros(3), t = np.zeros(3), f = np.array([f, f]), c = np.array([img_size[1], img_size[0]])/2., k = np.zeros(5) ) rn.frustum = {'near': .5, 'far': 10., 'width': img_size[1], 'height': img_size[0]} rn.v = mesh.points() rn.f = mesh.face_vertex_indices() rn.bgcolor = np.zeros(3) if require_id is False: return rn.boundarybool_image else: return rn.boundaryid_image
def __init__(self, m): self.m = m self.m.betas[:] = np.random.rand(m.betas.size) * .3 # m.pose[:] = np.random.rand(m.pose.size) * .2 self.m.pose[:3] = [0., 0., 0.] self.m.pose[3:] = np.zeros(45) # m.pose[3:] = [-0.42671473, -0.85829819, -0.50662164, +1.97374622, -0.84298473, -1.29958491] self.m.pose[0] = np.pi # compute inverse components to map from fullpose spec to coefficients hands_components = np.asarray(m.hands_components) self.hands_components_inv = np.linalg.inv(hands_components) # rendering components # Assign attributes to renderer w, h = (640, 480) # Create OpenDR renderer self.rn = ColoredRenderer() self.rn.camera = ProjectPoints(v=m, rt=np.zeros(3), t=np.array([-0.03, -0.04, 0.20]), f=np.array([w, w]) / 2., c=np.array([w, h]) / 2., k=np.zeros(5)) self.rn.frustum = {'near': 0.01, 'far': 2., 'width': w, 'height': h} self.rn.set(v=m, f=m.f, bgcolor=np.zeros(3)) # Construct point light source self.rn.vc = LambertianPointLight(f=m.f, v=self.rn.v, num_verts=len(m), light_pos=np.array( [-1000, -1000, -2000]), vc=np.ones_like(m) * .9, light_color=np.array([1., 1., 1.])) self.rn.vc += LambertianPointLight(f=m.f, v=self.rn.v, num_verts=len(m), light_pos=np.array( [+2000, +2000, +2000]), vc=np.ones_like(m) * .9, light_color=np.array([1., 1., 1.])) self.mvs = MeshViewers(window_width=2000, window_height=800, shape=[1, 3])
def render_depth_v(verts, faces, require_visi = False, img_size=(448, 448), f=1000): rn = DepthRenderer() rn.camera = ProjectPoints(rt = np.zeros(3), t = np.zeros(3), f = np.array([f, f]), c = np.array([img_size[1], img_size[0]])/2., k = np.zeros(5) ) rn.frustum = {'near': .5, 'far': 10., 'width': img_size[1], 'height': img_size[0]} rn.v = verts rn.f = faces rn.bgcolor = np.zeros(3) if require_visi is True: return rn.r, rn.visibility_image else: return rn.r
def _create_renderer(w=640, h=480, rt=np.zeros(3), t=np.zeros(3), f=None, c=None, k=None, near=.5, far=10.): f = np.array([w, w]) / 2. if f is None else f c = np.array([w, h]) / 2. if c is None else c k = np.zeros(5) if k is None else k rn = DepthRenderer() rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k) rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w} return rn
def initialize_camera(model, j2d, img, init_pose, flength=5000., pix_thsh=25., viz=False): """Initialize camera translation and body orientation :param model: SMPL model :param j2d: 14x2 array of CNN joints :param img: h x w x 3 image :param init_pose: 72D vector of pose parameters used for initialization :param flength: camera focal length (kept fixed) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the estimated camera, a boolean deciding if both the optimized body orientation and its flip should be considered, 3D vector for the body orientation """ # optimize camera translation and body orientation based on torso joints # LSP torso ids: # 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder torso_cids = [2, 3, 8, 9] # corresponding SMPL torso ids torso_smpl_ids = [2, 1, 17, 16] center = np.array([img.shape[1] / 2, img.shape[0] / 2]) # initialize camera rotation rt = ch.zeros(3) # initialize camera translation _LOGGER.info('initializing translation via similar triangles') init_t = guess_init(model, flength, j2d, init_pose) t = ch.array(init_t) # check how close the shoulder joints are try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh opt_pose = ch.array(init_pose) (_, A_global) = global_rigid_transformation( opt_pose, model.J, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) # initialize the camera cam = ProjectPoints( f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center) # we are going to project the SMPL joints cam.v = Jtr if viz: viz_img = img.copy() # draw the target (CNN) joints for coord in np.around(j2d).astype(int): if (coord[0] < img.shape[1] and coord[0] >= 0 and coord[1] < img.shape[0] and coord[1] >= 0): cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0]) import matplotlib.pyplot as plt plt.ion() # draw optimized joints at each iteration def on_step(_): """Draw a visualization.""" plt.figure(1, figsize=(5, 5)) plt.subplot(1, 1, 1) viz_img = img.copy() for coord in np.around(cam.r[torso_smpl_ids]).astype(int): if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and coord[1] < viz_img.shape[0] and coord[1] >= 0): cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255]) plt.imshow(viz_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-3) else: on_step = None # optimize for camera translation and body orientation free_variables = [cam.t, opt_pose[:3]] ch.minimize( # data term defined over torso joints... {'cam': j2d[torso_cids] - cam[torso_smpl_ids], # ...plus a regularizer for the camera translation 'cam_t': 1e2 * (cam.t[2] - init_t[2])}, x0=free_variables, method='dogleg', callback=on_step, options={'maxiter': 100, 'e_3': .0001, # disp set to 1 enables verbose output from the optimizer 'disp': 0}) if viz: plt.ioff() return (cam, try_both_orient, opt_pose[:3].r)
# SMPL parameters + camera print('opening %s' % results_path) with open(results_path, 'r') as f: res = pickle.load(f) poses = res['poses'] betas = res['betas'] # Camera rotation is always at identity, # The rotation of the body is encoded by the first 3 bits of poses. cam_ts = res['cam_ts'] focal_length = res['focal_length'] principal_pt = res['principal_pt'] # Setup camera: cam = ProjectPoints( f=focal_length, rt=np.zeros(3), k=np.zeros(5), c=principal_pt) h = 480 w = 640 # Corresponding ids of detected and SMPL joints (except head) lsp_ids = range(0, 12) smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] plt.ion() for i, (joints, pose, beta, cam_t) in enumerate(zip(est.T, poses, betas, cam_ts)): joints = joints[lsp_ids, :]