def chumpy_get_H(p1, p2): """ Compute differentiable homography from p1 to p2. Parameters ---------- p1,p2 : array_like, shape (4,2) Containing points to match. """ xmin = 0 ymin = 0 xmax = 1024 ymax = 576 p1 = 2 * p1 / ch.array([[xmax, ymax]]) p1 = p1 - 1.0 p2 = 2 * p2 / ch.array([[xmax, ymax]]) p2 = p2 - 1.0 N = p1.shape[0] A1 = ch.vstack((ch.zeros((3, N)), -p1.T, -ch.ones( (1, N)), p2[:, 1] * p1[:, 0], p2[:, 1] * p1[:, 1], p2[:, 1])).T A2 = ch.vstack((p1.T, ch.ones((1, N)), ch.zeros( (3, N)), -p2[:, 0] * p1[:, 0], -p2[:, 0] * p1[:, 1], -p2[:, 0])).T A = ch.vstack((A1, A2)) U, S, V = ch.linalg.svd(A.T.dot(A)) H_new = V[-1, :].reshape((3, 3)) # Re-normalize ML = ch.array([[xmax / 2.0, 0.0, xmax / 2.0], [0, ymax / 2.0, ymax / 2.0], [0, 0, 1.0]]) MR = ch.array([[2.0 / xmax, 0.0, -1.0], [0, 2.0 / ymax, -1.0], [0, 0, 1.0]]) H_new = ML.dot(H_new).dot(MR) return H_new / H_new[2, 2]
def chumpy_get_H(p1, p2): """ Compute differentiable homography from p1 to p2. """ N = p1.shape[0] A1 = ch.vstack((ch.zeros((3, N)), -p1.T, -ch.ones( (1, N)), p2[:, 1] * p1[:, 0], p2[:, 1] * p1[:, 1], p2[:, 1])).T A2 = ch.vstack((p1.T, ch.ones((1, N)), ch.zeros( (3, N)), -p2[:, 0] * p1[:, 0], -p2[:, 0] * p1[:, 1], -p2[:, 0])).T A = ch.vstack((A1, A2)) U, S, V = ch.linalg.svd(A.T.dot(A)) H_new = V[-1, :].reshape((3, 3)) return H_new
def _set_up(self): self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template self.v_shaped_personal = self.v_shaped + self.v_personal if sp.issparse(self.J_regressor): self.J = sp_dot(self.J_regressor, self.v_shaped) else: self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) * self.v_shaped.reshape(-1, 3, 1), axis=0).T self.v_posevariation = self.posedirs.dot( posemap(self.bs_type)(self.pose)) self.v_poseshaped = self.v_shaped_personal + self.v_posevariation self.A, A_global = self._global_rigid_transformation() self.Jtr = ch.vstack([g[:3, 3] for g in A_global]) self.J_transformed = self.Jtr + self.trans.reshape((1, 3)) self.V = self.A.dot(self.weights.T) rest_shape_h = ch.hstack( (self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1)))) self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3] self.v = self.v_posed + self.trans
def get_renderer(): import chumpy as ch from opendr.everything import * # Load mesh m = load_mesh('/Users/matt/geist/OpenDR/test_dr/nasa_earth.obj') m.v += ch.array([0, 0, 3.]) w, h = (320, 240) trans = ch.array([[0, 0, 0]]) # Construct renderer rn = TexturedRenderer() rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w, w]) / 2., c=ch.array([w, h]) / 2., k=ch.zeros(5)) rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h} rn.set(v=trans + m.v, f=m.f, texture_image=m.texture_image[:, :, ::-1], ft=m.ft, vt=m.vt, bgcolor=ch.zeros(3)) rn.vc = SphericalHarmonics(vn=VertNormals(v=rn.v, f=rn.f), components=ch.array([4., 0., 0., 0.]), light_color=ch.ones(3)) return rn
def run(self, beta=None, theta=None, garment_d=None, garment_class=None): """Outputs body and garment of specified garment class given theta, beta and displacements.""" if beta is not None: self.smpl_base.betas[:beta.shape[0]] = beta else: self.smpl_base.betas[:] = 0 if theta is not None: self.smpl_base.pose[:] = theta else: self.smpl_base.pose[:] = 0 self.smpl_base.v_personal[:] = 0 if garment_d is not None and garment_class is not None: if 'skirt' not in garment_class: vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] self.smpl_base.v_personal[vert_indices] = garment_d garment_m = Mesh(v=self.smpl_base.r[vert_indices], f=f) else: # vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] A = self.smpl_base.A.reshape((16, 24)).T skirt_V = self.skirt_skinning.dot(A).reshape((-1, 4, 4)) verts = self.skirt_weight.dot(self.smpl_base.v_poseshaped) verts = verts + garment_d verts_h = ch.hstack((verts, ch.ones((verts.shape[0], 1)))) verts = ch.sum(skirt_V * verts_h.reshape(-1, 1, 4), axis=-1)[:, :3] garment_m = Mesh(v=verts, f=f) else: garment_m = None self.smpl_base.v_personal[:] = 0 body_m = Mesh(v=self.smpl_base.r, f=self.smpl_base.f) return body_m, garment_m
def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False): options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]} options['disp'] = 1.0 options['delta_0'] = 0.1 options['e_3'] = 1e-4 s = ch.ones(1) r = ch.zeros(3) R = Rodrigues(r) t = ch.zeros(3) trafo_mesh = s*(R.dot(mesh.v.T)).T + t sampler = sample_from_mesh(scan, sample_type='vertices') s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False) if visualize: #Visualization code mv = MeshViewer() mv.set_static_meshes([scan]) tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) def on_show(_): tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) else: def on_show(_): pass ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options) return s,Rodrigues(r),t
def _set_up(self): self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template body_height = (self.v_shaped[2802, 1] + self.v_shaped[6262, 1]) - ( self.v_shaped[2237, 1] + self.v_shaped[6728, 1]) # print('111111111111111111111111111111111111111') # print(np.array(body_height)[0]) # print(type(body_height)) # print('22222222222222222222222222222222222222222') self.scale = 1.66 / np.array(body_height)[0] self.v_shaped_personal = self.scale * self.v_shaped + self.v_personal if sp.issparse(self.J_regressor): self.J = self.scale * sp_dot(self.J_regressor, self.v_shaped) else: self.J = self.scale * ch.sum(self.J_regressor.T.reshape(-1, 1, 24) * self.v_shaped.reshape(-1, 3, 1), axis=0).T self.v_posevariation = self.posedirs.dot( posemap(self.bs_type)(self.pose)) self.v_poseshaped = self.v_shaped_personal + self.v_posevariation self.A, A_global = self._global_rigid_transformation() self.Jtr = ch.vstack([g[:3, 3] for g in A_global]) self.J_transformed = self.Jtr + self.trans.reshape((1, 3)) self.V = self.A.dot(self.weights.T) rest_shape_h = ch.hstack( (self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1)))) self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3] self.v = self.v_posed + self.trans
def render_color_model_with_lighting(w, h, v, vn, vc, f, u, sh_comps=None, light_c=ch.ones(3), vlight_pos=None, vlight_color=None, bg_img=None): """renders colored model with lighting effect""" assert(sh_comps is not None or vlight_pos is not None) V = ch.array(v) A = np.zeros_like(v) # SH lighting if sh_comps is not None: A += vc * SphericalHarmonics(vn=vn, components=sh_comps, light_color=light_c) # single point lighting (grey light) if vlight_color is not None and vlight_pos is not None \ and len(vlight_pos.shape) == 1: A += LambertianPointLight(f=f, v=v, num_verts=len(v), light_pos=vlight_pos, light_color=vlight_color, vc=vc) # multiple point lighting (grey light) if vlight_color is not None and vlight_pos is not None \ and len(vlight_pos.shape) == 2: for vlp in vlight_pos: A += LambertianPointLight(f=f, v=v, num_verts=len(v), light_pos=vlp, light_color=vlight_color, vc=vc) black_img = np.array(np.zeros((w, h, 3)), dtype=np.float32) bg_img_ = bg_img if bg_img is not None else black_img rn = ColoredRenderer(camera=u, v=V, f=f, vc=A, background_image=bg_img_, frustum={'width': w, 'height': h, 'near': 0.1, 'far': 20}) return rn.r
def render_training_pairs( mesh, bone_pose, img_w, img_h, camera_r, camera_t, #color_bg, sh_comps=None, light_c=ch.ones(3), vlight_pos=None, vlight_color=None): """generates training image pairs Will generate color image, mask, semantic map, normal map """ v_ = mesh['v'] # render color image # To avoid aliasing, I render the image with 2x resolution and then resize it # See: https://stackoverflow.com/questions/22069167/opencv-how-to-smoothen-boundary u = _project_vertices(v_, img_w * 2, img_h * 2, camera_r, camera_t) # pose in camera coordinate, pose on image pose_to_camera, pose_on_image = _project_pose(bone_pose, img_w, img_h, camera_r, camera_t) # color_bg = cv.resize(color_bg, (img_w * 2, img_h * 2)) img = _render_color_model_with_lighting( img_w * 2, img_h * 2, v_, mesh['vn'], mesh['vc'], mesh['f'], u, sh_comps=sh_comps, light_c=light_c, vlight_pos=vlight_pos, vlight_color=vlight_color, ) # bg_img=color_bg) img = cv.resize(img, (img_w, img_h)) img = np.float32(np.copy(img)) # # render silhouette # u = _project_vertices(v_, img_w, img_h, camera_r, camera_t) # msk = _render_mask(img_w, img_h, v_, mesh['f'], u) # msk = np.float32(np.copy(msk)) # # render normal maps # if 'n' in mesh: # n_ = mesh['n'] * 0.5 + 0.5 # else: # vn = util.calc_normal(mesh) # n_ = vn * 0.5 + 0.5 # nml = _render_color_model_without_lighting(img_w, img_h, v_, n_, mesh['f'], # u, bg_img=None) # nml = np.float32(np.copy(nml)) return img, pose_to_camera, pose_on_image
def test_earth(): m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3)) # Create V, A, U, f: geometry, brightness, camera, renderer V = ch.array(m.v) A = SphericalHarmonics(vn=VertNormals(v=V, f=m.f), components=[3., 2., 0., 0., 0., 0., 0., 0., 0.], light_color=ch.ones(3)) # camera U = ProjectPoints(v=V, f=[w, w], c=[w / 2., h / 2.], k=ch.zeros(5), t=ch.zeros(3), rt=ch.zeros(3)) f = TexturedRenderer(vc=A, camera=U, f=m.f, bgcolor=[0., 0., 0.], texture_image=m.texture_image, vt=m.vt, ft=m.ft, frustum={ 'width': w, 'height': h, 'near': 1, 'far': 20 }) # Parameterize the vertices translation, rotation = ch.array([0, 0, 8]), ch.zeros(3) f.v = translation + V.dot(Rodrigues(rotation)) observed = f.r np.random.seed(1) # this is reactive # in the sense that changes to values will affect function which depend on them. translation[:] = translation.r + np.random.rand(3) rotation[:] = rotation.r + np.random.rand(3) * .2 # Create the energy E_raw = f - observed E_pyr = gaussian_pyramid(E_raw, n_levels=6, normalization='size') Image.fromarray((observed * 255).astype(np.uint8)).save( os.path.join(save_dir, "reference.png")) step = 0 Image.fromarray((f.r * 255).astype(np.uint8)).save( os.path.join(save_dir, "step_{:05d}.png".format(step))) print('OPTIMIZING TRANSLATION, ROTATION, AND LIGHT PARMS') free_variables = [translation, rotation] ch.minimize({'pyr': E_pyr}, x0=free_variables, callback=create_callback(f)) ch.minimize({'raw': E_raw}, x0=free_variables, callback=create_callback(f))
def compute_approx_scale(lmk_3d, model, lmk_face_idx, lmk_b_coords, opt_options=None): """ function: compute approximate scale to align scan and model input: lmk_3d: input landmark 3d, in shape (N,3) model: FLAME face model lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates opt_options: optimizaton options output: model.r: fitted result vertices model.f: fitted result triangulations (fixed in this code) parms: fitted model parameters """ scale = ch.ones(1) scan_lmks = scale * ch.array(lmk_3d) model_lmks = mesh_points_by_barycentric_coordinates( model, model.f, lmk_face_idx, lmk_b_coords) lmk_err = scan_lmks - model_lmks # options if opt_options is None: print("fit_lmk3d(): no 'opt_options' provided, use default settings.") import scipy.sparse as sp opt_options = {} opt_options['disp'] = 1 opt_options['delta_0'] = 0.1 opt_options['e_3'] = 1e-4 opt_options['maxiter'] = 2000 sparse_solver = lambda A, x: sp.linalg.cg( A, x, maxiter=opt_options['maxiter'])[0] opt_options['sparse_solver'] = sparse_solver # on_step callback def on_step(_): pass ch.minimize(fun=lmk_err, x0=[scale, model.trans, model.pose[:3]], method='dogleg', callback=on_step, options=opt_options) return scale.r
def test_sum_mean_std_var(self): for fn in [ch.sum, ch.mean, ch.var, ch.std]: # Create fake input and differences in input space data1 = ch.ones((3, 4, 7, 2)) data2 = ch.array(data1.r + .1 * np.random.rand(data1.size).reshape(data1.shape)) diff = data2.r - data1.r # Compute outputs result1 = fn(data1, axis=2) result2 = fn(data2, axis=2) # Empirical and predicted derivatives gt = result2.r - result1.r pred = result1.dr_wrt(data1).dot(diff.ravel()).reshape(gt.shape) #print(np.max(np.abs(gt - pred))) if fn in [ch.std, ch.var]: self.assertTrue(1e-2 > np.max(np.abs(gt - pred))) else: self.assertTrue(1e-14 > np.max(np.abs(gt - pred))) # test caching dr0 = result1.dr_wrt(data1) data1[:] = np.random.randn(data1.size).reshape(data1.shape) self.assertTrue( result1.dr_wrt(data1) is dr0) # changing values shouldn't force recompute result1.axis = 1 self.assertTrue(result1.dr_wrt(data1) is not dr0) self.assertEqual( ch.mean(ch.eye(3), axis=1).ndim, np.mean(np.eye(3), axis=1).ndim) self.assertEqual( ch.mean(ch.eye(3), axis=0).ndim, np.mean(np.eye(3), axis=0).ndim) self.assertEqual( ch.sum(ch.eye(3), axis=1).ndim, np.sum(np.eye(3), axis=1).ndim) self.assertEqual( ch.sum(ch.eye(3), axis=0).ndim, np.sum(np.eye(3), axis=0).ndim)
def run(self, beta=None, theta=None, garment_d=None, garment_class=None): """Outputs body and garment of specified garment class given theta, beta and displacements.""" if beta is not None: self.smpl_base.betas[:beta.shape[0]] = beta else: self.smpl_base.betas[:] = 0 if theta is not None: self.smpl_base.pose[:] = theta else: self.smpl_base.pose[:] = 0 self.smpl_base.v_personal[:] = 0 if garment_d is not None and garment_class is not None: if 'skirt' not in garment_class: vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] self.smpl_base.v_personal[vert_indices] = garment_d garment_m = Mesh(v=self.smpl_base.r[vert_indices], f=f) else: vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] verts = self.smpl_base.v_poseshaped[vert_indices] + garment_d verts_h = ch.hstack((verts, ch.ones((verts.shape[0], 1)))) verts = ch.sum(self.smpl_base.V.T[vert_indices] * verts_h.reshape(-1, 4, 1), axis=1)[:, :3] # if theta is not None: # rotmat = self.smpl_base.A.r[:, :, 0] # verts_homo = np.hstack( # (verts, np.ones((verts.shape[0], 1)))) # verts = verts_homo.dot(rotmat.T)[:, :3] garment_m = Mesh(v=verts, f=f) else: garment_m = None self.smpl_base.v_personal[:] = 0 body_m = Mesh(v=self.smpl_base.r, f=self.smpl_base.f) return body_m, garment_m
def _simple_renderer(rn, meshes, yrot=0, texture=None): mesh = meshes[0] if texture is not None: if not hasattr(mesh, 'ft'): """mesh.ft = copy(mesh.f) vt = copy(mesh.v[:, :2]) vt -= np.min(vt, axis=0).reshape((1, -1)) vt /= np.max(vt, axis=0).reshape((1, -1)) mesh.vt = vt""" mesh.vt, mesh.ft = Template_tex() mesh.texture_filepath = rn.texture_image # Set camers parameters if texture is not None: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, ft=mesh.ft, vt=mesh.vt, bgcolor=np.ones(3)) else: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, bgcolor=np.ones(3)) for next_mesh in meshes[1:]: _stack_with(rn, next_mesh, texture) # Construct Back Light (on back right corner) if texture is not None: rn.vc = ch.ones(rn.v.shape) else: albedo = rn.vc # Construct Back Light (on back right corner) rn.vc = LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=rotateY( np.array([-200, -100, -100]), yrot), vc=albedo, light_color=np.array([1, 1, 1])) # Construct Left Light rn.vc += LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=rotateY( np.array([800, 10, 300]), yrot), vc=albedo, light_color=np.array([1, 1, 1])) # Construct Right Light rn.vc += LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=rotateY( np.array([-500, 500, 1000]), yrot), vc=albedo, light_color=np.array([.7, .7, .7])) return rn.r
def setupTexturedRenderer(renderer, vstack, vch, f_list, vc_list, vnch, uv, haveTextures_list, textures_list, camera, frustum, sharedWin=None): f = [] f_listflat = [item for sublist in f_list for item in sublist] lenMeshes = 0 for mesh_i, mesh in enumerate(f_listflat): polygonLen = 0 for polygons in mesh: f = f + [polygons + lenMeshes] polygonLen += len(polygons) lenMeshes += len(vch[mesh_i]) fstack = np.vstack(f) if len(vnch) == 1: vnstack = vnch[0] else: vnstack = ch.vstack(vnch) if len(vc_list) == 1: vcstack = vc_list[0] else: vcstack = ch.vstack(vc_list) uvflat = [item for sublist in uv for item in sublist] ftstack = np.vstack(uvflat) texturesch = [] textures_listflat = [item for sublist in textures_list for item in sublist] # import ipdb; ipdb.set_trace() for texture_list in textures_listflat: if texture_list != None: for texture in texture_list: if isinstance(texture, np.ndarray): texturesch = texturesch + [ch.array(texture)] elif texture != None: texturesch = texturesch + [ch.array(texture)] if len(texturesch) == 0: texture_stack = ch.Ch([]) elif len(texturesch) == 1: texture_stack = texturesch[0].ravel() else: texture_stack = ch.concatenate([tex.ravel() for tex in texturesch]) haveTextures_listflat = [ item for sublist in haveTextures_list for item in sublist ] renderer.set(camera=camera, frustum=frustum, v=vstack, f=fstack, vn=vnstack, vc=vcstack, ft=ftstack, texture_stack=texture_stack, v_list=vch, f_list=f_listflat, vc_list=vc_list, ft_list=uvflat, textures_list=textures_listflat, haveUVs_list=haveTextures_listflat, bgcolor=ch.ones(3), overdraw=True) renderer.msaa = True renderer.sharedWin = sharedWin
chLightEl = ch.Ch([np.pi/2]) chLightDist = ch.Ch([0.5]) chLightDistGT = ch.Ch([0.5]) chLightAzGT = ch.Ch([0.0]) chLightElGT = ch.Ch([np.pi/4]) ligthTransf = computeHemisphereTransformation(chLightAz, chLightEl, chLightDist, targetPosition) ligthTransfGT = computeHemisphereTransformation(chLightAzGT, chLightElGT, chLightDistGT, targetPosition) lightPos = ch.dot(ligthTransf, ch.Ch([0.,0.,0.,1.]))[0:3] lightPos = ch.Ch([targetPosition[0]+0.5,targetPosition[1],targetPosition[2] + 0.5]) lightPosGT = ch.dot(ligthTransfGT, ch.Ch([0.,0.,0.,1.]))[0:3] chGlobalConstant = ch.Ch([0.5]) chGlobalConstantGT = ch.Ch([0.5]) light_color = ch.ones(3)*chPointLightIntensity light_colorGT = ch.ones(3)*chPointLightIntensityGT chVColors = ch.Ch([0.8,0.8,0.8]) chVColorsGT = ch.Ch([0.8,0.8,0.8]) shCoefficientsFile = 'data/sceneSH' + str(sceneIdx) + '.pickle' chAmbientIntensityGT = ch.Ch([0.025]) clampedCosCoeffs = clampedCosineCoefficients() chAmbientSHGT = ch.zeros([9]) envMapDic = {} SHFilename = 'data/LightSHCoefficients.pickle' with open(SHFilename, 'rb') as pfile: envMapDic = pickle.load(pfile)
def convert_BFM_mesh_to_FLAME(FLAME_model_fname, BFM_mesh_fname, FLAME_out_fname): ''' Convert Basel Face Model mesh to a FLAME mesh \param FLAME_model_fname path of the FLAME model \param BFM_mesh_fname path of the BFM mesh to be converted \param FLAME_out_fname path of the output file ''' # Regularizer weights for jaw pose (i.e. opening of mouth), shape, and facial expression. # Increase regularization in case of implausible output meshes. w_pose = 1e-4 w_shape = 1e-3 w_exp = 1e-4 if not os.path.exists(os.path.dirname(FLAME_out_fname)): os.makedirs(os.path.dirname(FLAME_out_fname)) if not os.path.exists(BFM_mesh_fname): print('BFM mesh not found %s' % BFM_mesh_fname) return BFM_mesh = Mesh(filename=BFM_mesh_fname) if not os.path.exists(FLAME_model_fname): print('FLAME model not found %s' % FLAME_model_fname) return model = load_model(FLAME_model_fname) if not os.path.exists('./data/BFM_to_FLAME_corr.npz'): print('Cached mapping not found') return cached_data = np.load('./data/BFM_to_FLAME_corr.npz', allow_pickle=True, encoding="latin1") BFM2017_corr = cached_data['BFM2017_corr'].item() BFM2009_corr = cached_data['BFM2009_corr'].item() BFM2009_cropped_corr = cached_data['BFM2009_cropped_corr'].item() if (2 * BFM_mesh.v.shape[0] == BFM2017_corr['mtx'].shape[1]) and ( BFM_mesh.f.shape[0] == BFM2017_corr['f_in'].shape[0]): conv_mesh = convert_mesh(BFM_mesh, BFM2017_corr) elif (2 * BFM_mesh.v.shape[0] == BFM2009_corr['mtx'].shape[1]) and ( BFM_mesh.f.shape[0] == BFM2009_corr['f_in'].shape[0]): conv_mesh = convert_mesh(BFM_mesh, BFM2009_corr) elif (2 * BFM_mesh.v.shape[0] == BFM2009_cropped_corr['mtx'].shape[1] ) and (BFM_mesh.f.shape[0] == BFM2009_cropped_corr['f_in'].shape[0]): conv_mesh = convert_mesh(BFM_mesh, BFM2009_cropped_corr) else: print('Conversion failed - input mesh does not match any setup') return FLAME_mask_ids = cached_data['FLAME_mask_ids'] scale = ch.ones(1) v_target = scale * ch.array(conv_mesh.v) dist = v_target[FLAME_mask_ids] - model[FLAME_mask_ids] pose_reg = model.pose[3:] shape_reg = model.betas[:300] exp_reg = model.betas[300:] obj = { 'dist': dist, 'pose_reg': w_pose * pose_reg, 'shape_reg': w_shape * shape_reg, 'exp_reg': w_exp * exp_reg } ch.minimize(obj, x0=[scale, model.trans, model.pose[:3]]) ch.minimize(obj, x0=[ scale, model.trans, model.pose[np.hstack( (np.arange(3), np.arange(6, 9)))], model.betas ]) v_out = model.r / scale.r Mesh(v_out, model.f).write_obj(FLAME_out_fname)
def get_renderer(): import chumpy as ch from opendr.everything import * # Load mesh m = load_mesh('/Users/matt/geist/OpenDR/test_dr/nasa_earth.obj') m.v += ch.array([0,0,3.]) w, h = (320, 240) trans = ch.array([[0,0,0]]) # Construct renderer rn = TexturedRenderer() rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5)) rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h} rn.set(v=trans+m.v, f=m.f, texture_image=m.texture_image[:,:,::-1], ft=m.ft, vt=m.vt, bgcolor=ch.zeros(3)) rn.vc = SphericalHarmonics(vn=VertNormals(v=rn.v, f=rn.f), components=ch.array([4.,0.,0.,0.]), light_color=ch.ones(3)) return rn
def render_training_pairs(mesh, smpl, img_w, img_h, camera_r, camera_t, color_bg, sh_comps=None, light_c=ch.ones(3), vlight_pos=None, vlight_color=None): """generates training image pairs Will generate color image, mask, semantic map, normal map """ v_, v_smpl_ = mesh['v'], smpl['v'] # render color image # To avoid aliasing, I render the image with 2x resolution and then resize it # See: https://stackoverflow.com/questions/22069167/opencv-how-to-smoothen-boundary u = _project_vertices(v_, img_w * 2, img_h * 2, camera_r, camera_t) color_bg = cv.resize(color_bg, (img_w * 2, img_h * 2)) img = _render_color_model_with_lighting(img_w * 2, img_h * 2, v_, mesh['vn'], mesh['vc'], mesh['f'], u, sh_comps=sh_comps, light_c=light_c, vlight_pos=vlight_pos, vlight_color=vlight_color, bg_img=color_bg) img = cv.resize(img, (img_w, img_h)) img = np.float32(np.copy(img)) # render silhouette u = _project_vertices(v_, img_w, img_h, camera_r, camera_t) msk = _render_mask(img_w, img_h, v_, mesh['f'], u) msk = np.float32(np.copy(msk)) # render normal maps if 'n' in mesh: n_ = mesh['n'] * 0.5 + 0.5 else: vn = util.calc_normal(mesh) n_ = vn * 0.5 + 0.5 nml = _render_color_model_without_lighting(img_w, img_h, v_, n_, mesh['f'], u, bg_img=None) nml = np.float32(np.copy(nml)) # render semantic map u = _project_vertices(v_smpl_, img_w, img_h, camera_r, camera_t) vc_smpl = util.get_smpl_semantic_code() smap = _render_color_model_without_lighting(img_w, img_h, v_smpl_, vc_smpl, smpl['f'], u, bg_img=None) smap = np.float32(np.copy(smap)) return img, msk, nml, smap
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None): from opendr.geometry import Rodrigues if length_regs is not None: n_shape_dofs = length_regs.shape[0] - 1 else: n_shape_dofs = model.betas.r.size segm = np.argmax(model.weights_prior, axis=1) J_off = ch.zeros((len(joint2name), 3)) rots = rots0.copy() mujoco_t_mid = [0, 3, 6, 9] if wrt_betas is not None: # if we want to differentiate wrt betas (shape), we must have the # regressors... assert (length_regs is not None and rad_regs is not None) # ... and betas must be a chumpy object assert (hasattr(wrt_betas, 'dterms')) pad = ch.concatenate( (wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1))) lengths = pad.dot(length_regs) rads = pad.dot(rad_regs) else: lengths = ch.ones(len(joint2name)) rads = ch.ones(len(joint2name)) betas = wrt_betas if wrt_betas is not None else model.betas n_betas = len(betas) # the joint regressors are the original, pre-optimized ones # (middle of the part frontier) myJ_regressor = model.J_regressor_prior myJ0 = ch.vstack((ch.ch.MatVecMult( myJ_regressor, model.v_template[:, 0] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]), ch.ch.MatVecMult( myJ_regressor, model.v_template[:, 1] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]), ch.ch.MatVecMult( myJ_regressor, model.v_template[:, 2] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T # with small adjustments for hips, spine and feet myJ = ch.vstack([ ch.concatenate([ myJ0[0, 0], (.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2] ]), ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate( [myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]), ch.concatenate( [myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]), ch.concatenate( [myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]), ch.vstack([myJ0[i] for i in range(10, 24)]) ]) capsules = [] # create one capsule per mujoco joint for ijoint, segms in enumerate(mujoco2segm): if wrt_betas is None: vidxs = np.asarray([segm == k for k in segms]).any(axis=0) verts = model.v_template[vidxs].r dims = (verts.max(axis=0) - verts.min(axis=0)) rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[ (np.argmax(dims) + 2) % 3]) / 4.) lengths[ijoint] = max(dims) - 2. * rads[ijoint].r # the core joints are different, since the capsule is not in the joint # but in the middle if ijoint in mujoco_t_mid: len_offset = ch.vstack( [ch.zeros(1), ch.abs(lengths[ijoint]) / 2., ch.zeros(1)]).reshape(3, 1) caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1) - Rodrigues(rots[ijoint]).dot(len_offset), rots[ijoint], rads[ijoint], lengths[ijoint]) else: caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1), rots[ijoint], rads[ijoint], lengths[ijoint]) caps.id = ijoint capsules.append(caps) return capsules
def _simple_renderer(rn, meshes, yrot=0, texture=None, out_mesh_name=None, out_texture_name=None): mesh = meshes[0] if texture is not None: if not hasattr(mesh, 'ft'): """mesh.ft = copy(mesh.f) vt = copy(mesh.v[:, :2]) vt -= np.min(vt, axis=0).reshape((1, -1)) vt /= np.max(vt, axis=0).reshape((1, -1)) mesh.vt = vt""" mesh.vt, mesh.ft = Template_tex() mesh.texture_filepath = rn.texture_image # Set camers parameters if texture is not None: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, ft=mesh.ft, vt=mesh.vt, bgcolor=np.ones(3)) else: rn.set(v=mesh.v, f=mesh.f, vc=mesh.vc, bgcolor=np.ones(3)) for next_mesh in meshes[1:]: _stack_with(rn, next_mesh, texture) if out_mesh_name is not None: write_obj(mesh, out_mesh_name) if (texture is not None) and (out_texture_name is not None): cv2.imwrite(out_texture_name, texture) with open(out_mesh_name[:-3] + "mtl", 'w') as my_file: my_file.write("newmtl Material.001" + "\n" + "Ns 96.078431" + "\n" + "Ka 1.000000 1.000000 1.000000" + "\n" + "Kd 0.640000 0.640000 0.640000" + "\n" + "Ks 0.000000 0.000000 0.000000" + "\n" + "Ke 0.412673 0.432000 0.226290" + "\n" + "Ni 1.000000" + "\n" + "d 1.000000" + "\n" + "illum 1" + "\n" + "map_Kd " + out_texture_name) # Construct Back Light (on back right corner) if texture is not None: rn.vc = ch.ones(rn.v.shape) else: albedo = rn.vc # Construct Back Light (on back right corner) rn.vc = LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=rotateY( np.array([-200, -100, -100]), yrot), vc=albedo, light_color=np.array([1, 1, 1])) # Construct Left Light rn.vc += LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=rotateY( np.array([800, 10, 300]), yrot), vc=albedo, light_color=np.array([1, 1, 1])) # Construct Right Light rn.vc += LambertianPointLight(f=rn.f, v=rn.v, num_verts=len(rn.v), light_pos=rotateY( np.array([-500, 500, 1000]), yrot), vc=albedo, light_color=np.array([.7, .7, .7])) return rn.r
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None): from opendr.geometry import Rodrigues if length_regs is not None: n_shape_dofs = length_regs.shape[0] - 1 else: n_shape_dofs = model.betas.r.size segm = np.argmax(model.weights_prior, axis=1) J_off = ch.zeros((len(joint2name), 3)) rots = rots0.copy() mujoco_t_mid = [0, 3, 6, 9] if wrt_betas is not None: # if we want to differentiate wrt betas (shape), we must have the # regressors... assert (length_regs is not None and rad_regs is not None) # ... and betas must be a chumpy object assert (hasattr(wrt_betas, 'dterms')) pad = ch.concatenate( (wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1))) lengths = pad.dot(length_regs) rads = pad.dot(rad_regs) else: lengths = ch.ones(len(joint2name)) rads = ch.ones(len(joint2name)) betas = wrt_betas if wrt_betas is not None else model.betas n_betas = len(betas) # the joint regressors are the original, pre-optimized ones # (middle of the part frontier) myJ_regressor = model.J_regressor_prior myJ0 = ch.vstack( (ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 0] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]), ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 1] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]), ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 2] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T # with small adjustments for hips, spine and feet myJ = ch.vstack( [ch.concatenate([myJ0[0, 0], ( .6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2]]), ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate( [myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]), ch.concatenate( [myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]), ch.concatenate( [myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]), ch.vstack([myJ0[i] for i in range(10, 24)])]) capsules = [] # create one capsule per mujoco joint for ijoint, segms in enumerate(mujoco2segm): if wrt_betas is None: vidxs = np.asarray([segm == k for k in segms]).any(axis=0) verts = model.v_template[vidxs].r dims = (verts.max(axis=0) - verts.min(axis=0)) rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[( np.argmax(dims) + 2) % 3]) / 4.) lengths[ijoint] = max(dims) - 2. * rads[ijoint].r # the core joints are different, since the capsule is not in the joint # but in the middle if ijoint in mujoco_t_mid: len_offset = ch.vstack([ch.zeros(1), ch.abs(lengths[ijoint]) / 2., ch.zeros(1)]).reshape(3, 1) caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape( 3, 1) - Rodrigues(rots[ijoint]).dot(len_offset), rots[ijoint], rads[ijoint], lengths[ijoint]) else: caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1), rots[ijoint], rads[ijoint], lengths[ijoint]) caps.id = ijoint capsules.append(caps) return capsules
def __call__(self, args): np.random.seed() di, data_item = args print("Processing data #", di) render_img_w = conf.render_img_w render_img_h = conf.render_img_h dataset_dir = conf.dataset_dir num_render_per_obj = self.render_setting["num_render_per_obj"] output_dir = self.render_setting["output_dir"] novel_view = self.render_setting["novel_view"] # preprocess 3D models mesh = load_models(dataset_dir, data_item) # align to smpl bone bone_pose, root_rot, root_trans = load_smpl_params(dataset_dir, data_item) mesh = util.translate_model_inplace(mesh, root_rot, root_trans) mesh, bone_pose = axis_transformation(mesh, bone_pose, conf.axis_transformation) # ---------- preprocess is done # save bone params np.save('%s/bone_params/bone_params_%08d.npy' % (output_dir, di), bone_pose) img_indices = np.arange(num_render_per_obj * di, num_render_per_obj * (di + 1)) if not self.random_lighting: # use one lighting condition vl_pos, vl_clr, sh = self.vl_pos, self.vl_clr, self.sh for vi in range(num_render_per_obj): mesh_ = copy.deepcopy(mesh) bone_pose_ = copy.deepcopy(bone_pose) mesh_, bone_pose_ = move_to_origin(mesh_, bone_pose_) mesh_, bone_pose_, param_0 = transform_model_randomly(mesh_, bone_pose_) img_ind = img_indices[vi] if self.random_lighting: # random lighting condition vl_pos, vl_clr = util.sample_verticle_lighting(3) sh = util.sample_sh_component() cam_t, cam_r = self.sample_view(novel_view) img, pose_to_camera, pose_on_image = rd.render_training_pairs(mesh_, bone_pose_, render_img_w, render_img_h, cam_r, cam_t, # bg, sh_comps=sh, light_c=ch.ones(3), vlight_pos=vl_pos, vlight_color=vl_clr) save_rendered_data(img, pose_on_image, output_dir, img_ind, pose_to_camera, bone_pose_) if hasattr(conf, "autoencoder") and conf.autoencoder: dir, base = os.path.split(output_dir) output_dir_view2 = os.path.join(dir + "_view2", base) cam_t, cam_r = self.sample_view(novel_view=False) img, pose_to_camera, pose_on_image = rd.render_training_pairs(mesh_, bone_pose_, render_img_w, render_img_h, cam_r, cam_t, # bg, sh_comps=sh, light_c=ch.ones(3), vlight_pos=vl_pos, vlight_color=vl_clr) save_rendered_data(img, pose_on_image, output_dir_view2, img_ind, pose_to_camera, bone_pose_)