def unproject_points(self, uvd, camera_space=False): cam = ProjectPoints3D( **{k: getattr(self, k) for k in self.dterms if hasattr(self, k)}) try: xy_undistorted_camspace = cv2.undistortPoints( np.asarray(uvd[:, :2].reshape((1, -1, 2)).copy()), np.asarray(cam.camera_mtx), cam.k.r) xyz_camera_space = np.hstack( (xy_undistorted_camspace.squeeze(), col(uvd[:, 2]))) xyz_camera_space[:, :2] *= col( xyz_camera_space[:, 2]) # scale x,y by z if camera_space: return xyz_camera_space other_answer = xyz_camera_space - row(cam.view_mtx[:, 3]) # translate result = other_answer.dot(cam.view_mtx[:, :3]) # rotate except: # slow way, probably not so good. But doesn't require cv2.undistortPoints. cam.v = np.ones_like(uvd) ch.minimize(cam - uvd, x0=[cam.v], method='dogleg', options={'disp': 0}) result = cam.v.r return result
def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False): options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]} options['disp'] = 1.0 options['delta_0'] = 0.1 options['e_3'] = 1e-4 s = ch.ones(1) r = ch.zeros(3) R = Rodrigues(r) t = ch.zeros(3) trafo_mesh = s*(R.dot(mesh.v.T)).T + t sampler = sample_from_mesh(scan, sample_type='vertices') s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False) if visualize: #Visualization code mv = MeshViewer() mv.set_static_meshes([scan]) tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) def on_show(_): tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) else: def on_show(_): pass ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options) return s,Rodrigues(r),t
def fit_pose(frame, last_smpl, frustum, nohands, viz_rn): if nohands: faces = faces_no_hands(frame.smpl.f) else: faces = frame.smpl.f dst_type = cv2.cv.CV_DIST_L2 if cv2.__version__[0] == '2' else cv2.DIST_L2 dist_i = cv2.distanceTransform(np.uint8(frame.mask * 255), dst_type, 5) - 1 dist_i[dist_i < 0] = 0 dist_i[dist_i > 50] = 50 dist_o = cv2.distanceTransform(255 - np.uint8(frame.mask * 255), dst_type, 5) dist_o[dist_o > 50] = 50 rn_m = ColoredRenderer(camera=frame.camera, v=frame.smpl, f=faces, vc=np.ones_like(frame.smpl), frustum=frustum, bgcolor=0, num_channels=1) E = { 'mask': gaussian_pyramid(rn_m * dist_o * 100. + (1 - rn_m) * dist_i, n_levels=4, normalization='size') * 80., '2dpose': GMOf(frame.pose_obj, 100), 'prior': frame.pose_prior_obj * 4., 'sp': frame.collision_obj * 1e3, } if last_smpl is not None: E['last_pose'] = GMOf(frame.smpl.pose - last_smpl.pose, 0.05) * 50. E['last_trans'] = GMOf(frame.smpl.trans - last_smpl.trans, 0.05) * 50. if nohands: x0 = [ frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)], frame.smpl.trans ] else: x0 = [ frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)], frame.smpl.trans ] ch.minimize(E, x0, method='dogleg', options={ 'e_3': .01, }, callback=get_cb(viz_rn, frame))
def compute_FLAME_params(source_path, params_out_fname, flame_model_fname, template_fname): ''' Load a template and an existing animation sequence in "zero pose" and compute the FLAME shape, jaw pose, and expression paramters. Outputs one set of shape paramters for the entire sequence, and pose and expression parameters for each frame. :param source_path: path of the animation sequence (files must be provided in OBJ file format) :param params_out_fname output path of the FLAME paramters file :param flame_model_fname: path of the FLAME model :param template_fname "zero pose" template used to generate the sequence ''' if not os.path.exists(os.path.dirname(params_out_fname)): os.makedirs(os.path.dirname(params_out_fname)) # Load sequence files sequence_fnames = sorted(glob.glob(os.path.join(source_path, '*.obj'))) num_frames = len(sequence_fnames) if num_frames == 0: print('No sequence meshes found') return model = load_model(flame_model_fname) print('Optimize for template identity parameters') template_mesh = Mesh(filename=template_fname) ch.minimize(template_mesh.v - model, x0=[model.betas[:300]], options={ 'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0] }) betas = model.betas.r[:300].copy() model.betas[:] = 0. model.v_template[:] = template_mesh.v model_pose = np.zeros((num_frames, model.pose.shape[0])) model_exp = np.zeros((num_frames, 100)) for frame_idx in range(num_frames): print('Process frame %d/%d' % (frame_idx + 1, num_frames)) model.betas[:] = 0. model.pose[:] = 0. frame_vertices = Mesh(filename=sequence_fnames[frame_idx]).v # Optimize for jaw pose and facial expression ch.minimize(frame_vertices - model, x0=[model.pose[6:9], model.betas[300:]], options={ 'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0] }) model_pose[frame_idx] = model.pose.r.copy() model_exp[frame_idx] = model.betas.r[300:].copy() np.save(params_out_fname, { 'shape': betas, 'pose': model_pose, 'expression': model_exp })
def fit_joints(both_joints, n_pose_params=15, shape_sigma=10.0, save_filename=None): """ Fits the MANO model to hand joint 3D locations both_jonts: tuple of length 2, 21 joints per hand, e.g. output of ContactPose.hand_joints() n_pose_params: number of pose parameters (excluding 3 global rotation params) shape_sigma: reciprocal of shape regularization strength save_filename: file where the fitting output will be saved in JSON format """ mano_params = [] for hand_idx, joints in enumerate(both_joints): if joints is None: # hand is not present mano_params.append(mano_param_dict(n_pose_params)) # dummy continue cp_joints = openpose2mano(joints) # MANO model m = mutils.load_mano_model(MANOFitter._mano_dicts[hand_idx], ncomps=n_pose_params, flat_hand_mean=False) m.betas[:] = np.zeros(m.betas.size) m.pose[:] = np.zeros(m.pose.size) mano_joints = mano_joints_with_fingertips(m) mano_joints_np = np.array([[float(mm) for mm in m] for m in mano_joints]) # align palm cp_palm = get_palm_joints(np.asarray(cp_joints)) mano_palm = get_palm_joints(np.asarray(mano_joints_np)) mTc = register_pcs(cp_palm, mano_palm) cp_joints = np.dot(mTc, np.vstack((cp_joints.T, np.ones(len(cp_joints))))) cp_joints = cp_joints[:3].T cp_joints = ch.array(cp_joints) # set up objective objective = [m-c for m,c in zip(mano_joints, cp_joints)] mean_betas = ch.array(np.zeros(m.betas.size)) objective.append((m.betas - mean_betas) / shape_sigma) # optimize ch.minimize(objective, x0=(m.pose, m.betas, m.trans), method='dogleg') p = mano_param_dict(n_pose_params) p['pose'] = np.array(m.pose).tolist() p['betas'] = np.array(m.betas).tolist() p['valid'] = True p['mTc']['translation'] = (mTc[:3, 3] - np.array(m.trans)).tolist() p['mTc']['rotation'] = txq.mat2quat(mTc[:3, :3]).tolist() mano_params.append(p) # # to access hand mesh vertices and faces # vertices = np.array(m.r) # vertices = mutils.tform_points(np.linalg.inv(mTc), vertices) # faces = np.array(m.f) if save_filename is not None: with open(save_filename, 'w') as f: json.dump(mano_params, f, indent=4, separators=(',', ':')) print('{:s} written'.format(save_filename)) return mano_params
def unpose_garment(smpl, v_free, vert_indices=None): smpl.v_personal[:] = 0 c = smpl[vert_indices] E = {'v_personal_high': c - v_free} ch.minimize(E, x0=[smpl.v_personal], options={'e_3': .00001}) smpl.pose[:] = 0 smpl.trans[:] = 0 return Mesh(smpl.r, smpl.f).keep_vertices(vert_indices), np.copy( np.array(smpl.v_personal))
def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display): if nohands: faces = faces_no_hands(base_smpl.f) else: faces = base_smpl.f vis_rn_b = BoundaryRenderer(camera=camera, frustum=frustum, f=faces, num_channels=1) vis_rn_m = ColoredRenderer(camera=camera, frustum=frustum, f=faces, vc=np.zeros_like(base_smpl), bgcolor=1, num_channels=1) model_template = Smpl(model_data) model_template.betas[:] = base_smpl.betas.r g_laplace = regularize_laplace() g_model = regularize_model() g_symmetry = regularize_symmetry() face_ids = get_face_vertex_ids() for step, (w_laplace, w_model, w_symmetry, sigma) in enumerate(zip( np.linspace(6.5, 4.0, icp_count) if naked else np.linspace(4.0, 2.0, icp_count), np.linspace(0.9, 0.6, icp_count) if naked else np.linspace(0.6, 0.3, icp_count), np.linspace(3.6, 1.8, icp_count), np.linspace(0.06, 0.003, icp_count), )): log.info('# Step {}'.format(step)) L = laplacian(model_template.r, base_smpl.f) delta = L.dot(model_template.r) w_laplace *= g_laplace.reshape(-1, 1) w_model *= g_model.reshape(-1, 1) w_symmetry *= g_symmetry.reshape(-1, 1) E = { 'laplace': (sp_dot(L, base_smpl.v_shaped_personal) - delta) * w_laplace, 'model': (base_smpl.v_shaped_personal - model_template) * w_model, 'symmetry': (base_smpl.v_personal + np.array([1, -1, -1]) * base_smpl.v_personal[model_data['vert_sym_idxs']]) * w_symmetry, } log.info('## Matching rays with contours') for current, f in enumerate(tqdm(frames)): E['silh_{}'.format(current)] = ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m) #paper 2 E['face_{}'.format(current)] = ray_face(f, sigma, base_smpl, camera, face_ids) log.info('## Run optimization') ch.minimize( E, [base_smpl.v_personal, model_template.betas], method='dogleg', options={'maxiter': 15, 'e_3': 0.001}, callback=get_cb(frames[0], base_smpl, camera, frustum) if display else None )
def _fit_rot_trans( model, # pylint: disable=too-many-arguments, too-many-locals j2d, center, init_t, init_pose, conf, flength): """Find a rotation and translation to minimize the projection error of the pose.""" opt_pose = _ch.array(init_pose) # pylint: disable=no-member opt_trans = _ch.array(init_t) # pylint: disable=no-member (_, A_global) = _global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=_ch) Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + opt_trans # pylint: disable=no-member cam = _ProjectPoints(f=_np.array([flength, flength]), rt=_np.zeros(3), t=_np.zeros(3), k=_np.zeros(5), c=center) cam_3d = _ProjectPoints3D(f=_np.array([flength, flength]), rt=_np.zeros(3), t=_np.zeros(3), k=_np.zeros(5), c=center) cam.v = Jtr cam_3d.v = Jtr free_variables = [opt_pose[:3], opt_trans] # Optimize global rotation and translation. j2d_ids = range(12) smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] #_CID_TORSO = [2, 3, 8, 9] #torso_smpl_ids = [2, 1, 17, 16] _ch.minimize( { 'j2d': (j2d.T[j2d_ids] - cam[smpl_ids]).T * conf[j2d_ids], # 'dev^2': 1e2 * (opt_trans[2] - init_t[2]) }, x0=free_variables, method='dogleg', callback=None, #on_step_vis, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) _LOGGER.debug("Global rotation: %s, global translation: %s.", str(opt_pose[:3].r), str(opt_trans.r)) _LOGGER.debug("Points 3D: %s.", str(cam_3d.r[:10, :])) _LOGGER.debug("Points 2D: %s.", str(cam.r[:10, :])) return opt_pose[:3].r, opt_trans.r, cam_3d.r[:, 2].min(), cam_3d.r[:, 2].max()
def test_earth(): m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3)) # Create V, A, U, f: geometry, brightness, camera, renderer V = ch.array(m.v) A = SphericalHarmonics(vn=VertNormals(v=V, f=m.f), components=[3., 2., 0., 0., 0., 0., 0., 0., 0.], light_color=ch.ones(3)) # camera U = ProjectPoints(v=V, f=[w, w], c=[w / 2., h / 2.], k=ch.zeros(5), t=ch.zeros(3), rt=ch.zeros(3)) f = TexturedRenderer(vc=A, camera=U, f=m.f, bgcolor=[0., 0., 0.], texture_image=m.texture_image, vt=m.vt, ft=m.ft, frustum={ 'width': w, 'height': h, 'near': 1, 'far': 20 }) # Parameterize the vertices translation, rotation = ch.array([0, 0, 8]), ch.zeros(3) f.v = translation + V.dot(Rodrigues(rotation)) observed = f.r np.random.seed(1) # this is reactive # in the sense that changes to values will affect function which depend on them. translation[:] = translation.r + np.random.rand(3) rotation[:] = rotation.r + np.random.rand(3) * .2 # Create the energy E_raw = f - observed E_pyr = gaussian_pyramid(E_raw, n_levels=6, normalization='size') Image.fromarray((observed * 255).astype(np.uint8)).save( os.path.join(save_dir, "reference.png")) step = 0 Image.fromarray((f.r * 255).astype(np.uint8)).save( os.path.join(save_dir, "step_{:05d}.png".format(step))) print('OPTIMIZING TRANSLATION, ROTATION, AND LIGHT PARMS') free_variables = [translation, rotation] ch.minimize({'pyr': E_pyr}, x0=free_variables, callback=create_callback(f)) ch.minimize({'raw': E_raw}, x0=free_variables, callback=create_callback(f))
def reinit_frame(frame, null_pose, nohands, viz_rn): mylog("# reinit_frame") if (np.sum(frame.pose_obj.r ** 2) > 625 or np.sum(frame.pose_prior_obj.r ** 2) > 75)\ and np.sum(frame.keypoints[[0, 2, 5, 8, 11], 2]) > 3.: log.info('Tracking error too large. Re-init frame...') x0 = [frame.smpl.pose[:3], frame.smpl.trans] frame.smpl.pose[3:] = null_pose if frame.keypoints[2, 0] > frame.keypoints[5, 0]: frame.smpl.pose[0] = 0 frame.smpl.pose[2] = np.pi E = { 'init_pose': frame.pose_obj[[0, 2, 5, 8, 11]], } ch.minimize(E, x0, method='dogleg', options={ 'e_3': .1, }, callback=get_cb(viz_rn, frame)) E = { 'pose': GMOf(frame.pose_obj, 100), 'prior': frame.pose_prior_obj * 8., } x0 = [frame.smpl.trans] if nohands: # x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 60)]) x0.append(frame.smpl.pose[list(range(21)) + list(range(27, 30)) + list(range(36, 60))]) else: # x0.append(frame.smpl.pose[range(21) + range(27, 30) + range(36, 72)]) x0.append(frame.smpl.pose[list(range(21)) + list(range(27, 30)) + list(range(36, 72))]) ch.minimize(E, x0, method='dogleg', options={ 'e_3': .01, }, callback=get_cb(viz_rn, frame))
def initialize_camera(model, j2d, img, init_pose, flength=flength): """Initialize camera translation and body orientation :param model: SMPL model :param j2d: 14x2 array of CNN joints :param img: h x w x 3 image :param init_pose: 72D vector of pose parameters used for initialization :param flength: camera focal length (kept fixed) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the estimated camera, a boolean deciding if both the optimized body orientation and its flip should be considered, 3D vector for the body orientation """ # optimize camera translation and body orientation based on torso joints # LSP torso ids: # 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder torso_cids = [2, 3, 8, 9] # corresponding SMPL torso ids torso_smpl_ids = [2, 1, 17, 16] center = np.array([img.shape[1] / 2, img.shape[0] / 2]) rt = ch.zeros(3) # initial camera rotation init_t = guess_init(model, flength, j2d, init_pose) t = ch.array(init_t) # initial camera translation opt_pose = ch.array(init_pose) _, A_global = global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) # initialize the camera and project SMPL joints cam = ProjectPoints(f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center) cam.v = Jtr # project SMPL joints # optimize for camera translation and body orientation free_variables = [cam.t, opt_pose[:3]] ch.minimize( # data term defined over torso joints... {'cam': j2d[torso_cids] - cam[torso_smpl_ids], # ...plus a regularizer for the camera translation 'cam_t': 1e2 * (cam.t[2] - init_t[2])}, x0=free_variables, method='dogleg', callback=None, options={'maxiter': 100, 'e_3': .0001, 'disp': 0}) return cam, opt_pose[:3].r
def compute_approx_scale(lmk_3d, model, lmk_face_idx, lmk_b_coords, opt_options=None): """ function: compute approximate scale to align scan and model input: lmk_3d: input landmark 3d, in shape (N,3) model: FLAME face model lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates opt_options: optimizaton options output: model.r: fitted result vertices model.f: fitted result triangulations (fixed in this code) parms: fitted model parameters """ scale = ch.ones(1) scan_lmks = scale * ch.array(lmk_3d) model_lmks = mesh_points_by_barycentric_coordinates( model, model.f, lmk_face_idx, lmk_b_coords) lmk_err = scan_lmks - model_lmks # options if opt_options is None: print("fit_lmk3d(): no 'opt_options' provided, use default settings.") import scipy.sparse as sp opt_options = {} opt_options['disp'] = 1 opt_options['delta_0'] = 0.1 opt_options['e_3'] = 1e-4 opt_options['maxiter'] = 2000 sparse_solver = lambda A, x: sp.linalg.cg( A, x, maxiter=opt_options['maxiter'])[0] opt_options['sparse_solver'] = sparse_solver # on_step callback def on_step(_): pass ch.minimize(fun=lmk_err, x0=[scale, model.trans, model.pose[:3]], method='dogleg', callback=on_step, options=opt_options) return scale.r
def unproject_points(self, uvd, camera_space=False): cam = ProjectPoints3D(**{k: getattr(self, k) for k in self.dterms if hasattr(self, k)}) try: xy_undistorted_camspace = cv2.undistortPoints(np.asarray(uvd[:,:2].reshape((1,-1,2)).copy()), np.asarray(cam.camera_mtx), cam.k.r) xyz_camera_space = np.hstack((xy_undistorted_camspace.squeeze(), col(uvd[:,2]))) xyz_camera_space[:,:2] *= col(xyz_camera_space[:,2]) # scale x,y by z if camera_space: return xyz_camera_space other_answer = xyz_camera_space - row(cam.view_mtx[:,3]) # translate result = other_answer.dot(cam.view_mtx[:,:3]) # rotate except: # slow way, probably not so good. But doesn't require cv2.undistortPoints. cam.v = np.ones_like(uvd) ch.minimize(cam - uvd, x0=[cam.v], method='dogleg', options={'disp': 0}) result = cam.v.r return result
def lift2Dto3D(projPtsGT, camMat, filename, img, JVis=np.ones((21, ), dtype=np.float32), trans=None, beta=None, wrist3D=None, withPoseCoeff=True, weights=1.0, relDepGT=None, rel3DCoordGT=None, rel3DCoordNormGT=None, img2DGT=None, outDir=None, poseCoeffInit=None, transInit=None, betaInit=None): loss = {} if withPoseCoeff: numComp = 30 m, poseCoeffCh, betaCh, transCh, fullposeCh = getHandModelPoseCoeffs( numComp) if poseCoeffInit is not None: poseCoeffCh[:] = poseCoeffInit if transInit is not None: transCh[:] = transInit if betaInit is not None: betaCh[:] = betaInit freeVars = [poseCoeffCh] if beta is None: freeVars = freeVars + [betaCh] loss['shape'] = 1e2 * betaCh else: betaCh[:] = beta if trans is None: freeVars = freeVars + [transCh] else: transCh[:] = trans # loss['pose'] = 0.5e2 * poseCoeffCh[3:]/stdPCACoeff[:numComp] thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints( fullposeCh[3:]) loss['constMin'] = 5e2 * thetaConstMin loss['constMax'] = 5e2 * thetaConstMax loss['invalidTheta'] = 1e3 * fullposeCh[Constraints().invalidThetaIDs] else: m, rotCh, jointsCh, transCh, betaCh = getHandModel() thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints( jointsCh) loss['constMin'] = 5e3 * thetaConstMin loss['constMax'] = 5e3 * thetaConstMax validTheta = jointsCh[Constraints().validThetaIDs[3:] - 3] freeVars = [validTheta, rotCh] if beta is None: freeVars = freeVars + [betaCh] loss['shape'] = 0.5e2 * betaCh else: betaCh[:] = beta if trans is None: freeVars = freeVars + [transCh] else: transCh[:] = trans if relDepGT is not None: relDepPred = m.J_transformed[:, 2] - m.J_transformed[0, 2] loss['relDep'] = (relDepPred - relDepGT) * weights[:, 0] * 5e1 if rel3DCoordGT is not None: rel3DCoordPred = m.J_transformed - m.J_transformed[0:1, :] loss['rel3DCoord'] = (rel3DCoordPred - rel3DCoordGT) * np.tile( weights[:, 0:1], [1, 3]) * 5e1 if rel3DCoordNormGT is not None: rel3DCoordPred = m.J_transformed[jointsMap][ 1:, :] - m.J_transformed[jointsMap][0:1, :] rel3DCoordPred = rel3DCoordPred / ch.expand_dims( ch.sqrt(ch.sum(ch.square(rel3DCoordPred), axis=1)), axis=1) loss['rel3DCoordNorm'] = ( 1. - ch.sum(rel3DCoordPred * rel3DCoordNormGT, axis=1)) * 1e4 # loss['rel3DCoordNorm'] = \ # (rel3DCoordNormGT*ch.expand_dims(ch.sum(rel3DCoordPred*rel3DCoordNormGT, axis=1), axis=1) - rel3DCoordPred) * 1e2#5e2 projPts = utilsEval.chProjectPoints(m.J_transformed, camMat, False)[jointsMap] JVis = np.tile(np.expand_dims(JVis, 1), [1, 2]) loss['joints2D'] = (projPts - projPtsGT) * JVis * weights * 1e0 loss['joints2DClip'] = clipIden(projPts - projPtsGT) * JVis * weights * 1e1 if wrist3D is not None: dep = wrist3D[2] if dep < 0: dep = -dep loss['wristDep'] = (m.J_transformed[0, 2] - dep) * 1e2 # vis_mesh(m) render = False def cbPass(_): pass # print(loss['joints'].r) print(filename) warnings.simplefilter('ignore') loss['joints2D'] = loss[ 'joints2D'] * 1e1 / weights # dont want to use confidence now if True: ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cbPass if render else cbPass, method='dogleg', options={'maxiter': 50}) else: manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) freeVars = [poseCoeffCh[:3], transCh] ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cbPass, method='dogleg', options={'maxiter': 20}) manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) freeVars = [poseCoeffCh[3:]] ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cb if render else cbPass, method='dogleg', options={'maxiter': 20}) manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) freeVars = [poseCoeffCh, transCh] if beta is None: freeVars = freeVars + [betaCh] ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cb if render else cbPass, method='dogleg', options={'maxiter': 20}) if False: open3dVisualize(m) else: manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) # vis_mesh(m) joints3D = m.J_transformed.r[jointsMap] # print(betaCh.r) # print((relDepPred.r - relDepGT)) return joints3D, poseCoeffCh.r.copy(), betaCh.r.copy(), transCh.r.copy( ), loss['joints2D'].r.copy(), m.r.copy()
def lift2Dto3DMultiFrame(projPtsGT, camMat, filename, JVis=np.ones((21, ), dtype=np.float32), trans=None, beta=None, wrist3D=None, withPoseCoeff=True, weights=None, relDepGT=None, rel3DCoordGT=None, isOpenGLCoord=False, transInit=None, rotInit=None, globalPoseCoeffInit=None, betaInit=None): ''' :param projPtsGT: :param camMat: :param filename: :param JVis: :param trans: :param beta: :param wrist3D: :param withPoseCoeff: :param weights: 21x1 array :param relDepGT: :param rel3DCoordGT: always in opencv :param isOpenGLCoord: :return: always in opencv ''' loss = {} numFrames = projPtsGT.shape[0] if weights is None: weights = [np.ones((21, 1), dtype=np.float32)] * numFrames numComp = 30 mList, chRotList, chGlobalPoseCoeff, chTransList, chGlobalBeta = getHandModelPoseCoeffsMultiFrame( numComp, numFrames, isOpenGLCoord) freeVars = [chGlobalPoseCoeff] if beta is None: freeVars = freeVars + [chGlobalBeta] loss['shape'] = 1e2 * chGlobalBeta else: chGlobalBeta[:] = beta if betaInit is not None: chGlobalBeta[:] = betaInit if trans is None: freeVars = freeVars + chTransList else: for i, t in enumerate(chTransList): t[:] = trans[i] if transInit is not None: for i in range(len(chTransList)): chTransList[i][:] = transInit[i] if rotInit is not None: for i in range(len(chRotList)): chRotList[i][:] = rotInit[i] if globalPoseCoeffInit is not None: chGlobalPoseCoeff[:] = globalPoseCoeffInit freeVars = freeVars + chRotList # loss['pose'] = 0.5e2 * poseCoeffCh[3:]/stdPCACoeff[:numComp] fullposeCh = mList[0].fullpose ## thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints( fullposeCh[3:]) loss['constMin'] = 5e2 * thetaConstMin loss['constMax'] = 5e2 * thetaConstMax loss['invalidTheta'] = 5e2 * fullposeCh[Constraints().invalidThetaIDs] if relDepGT is not None: for i in range(numFrames): relDepPred = mList[i].J_transformed[:, 2] - mList[i].J_transformed[0, 2] loss['relDep_%d' % (i)] = (relDepPred - relDepGT[i]) * weights[:, 0] * 5e1 coordChangeMat = np.array([[1., 0., 0.], [0, -1., 0.], [0., 0., -1.]], dtype=np.float32) if rel3DCoordGT is not None: for i in range(numFrames): rel3DCoordPred = mList[i].J_transformed - mList[i].J_transformed[ 0:1, :] if isOpenGLCoord: rel3DCoordPred = coordChangeMat.dot(coordChangeMat) loss['rel3DCoord_%d' % (i)] = (rel3DCoordPred - rel3DCoordGT[i]) * np.tile( weights[i][:, 0:1], [1, 3]) * 5e1 for i in range(numFrames): projPts = utilsEval.chProjectPoints(mList[i].J_transformed, camMat, isOpenGLCoord)[jointsMap] # JVis = np.tile(np.expand_dims(JVis, 1), [1, 2]) loss['joints2D_%d' % (i)] = (projPts - projPtsGT[i] ) # * np.tile(weights[i][:,0:1], [1,2])# * 1e1 render = False def cbPass(_): pass # print(loss['joints'].r) for i in range(numFrames): loss['joints2D_%d' % (i)] = loss['joints2D_%d' % (i)] * np.tile(weights[i][:, 0:1], [1, 2]) * 1e1 ch.minimize(loss, x0=freeVars, callback=cbPass, method='dogleg', options={'maxiter': 12}) # vis_mesh(mList[0]) # vis_mesh(m) joints3DList = [] for i in range(numFrames): joints3D = mList[i].J_transformed.r[jointsMap] if isOpenGLCoord: joints3D = joints3D.dot(coordChangeMat) joints3DList.append(joints3D) # fullpose list fullposeList = [] betaList = [] transList = [] for m in mList: fullposeList.append(m.fullpose.r) betaList.append(m.betas.r) transList.append(m.trans.r) # print(betaCh.r) # print((relDepPred.r - relDepGT)) return np.stack( joints3DList, axis=0 ), fullposeList, betaList, transList, chGlobalPoseCoeff.r.copy(), mList
def inverseKinematicCh(pts3D, fullposeInit = np.zeros((48,), dtype=np.float32), transInit = np.array([0., 0., -0.45]), betaInit = np.zeros((10,), dtype=np.float32)): ''' Performs inverse kinematic optimization when given the 3d points 3D points --> MANO params :param pts3D: :param fullposeInit: :param transInit: :param betaInit: :return: fullpose, trans and beta vector ''' import chumpy as ch from mano.chumpy.smpl_handpca_wrapper_HAND_only import load_model, load_model_withInputs from ghope.constraints import Constraints assert pts3D.shape == (21,3) constraints = Constraints() validTheta = ch.zeros((len(constraints.validThetaIDs,))) fullposeCh = constraints.fullThetaMat.dot(ch.expand_dims(validTheta,1))[:,0] transCh = ch.array(undo_chumpy(transInit)) betaCh = ch.array(undo_chumpy(betaInit)) pts3DGT = pts3D m = load_model_withInputs(MANO_MODEL_PATH, fullposeCh, transCh, betaCh, ncomps=6, flat_hand_mean=True) if fullposeInit.shape == (16,3,3): fpList = [] for i in range(16): fpList.append(cv2.Rodrigues(fullposeInit[i])[0][:, 0]) validTheta[:] = np.concatenate(fpList, 0)[constraints.validThetaIDs] else: validTheta[:] = undo_chumpy(fullposeInit[constraints.validThetaIDs]) loss = {} projPts = m.J_transformed projPtsGT = pts3DGT loss['joints'] = (projPts - projPtsGT) # m.fullpose[Constraints().invalidThetaIDs] = 0. thetaConstMin, thetaConstMax = Constraints().getHandJointConstraintsCh(fullposeCh[3:]) # loss['constMin'] = 1e4 * thetaConstMin # loss['constMax'] = 1e4 * thetaConstMax freeVars = [validTheta, transCh] def cbPass(_): print(np.max(np.abs(m.J_transformed - pts3DGT))) pass ch.minimize(loss, x0=freeVars, callback=cbPass, method='dogleg', options={'maxiter': 10}) if True: import matplotlib.pyplot as plt from mpl_toolkits.mplot3d import Axes3D fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.scatter(m.J_transformed[:, 0], m.J_transformed[:, 1], m.J_transformed[:, 2], c='r') ax.scatter(pts3DGT[:, 0], pts3DGT[:, 1], pts3DGT[:, 2], c='b') plt.show() return undo_chumpy(m.fullpose), undo_chumpy(m.trans), undo_chumpy(m.betas)
def fit_scan( scan, # input scan lmk_3d, # input scan landmarks model, # model lmk_face_idx, lmk_b_coords, # landmark embedding weights, # weights for the objectives gmo_sigma, # weight of the robustifier shape_num=300, expr_num=100, opt_options=None): """ function: fit FLAME model to a 3D scan input: scan: input scan lmk_3d: input landmark 3d, in shape (N,3) model: FLAME face model lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates weights: weights for each objective shape_num, expr_num: numbers of shape and expression compoenents used opt_options: optimizaton options output: model.r: fitted result vertices model.f: fitted result triangulations (fixed in this code) parms: fitted model parameters """ # variables shape_idx = np.arange(0, min( 300, shape_num)) # valid shape component range in "betas": 0-299 expr_idx = np.arange(300, 300 + min( 100, expr_num)) # valid expression component range in "betas": 300-399 used_idx = np.union1d(shape_idx, expr_idx) model.betas[:] = np.random.rand( model.betas.size) * 0.0 # initialized to zero model.pose[:] = np.random.rand( model.pose.size) * 0.0 # initialized to zero free_variables = [model.trans, model.pose, model.betas[used_idx]] # weights print("fit_scan(): use the following weights:") for kk in weights.keys(): print("fit_scan(): weights['%s'] = %f" % (kk, weights[kk])) # objectives # landmark error lmk_err = landmark_error_3d(mesh_verts=model, mesh_faces=model.f, lmk_3d=lmk_3d, lmk_face_idx=lmk_face_idx, lmk_b_coords=lmk_b_coords) # scan-to-mesh distance, measuring the distance between scan vertices and the closest points in the model's surface sampler = sample_from_mesh(scan, sample_type='vertices') s2m = ScanToMesh(scan, model, model.f, scan_sampler=sampler, rho=lambda x: GMOf(x, sigma=gmo_sigma)) # regularizer shape_err = weights['shape'] * model.betas[shape_idx] expr_err = weights['expr'] * model.betas[expr_idx] pose_err = weights['pose'] * model.pose[3:] # exclude global rotation objectives = { 's2m': weights['s2m'] * s2m, 'lmk': weights['lmk'] * lmk_err, 'shape': shape_err, 'expr': expr_err, 'pose': pose_err } # options if opt_options is None: print("fit_lmk3d(): no 'opt_options' provided, use default settings.") import scipy.sparse as sp opt_options = {} opt_options['disp'] = 1 opt_options['delta_0'] = 0.1 opt_options['e_3'] = 1e-4 opt_options['maxiter'] = 2000 sparse_solver = lambda A, x: sp.linalg.cg( A, x, maxiter=opt_options['maxiter'])[0] opt_options['sparse_solver'] = sparse_solver # on_step callback def on_step(_): pass # optimize # step 1: rigid alignment from time import time timer_start = time() print("\nstep 1: start rigid fitting...") ch.minimize(fun=lmk_err, x0=[model.trans, model.pose[:3]], method='dogleg', callback=on_step, options=opt_options) timer_end = time() print("step 1: fitting done, in %f sec\n" % (timer_end - timer_start)) # step 2: non-rigid alignment timer_start = time() print("step 2: start non-rigid fitting...") ch.minimize(fun=objectives, x0=free_variables, method='dogleg', callback=on_step, options=opt_options) timer_end = time() print("step 2: fitting done, in %f sec\n" % (timer_end - timer_start)) # return results parms = { 'trans': model.trans.r, 'pose': model.pose.r, 'betas': model.betas.r } return model.r, model.f, parms
def estimate_global_pose(landmarks, key_vids, model, cam, img, fix_t=False, viz=False, SOLVE_FLATER=True): ''' Estimates the global rotation and translation. only diff in estimate_global_pose from single_frame_ferrari is that all animals have the same kp order. ''' # Redefining part names.. part_names = [ 'leftEye', 'rightEye', 'chin', 'frontLeftFoot', 'frontRightFoot', 'backLeftFoot', 'backRightFoot', 'tailStart', 'frontLeftKnee', 'frontRightKnee', 'backLeftKnee', 'backRightKnee', 'leftShoulder', 'rightShoulder', 'frontLeftAnkle', 'frontRightAnkle', 'backLeftAnkle', 'backRightAnkle', 'neck', 'TailTip' ] # Use shoulder to "knee"(elbow) distance. also tail to "knee" if available. use_names = [ 'neck', 'leftShoulder', 'rightShoulder', 'backLeftKnee', 'backRightKnee', 'tailStart', 'frontLeftKnee', 'frontRightKnee' ] use_ids = [part_names.index(name) for name in use_names] # These might not be visible visible = landmarks[:, 2].astype(bool) use_ids = [id for id in use_ids if visible[id]] if len(use_ids) < 3: print('Frontal?..') use_names += [ 'frontLeftAnkle', 'frontRightAnkle', 'backLeftAnkle', 'backRightAnkle' ] model.pose[1] = np.pi / 2 init_t = estimate_translation(landmarks, key_vids, cam.f[0].r, model) use_ids = [part_names.index(name) for name in use_names] use_ids = [id for id in use_ids if visible[id]] # Setup projection error: all_vids = np.hstack([key_vids[id] for id in use_ids]) cam.v = model[all_vids] keypoints = landmarks[use_ids, :2].astype(float) # Duplicate keypoints for the # of vertices for that kp. num_verts_per_kp = [len(key_vids[row_id]) for row_id in use_ids] j2d = np.vstack([ np.tile(kp, (num_rep, 1)) for kp, num_rep in zip(keypoints, num_verts_per_kp) ]) assert (cam.r.shape == j2d.shape) # SLOW but correct method!! # remember which ones belongs together,, group = np.hstack([ index * np.ones(len(key_vids[row_id])) for index, row_id in enumerate(use_ids) ]) assignments = np.vstack([group == i for i in np.arange(group[-1] + 1)]) num_points = len(use_ids) proj_error = (ch.vstack([ cam[choice] if np.sum(choice) == 1 else cam[choice].mean(axis=0) for choice in assignments ]) - keypoints) / np.sqrt(num_points) # Fast but not matching average: # Normalization weight j2d_norm_weights = np.sqrt( 1. / len(use_ids) * np.vstack([1. / num * np.ones((num, 1)) for num in num_verts_per_kp])) proj_error_fast = j2d_norm_weights * (cam - j2d) if fix_t: obj = {'cam': proj_error_fast} else: obj = { 'cam': proj_error_fast, 'cam_t': 1e1 * (model.trans[2] - init_t[2]) } # Only estimate body orientation if fix_t: free_variables = [model.pose[:3]] else: free_variables = [model.trans, model.pose[:3]] if not SOLVE_FLATER: obj['feq'] = 1e3 * (cam.f[0] - cam.f[1]) # So it's under control obj['freg'] = 1e1 * (cam.f[0] - 3000) / 1000. # here without this cam.f goes negative.. (asking margin of 500) obj['fpos'] = ch.maximum(0, 500 - cam.f[0]) # cam t also has to be positive! obj['cam_t_pos'] = ch.maximum(0, 0.01 - model.trans[2]) del obj['cam_t'] free_variables.append(cam.f) if viz: import matplotlib.pyplot as plt plt.ion() def on_step(_): plt.figure(1, figsize=(5, 5)) plt.cla() plt.imshow(img[:, :, ::-1]) img_here = render_mesh(Mesh(model.r, model.f), img.shape[1], img.shape[0], cam) plt.imshow(img_here) plt.scatter(j2d[:, 0], j2d[:, 1], c='w') plt.scatter(cam.r[:, 0], cam.r[:, 1]) plt.draw() plt.pause(1e-3) if 'feq' in obj: print('flength %.1f %.1f, z %.f' % (cam.f[0], cam.f[1], model.trans[2])) else: on_step = None from time import time t0 = time() init_angles = [[0, 0, 0]] #, [1.5,0,0], [1.5,-1.,0]] scores = np.zeros(len(init_angles)) for i, angle in enumerate(init_angles): # Init translation model.trans[:] = init_t model.pose[:3] = angle ch.minimize(obj, x0=free_variables, method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001 }) scores[i] = np.sum(obj['cam'].r**2.) j = np.argmin(scores) model.trans[:] = init_t model.pose[:3] = init_angles[j] ch.minimize(obj, x0=free_variables, method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001 }) print('Took %g' % (time() - t0)) #import pdb; pdb.set_trace() if viz: dist = np.mean(model.r, axis=0)[2] img_here = render_mesh(Mesh(model.r, model.f), img.shape[1], img.shape[0], cam) plt.imshow(img[:, :, ::-1]) plt.imshow(img_here) return model.pose[:3].r, model.trans.r
def initialize_camera(model, j2d, img, init_pose, flength=5000., pix_thsh=25., viz=False): """Initialize camera translation and body orientation :param model: SMPL model :param j2d: 14x2 array of CNN joints :param img: h x w x 3 image :param init_pose: 72D vector of pose parameters used for initialization :param flength: camera focal length (kept fixed) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the estimated camera, a boolean deciding if both the optimized body orientation and its flip should be considered, 3D vector for the body orientation """ # optimize camera translation and body orientation based on torso joints # LSP torso ids: # 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder torso_cids = [2, 3, 8, 9] # corresponding SMPL torso ids torso_smpl_ids = [2, 1, 17, 16] center = np.array([img.shape[1] / 2, img.shape[0] / 2]) # initialize camera rotation rt = ch.zeros(3) # initialize camera translation _LOGGER.info('initializing translation via similar triangles') init_t = guess_init(model, flength, j2d, init_pose) t = ch.array(init_t) # check how close the shoulder joints are try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh opt_pose = ch.array(init_pose) (_, A_global) = global_rigid_transformation( opt_pose, model.J, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) # initialize the camera cam = ProjectPoints( f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center) # we are going to project the SMPL joints cam.v = Jtr if viz: viz_img = img.copy() # draw the target (CNN) joints for coord in np.around(j2d).astype(int): if (coord[0] < img.shape[1] and coord[0] >= 0 and coord[1] < img.shape[0] and coord[1] >= 0): cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0]) import matplotlib.pyplot as plt plt.ion() # draw optimized joints at each iteration def on_step(_): """Draw a visualization.""" plt.figure(1, figsize=(5, 5)) plt.subplot(1, 1, 1) viz_img = img.copy() for coord in np.around(cam.r[torso_smpl_ids]).astype(int): if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and coord[1] < viz_img.shape[0] and coord[1] >= 0): cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255]) plt.imshow(viz_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-3) else: on_step = None # optimize for camera translation and body orientation free_variables = [cam.t, opt_pose[:3]] ch.minimize( # data term defined over torso joints... {'cam': j2d[torso_cids] - cam[torso_smpl_ids], # ...plus a regularizer for the camera translation 'cam_t': 1e2 * (cam.t[2] - init_t[2])}, x0=free_variables, method='dogleg', callback=on_step, options={'maxiter': 100, 'e_3': .0001, # disp set to 1 enables verbose output from the optimizer 'disp': 0}) if viz: plt.ioff() return (cam, try_both_orient, opt_pose[:3].r)
def fit_silhouettes_multi_model(objs, sv, silhs, cameras, w_s2m=10, w_m2s=20, max_iter=100, mv=None, fix_shape=False, kp_camera=None, imgs=None, alpha=None, fix_trans=False, pyr_scale=1.0, vc=None, symIdx=None, mv2=None, FIX_POSE=False): nCameras = len(cameras) # Projected sihlouette #dist = [np.mean(sv[i].r, axis=0)[2] for i in range(nCameras)] frustums = [{'near': ch.min(sv[i], axis=0)[2], 'far': ch.max(sv[i], axis=0)[2], 'width': silhs[i].shape[1], 'height': silhs[i].shape[0]} for i,camera in enumerate(cameras)] rends = [ColoredRenderer( vc=np.ones_like(sv[i].r), v=sv[i], f=sv[i].f, camera=cameras[i], frustum=frustums[i], bgcolor=ch.array([0, 0, 0])) for i in range(nCameras)] # silhouette error term (model-to-scan) #obj_m2s, obj_s2m, dist_tsf = setup_silhouette_obj(silhs, rends, sv[0].model.f) obj_m2s, obj_s2m, dist_tsf = setup_silhouette_obj(silhs, rends, sv[0].f) if mv is not None: import matplotlib.pyplot as plt plt.ion() def on_step(_): for i in range(len(rends)): rend = rends[i] img = imgs[i] edges = cv2.Canny(np.uint8(rend[:, :, 0].r * 255), 100, 200) coords = np.array(np.where(edges > 0)).T plt.figure(200+i, facecolor='w') plt.clf() plt.subplot(2, 2, 1) plt.imshow(dist_tsf[i]) plt.plot(coords[:, 1], coords[:, 0], 'w.') plt.axis('off') plt.subplot(2, 2, 2) plt.imshow(rend[:, :, 0].r) plt.title('fitted silhouette') plt.draw() plt.axis('off') if img is not None and kp_camera is not None: plt.subplot(2, 2, 3) plt.imshow(img[:, :, ::-1]) plt.scatter(kp_camera[i].r[:, 0], kp_camera[i].r[:, 1]) plt.axis('off') plt.subplot(2, 2, 4) plt.imshow((silhs[i]+rend[:, :, 0].r)/2.0) plt.axis('off') plt.draw() plt.show(block=False) plt.pause(1e-5) if vc is not None: vc1 = vc[i].r.copy() vc1[:,0] = vc[i].r.copy()[:,2] vc1[:,2] = vc[i].r.copy()[:,0] mv[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].model.f, vc=vc1)]) vc2 = vc[i].r.copy()[symIdx,:] vc2[:,0] = vc[i].r.copy()[symIdx,2] vc2[:,2] = vc[i].r.copy()[symIdx,0] mv2[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].model.f, vc=vc2)]) else: mv[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].f)]) else: on_step = None new_objs = objs.copy() for i in range(nCameras): new_objs['s2m_'+str(i)] = obj_s2m(w_s2m, i) new_objs['m2s_'+str(i)] = obj_m2s(w_m2s, i) print('weights: s2m %.2f m2s %.2f' % (w_s2m, w_m2s)) nbetas = len(sv[0].betas.r) free_variables = [] if fix_trans is False: for i in range(nCameras): free_variables.append(sv[i].trans) else: free_variables = [] if fix_shape is False: if alpha is not None: free_variables.append(alpha) else: if not fix_shape: for i in range(nCameras): free_variables.append(sv[i].betas) for i in range(nCameras): if not FIX_POSE: free_variables.append(sv[i].pose) # If objective contains 'feq', then add cam.f to free variables. if 'feq' in new_objs: for i in range(len(rends)): free_variables.append(cameras[i].f) opt = {'maxiter': max_iter, 'e_3': 1e-2} if max_iter > 0: ch.minimize(new_objs, x0=free_variables, method='dogleg', callback=on_step, options=opt) def render_and_show(sv): for i in range(len(rends)): img = imgs[i] img_res = render_mesh(Mesh(sv[i].r, sv[i].model.f), img.shape[1], img.shape[0], kp_camera[i], near=0.5, far=20) plt.figure() plt.imshow(img[:, :, ::-1]) plt.imshow(img_res) plt.axis('off') return rends[0].r, new_objs
def optimize_on_joints(j2d, model, cam, img, prior, init_pose, init_shape, n_betas=10, conf=None): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param init_pose: 72D vector, pose prediction results provided by HMR :param init_shape: 10D vector, shape prediction results provided by HMR :param n_betas: number of shape coefficients considered during optimization :param conf: 14D vector storing the confidence values from the CNN :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: cids = range(12) + [13] # joint ids for SMPL # SMPL does not have a joint for head, instead we use a vertex for the head # and append it later. smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # the vertex id for the joint corresponding to the head head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) # initialize the shape to the mean shape in the SMPL training set betas = ch.array(init_shape) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint, corresponding to a vertex... Jtr = ch.vstack((Jtr, sv[head_id])) # ... and add the joint id to the list smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[cids] if conf is not None else base_weights # project SMPL joints on the image plane using the estimated camera cam.v = Jtr # data term: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=None, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) return sv, cam.r, cam.t.r
result.betas[:]=np.zeros(10); result.pose[:]= np.zeros(72); print lista[iters] Target = loadObj(lista[iters]) #Rigid alignment scale=Ch(1); trans=ch.array([0,0,0]); Tar_shift = Target.vertices+trans; indexes=a['pF_lb2'].reshape(6890); distances=Tar_shift[indexes-1]-result*scale; (t)=ch.minimize(distances, x0 = [trans,result.pose[[0,1,2]]], method = 'dogleg', callback = None, options = {'maxiter': 50, 'e_3': .0001, 'disp': 1}) #Non-rigid Alignment c_pre={}; if (W_Joints): k=a['Joints_Target']; j_to_consider = range(0,24) J_distances = J_reposed[j_to_consider,:] - (k[j_to_consider,:]+trans); c_pre['Joints']= J_distances*W_Joints; if(W_FMP2P): c_pre['FMP2P']= distances*W_FMP2P; if(W_Norm_B): c_pre['Norm_B']= ((result.betas)**2)*W_Norm_B;
def initialize_camera( model, # pylint: disable=too-many-arguments, too-many-locals j2d, center, img, init_t, init_pose, conf, is_gt, flength=1000., pix_thsh=25., viz=False): """Initialize the camera.""" # try to optimize camera translation and rotation based on torso joints # right shoulder, left shoulder, right hip, left hip torso_cids = _CID_TORSO torso_smpl_ids = [2, 1, 17, 16] # initialize camera rotation and translation rt = _ch.zeros(3) # pylint: disable=no-member _LOGGER.info('Initializing camera: guessing translation via similarity') init_t = guess_init(model, flength, j2d, conf, init_pose, is_gt) t = _ch.array(init_t) # pylint: disable=no-member # check how close the shoulders are try_both_orient = _np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh opt_pose = _ch.array(init_pose) # pylint: disable=no-member (_, A_global) = _global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=_ch) Jtr = _ch.vstack([g[:3, 3] for g in A_global]) # pylint: disable=no-member # initialize the camera cam = _ProjectPoints(f=_np.array([flength, flength]), rt=rt, t=t, k=_np.zeros(5), c=center) # we are going to project the SMPL joints cam.v = Jtr if viz: viz_img = img.copy() # draw the target joints for coord in _np.around(j2d).astype(int): if (coord[0] < img.shape[1] and coord[0] >= 0 and coord[1] < img.shape[0] and coord[1] >= 0): _cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0]) import matplotlib.pyplot as plt plt.ion() # draw optimized joints at each iteration def on_step(_): """Draw a visualization.""" plt.figure(1, figsize=(5, 5)) plt.subplot(1, 1, 1) viz_img = img.copy() for coord in _np.around(cam.r[torso_smpl_ids]).astype(int): if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and coord[1] < viz_img.shape[0] and coord[1] >= 0): _cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255]) plt.imshow(viz_img[:, :, ::-1]) plt.draw() plt.show() else: on_step = None free_variables = [cam.t, opt_pose[:3]] # pylint: disable=no-member _ch.minimize( [(j2d[torso_cids] - cam[torso_smpl_ids]).T * conf[torso_cids], 1e2 * (cam.t[2] - init_t[2])], # pylint: disable=no-member # The same for verbose output. #{'cam': (j2d[torso_cids] - cam[torso_smpl_ids]).T * conf[torso_cids], # # Reduce the weight here to avoid the 'small people' problem. # 'cam_t': 1e2*(cam.t[2]-init_t[2])}, # pylint: disable=no-member x0=free_variables, method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) if viz: plt.ioff() return (cam, try_both_orient, opt_pose[:3].r)
def optimize_on_joints(j2d, model, cam, img, prior, try_both_orient, body_orient, exp_logistic, n_betas=10, inner_penetration=False, silh=None, conf=None, viz=False): """Run the optimization.""" if silh is not None: raise NotImplementedError("Silhouette fitting is not supported in " "this code release due to dependencies on " "proprietary code for the " "distance computation.") t0 = _time() # define the mapping LSP joints -> SMPL joints if j2d.shape[0] == 14: cids = range(12) + [13] elif j2d.shape[0] == 91: cids = range(j2d.shape[0]) else: raise Exception("Unknown number of joints: %d! Mapping not defined!" % j2d.shape[0]) # joint ids for SMPL smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # weight given to each joint during optimization; if j2d.shape[0] == 14: weights = [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1] else: weights = [1] * (len(smpl_ids) + len(landmark_mesh_91)) # The non-skeleton vertex ids are added later. if try_both_orient: flipped_orient = _cv2.Rodrigues(body_orient)[0].dot( _cv2.Rodrigues(_np.array([0., _np.pi, 0]))[0]) flipped_orient = _cv2.Rodrigues(flipped_orient)[0].ravel() orientations = [body_orient, flipped_orient] else: orientations = [body_orient] if try_both_orient: errors = [] svs = [] cams = [] # rends = [] for o_id, orient in enumerate(orientations): # initialize betas betas = _ch.zeros(n_betas) # pylint: disable=no-member init_pose = _np.hstack((orient, prior.weights.dot(prior.means))) # 2D joint error term # make the SMPL joint depend on betas Jdirs = _np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) # pylint: disable=no-member J_onbetas = _ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # instantiate the model sv = verts_decorated(trans=_ch.zeros(3), pose=_ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # get joint positions as a function of model pose, betas and trans (_, A_global) = _global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=_ch) Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + sv.trans if j2d.shape[0] == 14: # add the "fake" joint for the head head_id = _HEAD_REGR[0] Jtr = _ch.vstack((Jtr, sv[head_id])) if o_id == 0: smpl_ids.append(len(Jtr) - 1) else: # add the plain vertex IDs on the mesh surface. for vertex_id in landmark_mesh_91.values(): Jtr = _ch.vstack((Jtr, sv[vertex_id])) # add the joint id # for SMPL it's the last one added if o_id == 0: smpl_ids.append(len(Jtr) - 1) weights = _np.array(weights, dtype=_np.float64) if conf is not None: weights *= conf[cids] # we'll project the joints on the image plane cam.v = Jtr # data term: difference between observed and estimated joints obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * _GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # pose prior pprior = lambda w: w * prior(sv.pose) # pylint: disable=cell-var-from-loop # joint angles prior # 55: left elbow, should bend -np.pi/2 # 58: right elbow, should bend np.pi/2 # 12: left knee, should bend np.pi/2 # 15: right knee, should bend np.pi/2 if exp_logistic: _LOGGER.info('USING LOGISTIC') # Skinny Logistic function. as 50-> inf we get a step function at # 0.1. (0.1) is a margin bc 0 is still ok. my_exp = lambda x: 1 / (1 + _ch.exp(100 * (0.1 + -x))) else: x_0 = 0 #10 alpha = 10 my_exp = lambda x: alpha * _ch.exp((x - x_0)) # pylint: disable=cell-var-from-loop obj_angle = lambda w: w * _ch.concatenate([ my_exp(sv.pose[55]), # pylint: disable=cell-var-from-loop my_exp(-sv.pose[58]), # pylint: disable=cell-var-from-loop my_exp(-sv.pose[12]), # pylint: disable=cell-var-from-loop my_exp(-sv.pose[15]) ]) # pylint: disable=cell-var-from-loop if viz: from body.mesh.sphere import Sphere from body.mesh.meshviewer import MeshViewer import matplotlib.pyplot as plt # set up visualization # openGL window mv = MeshViewer(window_width=120, window_height=120) # and ids show_ids = _np.array(smpl_ids)[weights > 0] vc = _np.ones((len(Jtr), 3)) vc[show_ids] = [0, 1, 0] plt.ion() def on_step(_): """Create visualization.""" # show optimized joints in 3D # pylint: disable=cell-var-from-loop mv.set_dynamic_meshes([_Mesh(v=sv.r, f=[]), Sphere(center=cam.t.r, radius=.1).to_mesh()] \ + [Sphere(center=jc, radius=.01).to_mesh(vc[ijc]) for ijc, jc in enumerate(Jtr.r)]) plt.figure(1, figsize=(10, 10)) plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = img.copy() for coord, target_coord in zip( _np.around(cam.r[smpl_ids]).astype(int), _np.around(j2d[cids]).astype(int)): if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and coord[1] < tmp_img.shape[0] and coord[1] >= 0): _cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255]) if (target_coord[0] < tmp_img.shape[1] and target_coord[0] >= 0 and target_coord[1] < tmp_img.shape[0] and target_coord[1] >= 0): _cv2.circle(tmp_img, tuple(target_coord), 3, [0, 255, 0]) plt.imshow(tmp_img) plt.draw() plt.show() on_step(_) else: on_step = None sp = _SphereCollisions(pose=sv.pose, betas=sv.betas, model=model, regs=_REGRESSORS) sp.no_hands = True # configuration used with conf joints opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} #if stage < 2: objs['j2d'] = obj_j2d(1., 100) # TODO: evaluate. objs['pose'] = pprior(w) # WEIGHT FOR ANGLE if exp_logistic: # Set to high weight always. objs['pose_exp'] = obj_angle(5 * 1e3) else: objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas if inner_penetration: objs['sph_coll'] = 1e3 * sp try: _ch.minimize(objs.values(), x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) except AssertionError: # Divergence detected. _LOGGER.warn("Diverging optimization! Breaking!") break t1 = _time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if try_both_orient: errors.append((objs['j2d'].r**2).sum()) svs.append(sv) cams.append(cam) # rends.append(rend) if try_both_orient and errors[0] > errors[1]: choose_id = 1 else: choose_id = 0 if viz: plt.ioff() return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r, cams[choose_id].rt.r)
def final_fit( opt, part_mesh, v, v_offset, dist_o, dist_i, smpl_h_ref, rn_m, debug_rn, dif_mask, v_ids_template, faces_template, v_ids_side, faces_side, max_y, proj_cam, ref_joint_list_coup, ): if opt.disp_mesh_side: mv = MeshViewer() else: mv = None if opt.disp_mesh_whl: mv2 = MeshViewer() else: mv2 = None import scipy.sparse as sp sparse_solver = lambda A, x: sp.linalg.cg(A, x, maxiter=500)[0] tgt_pose = get_pose_prior(init_pose_path=opt.init_pose_path, gar_type=opt.gar_type) E = { 'mask': gaussian_pyramid(rn_m * dist_o * opt.ref_wt_dist_o + (1 - rn_m) * dist_i, n_levels=4, normalization='size') * opt.ref_wt_mask } x0 = [v_offset] if opt.ref_wt_coup: # x0 = [smpl_h_ref.trans, smpl_h_ref.betas, v_offset] E['coupling'] = (v + v_offset - smpl_h_ref[v_ids_template]) * opt.ref_wt_coup if opt.ref_wt_shp: E['beta_prior'] = ch.linalg.norm(smpl_h_ref.betas) * opt.ref_wt_shp if opt.ref_wt_pose: E['pose'] = (smpl_h_ref.pose - tgt_pose) * opt.ref_wt_pose if ref_joint_list_coup != None: range_joint = [] for item in ref_joint_list_coup: range_joint.append(3 * int(item)) range_joint.append(3 * int(item) + 1) range_joint.append(3 * int(item) + 2) x0 = x0 + [smpl_h_ref.pose[range_joint]] if opt.ref_use_betas: x0 = x0 + [smpl_h_ref.betas] if opt.ref_wt_proj: error_bd = get_rings_error(proj_cam, max_y) E['proj_bd'] = error_bd * opt.ref_wt_proj if opt.ref_wt_bd: gar_rings = compute_boundaries(v + v_offset, faces_template) error = smooth_rings(gar_rings, v + v_offset) E['boundary'] = error * opt.ref_wt_bd if opt.ref_wt_lap: lap_op = np.asarray(laplacian(part_mesh).todense()) lap_err = ch.dot(lap_op, v + v_offset) E['laplacian'] = lap_err * opt.ref_wt_lap ch.minimize(E, x0, method='dogleg', options={ 'e_3': .000001, 'sparse_solver': sparse_solver }, callback=get_callback_ref(rend=debug_rn, mask=dif_mask, vertices=v + v_offset, display=opt.display, v_ids_sides=v_ids_side, faces_template=faces_template, faces_side=faces_side, disp_mesh_side=opt.disp_mesh_side, disp_mesh_whl=opt.disp_mesh_whl, save_dir=opt.save_opt_images, mv=mv, mv2=mv2, show=opt.show)) final_mask = rn_m.r mask = dif_mask.copy() mask[dif_mask == 0.5] = 1 final_iou = compute_iou(mask, final_mask) return v + v_offset, final_iou
def optimize_on_joints(j2d, model, cam, img, prior, try_both_orient, body_orient, n_betas=10, regs=None, conf=None, viz=False): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: cids = range(12) + [13] # joint ids for SMPL # SMPL does not have a joint for head, instead we use a vertex for the head # and append it later. smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # the vertex id for the joint corresponding to the head head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array( [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) if try_both_orient: flipped_orient = cv2.Rodrigues(body_orient)[0].dot( cv2.Rodrigues(np.array([0., np.pi, 0]))[0]) flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel() orientations = [body_orient, flipped_orient] else: orientations = [body_orient] if try_both_orient: # store here the final error for both orientations, # and pick the orientation resulting in the lowest error errors = [] svs = [] cams = [] for o_id, orient in enumerate(orientations): # initialize the shape to the mean shape in the SMPL training set betas = ch.zeros(n_betas) # initialize the pose by using the optimized body orientation and the # pose prior init_pose = np.hstack((orient, prior.weights.dot(prior.means))) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated( trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas))]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation( sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint, corresponding to a vertex... Jtr = ch.vstack((Jtr, sv[head_id])) # ... and add the joint id to the list if o_id == 0: smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[ cids] if conf is not None else base_weights # project SMPL joints on the image plane using the estimated camera cam.v = Jtr # data term: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: ( w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[ 58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])]) if viz: import matplotlib.pyplot as plt plt.ion() def on_step(_): """Create visualization.""" plt.figure(1, figsize=(10, 10)) plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = img.copy() for coord, target_coord in zip( np.around(cam.r[smpl_ids]).astype(int), np.around(j2d[cids]).astype(int)): if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and coord[1] < tmp_img.shape[0] and coord[1] >= 0): cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255]) if (target_coord[0] < tmp_img.shape[1] and target_coord[0] >= 0 and target_coord[1] < tmp_img.shape[0] and target_coord[1] >= 0): cv2.circle(tmp_img, tuple(target_coord), 3, [0, 255, 0]) plt.imshow(tmp_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-2) on_step(_) else: on_step = None if regs is not None: # interpenetration term sp = SphereCollisions( pose=sv.pose, betas=sv.betas, model=model, regs=regs) sp.no_hands = True # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas if regs is not None: objs['sph_coll'] = 1e3 * sp ch.minimize( objs, x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={'maxiter': 100, 'e_3': .0001, 'disp': 0}) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if try_both_orient: errors.append((objs['j2d'].r**2).sum()) svs.append(sv) cams.append(cam) if try_both_orient and errors[0] > errors[1]: choose_id = 1 else: choose_id = 0 if viz: plt.ioff() return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)
def optimize_on_joints_and_silhouette(j2d, sil, model, cam, img, prior, init_pose, init_shape, n_betas=10, conf=None): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param sil: h x w silhouette with soft boundaries (np.float32, range(-1, 1)) :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param init_pose: 72D vector, pose prediction results provided by HMR :param init_shape: 10D vector, shape prediction results provided by HMR :param n_betas: number of shape coefficients considered during optimization :param conf: 14D vector storing the confidence values from the CNN :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ # define the mapping LSP joints -> SMPL joints cids = range(12) + [13] smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) betas = ch.array(init_shape) # instantiate the model: sv = verts_decorated(trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint Jtr = ch.vstack((Jtr, sv[head_id])) smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[cids] if conf is not None else base_weights # project SMPL joints and vertex on the image plane using the estimated camera cam.v = ch.vstack([Jtr, sv]) # obtain a gradient map of the soft silhouette grad_x = cv2.Sobel(sil, cv2.CV_32FC1, 1, 0) * 0.125 grad_y = cv2.Sobel(sil, cv2.CV_32FC1, 0, 1) * 0.125 # data term #1: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # data term #2: distance between the observed and projected boundaries obj_s2d = lambda w, sigma, flag, target_pose: (w * flag * GMOf( (target_pose - cam[len(Jtr):(len(Jtr) + 6890)]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors print('****** Optimization on joints') curr_pose = sv.pose.r opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas objs['thetas'] = wbetas * (sv.pose - curr_pose ) # constrain theta changes ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=None, options={ 'maxiter': 100, 'e_3': .001, 'disp': 0 }) curr_pose = sv.pose.r # cam.v = ch.vstack([Jtr, sv.r]) # run the optimization in 2 stages, progressively decreasing the # weights for the priors print('****** Optimization on silhouette and joints') opt_weights = zip([57.4, 4.78], [2e2, 1e2]) for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) # find the boundary vertices and estimate their expected location smpl_vs = cam.r[len(Jtr):, :] boundary_flag = np.zeros((smpl_vs.shape[0], 1)) expected_pos = np.zeros((smpl_vs.shape[0], 2)) for vi, v in enumerate(smpl_vs): r, c = int(v[1]), int(v[0]) if r < 0 or r >= sil.shape[0] or c < 0 or c >= sil.shape[1]: continue sil_v = sil[r, c] grad = np.array([grad_x[r, c], grad_y[r, c]]) grad_n = np.linalg.norm(grad) if grad_n > 1e-1 and sil_v < 0.4: # vertex on or out of the boundaries boundary_flag[vi] = 1.0 step = (grad / grad_n) * (sil_v / grad_n) expected_pos[vi] = np.array([c - step[0], r - step[1]]) # run optimization objs = {} objs['j2d'] = obj_j2d(1., 100) objs['s2d'] = obj_s2d(5., 100, boundary_flag, expected_pos) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas # constrain beta changes objs['thetas'] = wbetas * (sv.pose - curr_pose ) # constrain theta changes ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=None, options={ 'maxiter': 100, 'e_3': .001, 'disp': 0 }) return sv, cam.r, cam.t.r
def init(frames, body_height, b2m, viz_rn): betas = frames[0].smpl.betas E_height = None if body_height is not None: E_height = height_predictor(b2m, betas) - body_height * 1000. # first get a rough pose for all frames individually for i, f in enumerate(frames): if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.: if f.keypoints[2, 0] > f.keypoints[5, 0]: f.smpl.pose[0] = 0 f.smpl.pose[2] = np.pi E_init = {'init_pose_{}'.format(i): f.pose_obj[[0, 2, 5, 8, 11]]} x0 = [f.smpl.trans, f.smpl.pose[:3]] if E_height is not None and i == 0: E_init['height'] = E_height E_init['betas'] = betas x0.append(betas) ch.minimize(E_init, x0, method='dogleg', options={ 'e_3': .01, }, callback=get_cb(viz_rn, f)) weights = zip([5., 4.5, 4.], [5., 4., 3.]) E_betas = betas - betas.r for w_prior, w_betas in weights: x0 = [betas] E = { 'betas': E_betas * w_betas, } if E_height is not None: E['height'] = E_height for i, f in enumerate(frames): if np.sum(f.keypoints[[0, 2, 5, 8, 11], 2]) > 3.: x0.extend([ f.smpl.pose[range(21) + range(27, 30) + range(36, 60)], f.smpl.trans ]) E['pose_{}'.format(i)] = f.pose_obj E['prior_{}'.format(i)] = f.pose_prior_obj * w_prior ch.minimize(E, x0, method='dogleg', options={ 'e_3': .01, }, callback=get_cb(viz_rn, frames[0]))
def optimize_on_joints(j2d, model, cam, img, prior, try_both_orient, body_orient, n_betas=10, regs=None, conf=None, viz=False): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: cids = range(12) + [13] # joint ids for SMPL # SMPL does not have a joint for head, instead we use a vertex for the head # and append it later. smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # the vertex id for the joint corresponding to the head head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) if try_both_orient: flipped_orient = cv2.Rodrigues(body_orient)[0].dot( cv2.Rodrigues(np.array([0., np.pi, 0]))[0]) flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel() orientations = [body_orient, flipped_orient] else: orientations = [body_orient] if try_both_orient: # store here the final error for both orientations, # and pick the orientation resulting in the lowest error errors = [] svs = [] cams = [] for o_id, orient in enumerate(orientations): # initialize the shape to the mean shape in the SMPL training set betas = ch.zeros(n_betas) # initialize the pose by using the optimized body orientation and the # pose prior init_pose = np.hstack((orient, prior.weights.dot(prior.means))) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint, corresponding to a vertex... Jtr = ch.vstack((Jtr, sv[head_id])) # ... and add the joint id to the list if o_id == 0: smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[ cids] if conf is not None else base_weights # project SMPL joints on the image plane using the estimated camera cam.v = Jtr # data term: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) if viz: import matplotlib.pyplot as plt plt.ion() def on_step(_): """Create visualization.""" plt.figure(1, figsize=(10, 10)) plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = img.copy() for coord, target_coord in zip( np.around(cam.r[smpl_ids]).astype(int), np.around(j2d[cids]).astype(int)): if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and coord[1] < tmp_img.shape[0] and coord[1] >= 0): cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255]) if (target_coord[0] < tmp_img.shape[1] and target_coord[0] >= 0 and target_coord[1] < tmp_img.shape[0] and target_coord[1] >= 0): cv2.circle(tmp_img, tuple(target_coord), 3, [0, 255, 0]) plt.imshow(tmp_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-2) on_step(_) else: on_step = None if regs is not None: # interpenetration term sp = SphereCollisions(pose=sv.pose, betas=sv.betas, model=model, regs=regs) sp.no_hands = True # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas if regs is not None: objs['sph_coll'] = 1e3 * sp ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if try_both_orient: errors.append((objs['j2d'].r**2).sum()) svs.append(sv) cams.append(cam) if try_both_orient and errors[0] > errors[1]: choose_id = 1 else: choose_id = 0 if viz: plt.ioff() return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r, Jtr)
def fit_lmk3d( lmk_3d, # input landmark 3d model, # model lmk_face_idx, lmk_b_coords, # landmark embedding weights, # weights for the objectives shape_num=300, expr_num=100, opt_options=None): """ function: fit FLAME model to 3d landmarks input: lmk_3d: input landmark 3d, in shape (N,3) model: FLAME face model lmk_face_idx, lmk_b_coords: landmark embedding, in face indices and barycentric coordinates weights: weights for each objective shape_num, expr_num: numbers of shape and expression compoenents used opt_options: optimizaton options output: model.r: fitted result vertices model.f: fitted result triangulations (fixed in this code) parms: fitted model parameters """ # variables shape_idx = np.arange(0, min( 300, shape_num)) # valid shape component range in "betas": 0-299 expr_idx = np.arange(300, 300 + min( 100, expr_num)) # valid expression component range in "betas": 300-399 used_idx = np.union1d(shape_idx, expr_idx) model.betas[:] = np.random.rand( model.betas.size) * 0.0 # initialized to zero model.pose[:] = np.random.rand( model.pose.size) * 0.0 # initialized to zero #free_variables = [ model.trans, model.pose, model.betas[used_idx] ] free_variables = [model.pose, model.betas[used_idx]] # weights print "fit_lmk3d(): use the following weights:" for kk in weights.keys(): print "fit_lmk3d(): weights['%s'] = %f" % (kk, weights[kk]) # objectives # lmk scale = ch.array([1]) lmk_err = landmark_error_3d(mesh_verts=scale * model, mesh_faces=model.f, lmk_3d=lmk_3d, lmk_face_idx=lmk_face_idx, lmk_b_coords=lmk_b_coords, weight=weights['lmk']) # regularizer shape_err = weights['shape'] * model.betas[shape_idx] expr_err = weights['expr'] * model.betas[expr_idx] pose_err = weights['pose'] * model.pose[3:] # exclude global rotation objectives = {} objectives.update({ 'lmk': lmk_err, 'shape': shape_err, 'expr': expr_err, 'pose': pose_err }) # options if opt_options is None: print "fit_lmk3d(): no 'opt_options' provided, use default settings." import scipy.sparse as sp opt_options = {} opt_options['disp'] = 1 opt_options['delta_0'] = 0.1 opt_options['e_3'] = 1e-4 opt_options['maxiter'] = 100 sparse_solver = lambda A, x: sp.linalg.cg( A, x, maxiter=opt_options['maxiter'])[0] opt_options['sparse_solver'] = sparse_solver # on_step callback def on_step(_): pass # optimize # step 1: rigid alignment from time import time timer_start = time() ''' print "\nstep 1: start rigid fitting..." ch.minimize( fun = lmk_err, x0 = [ model.trans, model.pose[0:3],scale ], method = 'dogleg', callback = on_step, options = opt_options ) timer_end = time() align_v = scale*model.r align_f = model.f print "step 1: fitting done, in %f sec\n" % ( timer_end - timer_start ) print "scaleļ¼ %f\n" % scale print "model.trans" print model.trans print "model.rot" print model.pose[0:3] ''' # step 2: non-rigid alignment timer_start = time() print "step 2: start non-rigid fitting..." ch.minimize(fun=objectives, x0=free_variables, method='dogleg', callback=on_step, options=opt_options) timer_end = time() print "step 2: fitting done, in %f sec\n" % (timer_end - timer_start) # return results parms = { 'trans': model.trans.r, 'pose': model.pose.r, 'betas': model.betas.r } return scale * model.r, model.f, parms #,align_v,align_f
def initialize_camera(model, j2d, img, init_pose, flength=5000., pix_thsh=25., viz=False): """Initialize camera translation and body orientation :param model: SMPL model :param j2d: 14x2 array of CNN joints :param img: h x w x 3 image :param init_pose: 72D vector of pose parameters used for initialization :param flength: camera focal length (kept fixed) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the estimated camera, a boolean deciding if both the optimized body orientation and its flip should be considered, 3D vector for the body orientation """ # optimize camera translation and body orientation based on torso joints # LSP torso ids: # 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder torso_cids = [2, 3, 8, 9] # corresponding SMPL torso ids torso_smpl_ids = [2, 1, 17, 16] center = np.array([img.shape[1] / 2, img.shape[0] / 2]) # initialize camera rotation rt = ch.zeros(3) # initialize camera translation _LOGGER.info('initializing translation via similar triangles') init_t = guess_init(model, flength, j2d, init_pose) t = ch.array(init_t) # check how close the shoulder joints are try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh opt_pose = ch.array(init_pose) (_, A_global) = global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) # initialize the camera cam = ProjectPoints(f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center) # we are going to project the SMPL joints cam.v = Jtr if viz: viz_img = img.copy() # draw the target (CNN) joints for coord in np.around(j2d).astype(int): if (coord[0] < img.shape[1] and coord[0] >= 0 and coord[1] < img.shape[0] and coord[1] >= 0): cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0]) import matplotlib.pyplot as plt plt.ion() # draw optimized joints at each iteration def on_step(_): """Draw a visualization.""" plt.figure(1, figsize=(5, 5)) plt.subplot(1, 1, 1) viz_img = img.copy() for coord in np.around(cam.r[torso_smpl_ids]).astype(int): if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and coord[1] < viz_img.shape[0] and coord[1] >= 0): cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255]) plt.imshow(viz_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-3) else: on_step = None # optimize for camera translation and body orientation free_variables = [cam.t, opt_pose[:3]] ch.minimize( # data term defined over torso joints... { 'cam': j2d[torso_cids] - cam[torso_smpl_ids], # ...plus a regularizer for the camera translation 'cam_t': 1e2 * (cam.t[2] - init_t[2]) }, x0=free_variables, method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001, # disp set to 1 enables verbose output from the optimizer 'disp': 0 }) if viz: plt.ioff() return (cam, try_both_orient, opt_pose[:3].r)
def init_fit(opt, dist_o, dist_i, dif_mask, rn_m, smpl_h, v_ids_template, faces_template, debug_rn, v_ids_side, faces_side, joints_list): range_joint = [] for item in joints_list: range_joint.append(3 * int(item)) range_joint.append(3 * int(item) + 1) range_joint.append(3 * int(item) + 2) tgt_pose = get_pose_prior(init_pose_path=opt.init_pose_path, gar_type=opt.gar_type) # ============================================ # FIRST STAGE # ============================================ from psbody.mesh import MeshViewer if opt.disp_mesh_side: mv = MeshViewer() else: mv = None if opt.disp_mesh_whl: mv2 = MeshViewer() else: mv2 = None if opt.init_first_stage == "Trans": x0 = [smpl_h.trans] elif opt.init_first_stage == 'Pose': x0 = [smpl_h.trans, smpl_h.pose[range_joint]] # x0 = [smpl_h.trans] elif opt.init_first_stage == 'Shape': x0 = [smpl_h.trans, smpl_h.betas] elif opt.init_first_stage == 'Both': x0 = [smpl_h.trans, smpl_h.betas, smpl_h.pose[range_joint]] E = { 'mask': gaussian_pyramid(rn_m * dist_o * opt.init_fst_wt_dist_o + (1 - rn_m) * dist_i, n_levels=4, normalization='size') * opt.init_fst_wt_mask } if opt.init_fst_wt_betas: E['beta_prior'] = ch.linalg.norm(smpl_h.betas) * opt.init_fst_wt_betas if opt.init_fst_wt_pose: E['pose'] = (smpl_h.pose - tgt_pose) * opt.init_fst_wt_pose ch.minimize(E, x0, method='dogleg', options={ 'e_3': .0001, 'disp': True }, callback=get_callback(rend=debug_rn, mask=dif_mask, smpl=smpl_h, v_ids_template=v_ids_template, v_ids_sides=v_ids_side, faces_template=faces_template, faces_side=faces_side, display=opt.display, disp_mesh_side=opt.disp_mesh_side, disp_mesh_whl=opt.disp_mesh_whl, mv=mv, mv2=mv2, save_dir=opt.save_opt_images, show=opt.show)) # =============================================== # SECOND STAGE # =============================================== if opt.init_second_stage != "None": if opt.init_second_stage == 'Pose': x0 = [smpl_h.trans, smpl_h.pose[range_joint]] elif opt.init_second_stage == 'Shape': x0 = [smpl_h.trans, smpl_h.betas] elif opt.init_second_stage == 'Both': x0 = [smpl_h.trans, smpl_h.betas, smpl_h.pose[range_joint]] E = { 'mask': gaussian_pyramid(rn_m * dist_o * opt.init_sec_wt_dist_o + (1 - rn_m) * dist_i, n_levels=4, normalization='size') * opt.init_sec_wt_mask } if opt.init_sec_wt_betas: E['beta_prior'] = ch.linalg.norm( smpl_h.betas) * opt.init_sec_wt_betas if opt.init_sec_wt_pose: E['pose'] = (smpl_h.pose - tgt_pose) * opt.init_sec_wt_pose ch.minimize(E, x0, method='dogleg', options={'e_3': .0001}, callback=get_callback(rend=debug_rn, mask=dif_mask, smpl=smpl_h, v_ids_template=v_ids_template, v_ids_sides=v_ids_side, faces_template=faces_template, faces_side=faces_side, display=opt.display, disp_mesh_side=opt.disp_mesh_side, disp_mesh_whl=opt.disp_mesh_whl, mv=mv, mv2=mv2, save_dir=opt.save_opt_images, show=opt.show)) temp_params = { 'pose': smpl_h.pose.r, 'betas': smpl_h.betas.r, 'trans': smpl_h.trans.r, 'v_personal': smpl_h.v_personal.r } part_mesh = Mesh(smpl_h.r[v_ids_template], faces_template) return part_mesh, temp_params
def initialize_camera(model, j2d, img, init_pose, flength=5000., pix_thsh=25.): """Initialize camera translation and body orientation :param model: SMPL model :param j2d: 14x2 array of CNN joints :param img: h x w x 3 image :param init_pose: 72D vector of pose parameters used for initialization :param flength: camera focal length (kept fixed) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :returns: a tuple containing the estimated camera, a boolean deciding if both the optimized body orientation and its flip should be considered, 3D vector for the body orientation """ # optimize camera translation and body orientation based on torso joints # LSP torso ids: # 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder torso_cids = [2, 3, 8, 9] # corresponding SMPL torso ids torso_smpl_ids = [2, 1, 17, 16] center = np.array([img.shape[1] / 2, img.shape[0] / 2]) # initialize camera rotation rt = ch.zeros(3) # initialize camera translation print 'initializing translation via similar triangles' init_t = guess_init(model, flength, j2d, init_pose) t = ch.array(init_t) # check how close the shoulder joints are try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh opt_pose = ch.array(init_pose) (_, A_global) = global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) # initialize the camera cam = ProjectPoints(f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center) # we are going to project the SMPL joints cam.v = Jtr on_step = None # optimize for camera translation and body orientation free_variables = [cam.t, opt_pose[:3]] ch.minimize( # data term defined over torso joints... { 'cam': j2d[torso_cids] - cam[torso_smpl_ids], # ...plus a regularizer for the camera translation 'cam_t': 1e2 * (cam.t[2] - init_t[2]) }, x0=free_variables, method='dogleg', callback=on_step, # maxiter: 100, e_3: .0001 options={ 'maxiter': 100, 'e_3': .0001, # disp set to 1 enables verbose output from the optimizer 'disp': 0 }) return (cam, try_both_orient, opt_pose[:3].r)