def run_single_fit(img, j2d, conf, seg, model, init_pose, init_shape, n_betas=10, flength=flength): """Run the fit for one specific image. :param img: h x w x 3 image :param j2d: 14x2 array of CNN joints :param conf: 14D vector storing the confidence values from the CNN :param seg: h x w soft silhouette :param model: SMPL model :param init_pose: 72D vector, pose prediction results provided by HMR :param init_shape: 10D vector, shape prediction results provided by HMR :param n_betas: number of shape coefficients considered during optimization :param flength: camera focal length (kept fixed during optimization) :returns: a tuple containing camera/model parameters """ # create the pose prior (GMM over CMU) prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior() # get the mean pose as our initial pose # init_pose = np.hstack((np.zeros(3), prior.weights.dot(prior.means))) # estimate the camera parameters print('**** Run camera optimization') (cam, body_orient) = initialize_camera(model, j2d, img, init_pose, flength=flength) # fit # print('**** Run body model optimization (on joints)') # (sv, opt_j2d, t) = optimize_on_joints(j2d, model, cam, img, prior, # init_pose, init_shape, # n_betas=n_betas, conf=conf) print('**** Run body model optimization (on silhouette and joints)') (sv, opt_j2d, t) = optimize_on_joints_and_silhouette(j2d, seg, model, cam, img, prior, init_pose, init_shape, n_betas=n_betas, conf=conf) # return fit parameters params = {'cam_t': cam.t.r, 'f': cam.f.r, 'pose': sv.pose.r, 'betas': sv.betas.r} return params
def run_single_fit(img, j2d, conf, model, regs=None, n_betas=10, flength=5000., pix_thsh=25., scale_factor=1, viz=False, do_degrees=None): """Run the fit for one specific image. :param img: h x w x 3 image :param j2d: 14x2 array of CNN joints :param conf: 14D vector storing the confidence values from the CNN :param model: SMPL model :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param n_betas: number of shape coefficients considered during optimization :param flength: camera focal length (kept fixed during optimization) :param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both the estimated one and its flip) :param scale_factor: int, rescale the image (for LSP, slightly greater images -- 2x -- help obtain better fits) :param viz: boolean, if True enables visualization during optimization :param do_degrees: list of degrees in azimuth to render the final fit when saving results :returns: a tuple containing camera/model parameters and images with rendered fits """ if do_degrees is None: do_degrees = [] # create the pose prior (GMM over CMU) prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior() # get the mean pose as our initial pose init_pose = np.hstack((np.zeros(3), prior.weights.dot(prior.means))) if scale_factor != 1: img = cv2.resize( img, (img.shape[1] * scale_factor, img.shape[0] * scale_factor)) j2d[:, 0] *= scale_factor j2d[:, 1] *= scale_factor # estimate the camera parameters (cam, try_both_orient, body_orient) = initialize_camera(model, j2d, img, init_pose, flength=flength, pix_thsh=pix_thsh, viz=viz) # fit (sv, opt_j2d, t) = optimize_on_joints( j2d, model, cam, img, prior, try_both_orient, body_orient, n_betas=n_betas, conf=conf, viz=viz, regs=regs, ) h = img.shape[0] w = img.shape[1] dist = np.abs(cam.t.r[2] - np.mean(sv.r, axis=0)[2]) images = [] orig_v = sv.r for deg in do_degrees: if deg != 0: aroundy = cv2.Rodrigues(np.array([0, np.radians(deg), 0]))[0] center = orig_v.mean(axis=0) new_v = np.dot((orig_v - center), aroundy) verts = new_v + center else: verts = orig_v # now render im = (render_model(verts, model.f, w, h, cam, far=20 + dist) * 255.).astype('uint8') images.append(im) # return fit parameters params = { 'cam_t': cam.t.r, 'f': cam.f.r, 'pose': sv.pose.r, 'betas': sv.betas.r } return params, images
def optimize_on_joints3D(model, joints3D, opt_shape=False, viz=True): """Fit the model to the given set of 3D joints :param model: initial SMPL model ===> is modified after optimization :param joints3D: 3D joint locations [16 x 3] :param opt_shape: boolean, if True optimizes for shape parameter betas :param viz: boolean, if True enables visualization during optimization """ t0 = time() if joints3D.shape[0] == 16: obj_joints3D = lambda w, sigma: (w * GMOf( (joints3D - model.J_transformed[get_indices_16()]), sigma)) elif joints3D.shape[0] == 24: obj_joints3D = lambda w, sigma: (w * GMOf( (joints3D - model.J_transformed), sigma)) else: raise ('How many joints?') # Create the pose prior (GMM over CMU) prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior() pprior = lambda w: w * prior(model.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 my_exp = lambda x: 10 * chumpy.exp(x) obj_angle = lambda w: w * chumpy.concatenate([ my_exp(model.pose[55]), my_exp(-model.pose[58]), my_exp(-model.pose[12]), my_exp(-model.pose[15]) ]) # Visualization at optimization step if viz: def on_step(_): """Draw a visualization.""" plt.figure(1, figsize=(5, 5)) renderBody(model) plt.draw() plt.pause(1e-3) else: on_step = None # weight configuration (pose and shape: original values as in SMPLify) # the first list contains the weights for the pose prior, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) print('Initial: error(joints3D) = %.2f' % (obj_joints3D(100, 100).r**2).sum()) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (wpose, wbetas) in enumerate(opt_weights): objs = {} objs['joints3D'] = obj_joints3D(100., 100) objs['pose'] = pprior(wpose) objs['pose_exp'] = obj_angle(0.317 * wpose) if opt_shape: objs['betas'] = wbetas * model.betas chumpy.minimize(objs, x0=[model.pose, model.betas], method='dogleg', callback=on_step, options={ 'maxiter': 1000, 'e_3': .0001, 'disp': 0 }) else: chumpy.minimize(objs, x0=[model.pose], method='dogleg', callback=on_step, options={ 'maxiter': 1000, 'e_3': .0001, 'disp': 0 }) print('Stage %d: error(joints3D) = %.2f' % (stage, (objs['joints3D'].r**2).sum())) print('\nElapsed theta fitting (%d joints): %.2f sec.' % (joints3D.shape[0], (time() - t0)))
def optimize_on_vertices(model, vertices, joints3D=np.zeros(1), weights_corr=np.zeros(1), vertices_cross_corr=np.zeros(1), indices_cross_corr=np.zeros(1), weights_cross_corr=np.zeros(1), opt_trans=False, viz=True): """Fit the model to the given set of 3D vertices and 3D joints :param model: initial SMPL model ===> is modified after optimization :param vertices: 3D vertex locations to fit [num_vertices x 3] :param joints3D: 3D joint locations to fit [24 x 3] :param vertices_cross_corr, indices_cross_corr, weights_cross_corr: :for each point in vertices_cross_corr, we have the index of its corresponding smpl vertex and the weight :for this correspondence :param opt_trans: boolean, if True optimizes only translation :param viz: boolean, if True enables visualization during optimization """ t0 = time() # Optimization term on the joints3D distance if joints3D.shape[0] > 1: if joints3D.shape[0] == 16: obj_joints3d = lambda w, sigma: (w * GMOf( (joints3D - model.J_transformed[get_indices_16()]), sigma)) elif joints3D.shape[0] == 24: obj_joints3d = lambda w, sigma: (w * GMOf( (joints3D - model.J_transformed), sigma)) else: raise ('How many joints?') # data term: distance between observed and estimated points in 3D if (weights_corr.shape[0] == 1): weights_corr = np.ones((vertices.shape[0])) obj_vertices = lambda w, sigma: (w * GMOf( ((vertices.T * weights_corr) - (model.T * weights_corr)).T, sigma)) if (vertices_cross_corr.shape[0] > 1): smplV = model[indices_cross_corr.astype(int), :] obj_vertices_cross = lambda w, sigma: (w * GMOf( ((vertices_cross_corr.T * weights_cross_corr) - (smplV.T * weights_cross_corr)).T, sigma)) # Create the pose prior (GMM over CMU) prior = MaxMixtureCompletePrior(n_gaussians=8).get_gmm_prior() pprior = lambda w: w * prior(model.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 my_exp = lambda x: 10 * chumpy.exp(x) obj_angle = lambda w: w * chumpy.concatenate([ my_exp(model.pose[55]), my_exp(-model.pose[58]), my_exp(-model.pose[12]), my_exp(-model.pose[15]) ]) # Visualization at optimization step if viz: def on_step(_): """Draw a visualization.""" plt.figure(1, figsize=(5, 5)) renderBody(model) plt.draw() plt.pause(1e-3) else: on_step = None # weight configuration (pose and shape: original values as in SMPLify) # the first list contains the weights for the pose prior, # the second list contains the weights for the shape prior # the third list contains the weights for the joints3D loss opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1], [5, 5, 5, 5]) print('Initial:') print('\terror(vertices) = %.2f' % (obj_vertices(100, 100).r**2).sum()) if (joints3D.shape[0] > 1): print('\terror(joints3d) = %.2f' % (obj_joints3d(100, 100).r**2).sum()) if (vertices_cross_corr.shape[0] > 1): print('\terror(vertices_cross) = %.2f' % (obj_vertices_cross(100, 100).r**2).sum()) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (wpose, wbetas, wjoints3D) in enumerate(opt_weights): print('Stage %d' % stage) objs = {} if (joints3D.shape[0] > 1): objs['joints3D'] = wjoints3D * obj_joints3d(100., 100) objs['vertices'] = obj_vertices(100., 100) if (vertices_cross_corr.shape[0] > 1): objs['vertices_cross'] = obj_vertices_cross(100., 100) objs['pose'] = pprior(wpose) objs['pose_exp'] = obj_angle(0.317 * wpose) objs['betas'] = wbetas * model.betas if opt_trans: chumpy.minimize(objs, x0=[model.trans], method='dogleg', callback=on_step, options={ 'maxiter': 1000, 'e_3': .0001, 'disp': 0 }) else: chumpy.minimize(objs, x0=[model.pose, model.betas], method='dogleg', callback=on_step, options={ 'maxiter': 1000, 'e_3': .0001, 'disp': 0 }) print('\terror(vertices) = %.2f' % (objs['vertices'].r**2).sum()) if (joints3D.shape[0] > 1): print('\terror(joints3D) = %.2f' % (objs['joints3D'].r**2).sum()) if (vertices_cross_corr.shape[0] > 1): print('\terror(vertices_cross) = %.2f' % (objs['vertices_cross'].r**2).sum()) print('Elapsed iteration %.2f sec.' % (time() - t0))