def make_prdicted_mesh_neutral(predicted_params_path, flame_model_path): params = np.load(predicted_params_path, allow_pickle=True, encoding='latin1') #print(params) params = params[()] pose = np.zeros(15) #expression = np.zeros(100) shape = np.hstack( (params['shape'], np.zeros(300 - params['shape'].shape[0]))) #pose = np.hstack((params['pose'], np.zeros(15-params['pose'].shape[0]))) expression = np.hstack( (params['expression'], np.zeros(100 - params['expression'].shape[0]))) flame_genral_model = load_model(flame_model_path) generated_neutral_mesh = verts_decorated( #ch.array([0.0,0.0,0.0]), ch.array(params['cam']), ch.array(pose), ch.array(flame_genral_model.r), flame_genral_model.J_regressor, ch.array(flame_genral_model.weights), flame_genral_model.kintree_table, flame_genral_model.bs_style, flame_genral_model.f, bs_type=flame_genral_model.bs_type, posedirs=ch.array(flame_genral_model.posedirs), betas=ch.array( np.hstack((shape, expression)) ), #betas=ch.array(np.concatenate((theta[0,75:85], np.zeros(390)))), # shapedirs=ch.array(flame_genral_model.shapedirs), want_Jtr=True) # neutral_mesh = Mesh(v=generated_neutral_mesh.r, f=generated_neutral_mesh.f) neutral_mesh = trimesh.Trimesh(vertices=generated_neutral_mesh.r, faces=generated_neutral_mesh.f) return neutral_mesh
def load_shape_models(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, pose, trans, betas): shape_model = [None] * nViews v_template = ch.array(model.v_template) + dv if exists(mesh_v_opt_save_path): opti_mesh = Mesh(filename=mesh_v_opt_save_path) v_opt = ch.array(opti_mesh.v.copy()) compute = False else: v_opt = None compute = True for i in range(nViews): if v_opt is not None: v_template = v_opt model.betas[:] = 0 np_betas = np.zeros_like(model.betas) else: v_template = ch.array(model.v_template) + dv np_betas = np.zeros_like(model.betas) np_betas[:len(betas[i])] = betas[i] shape_model[i] = verts_decorated(v_template=v_template, pose=ch.array(pose[i]), trans=ch.array(trans[i]), J=model.J_regressor, kintree_table=model.kintree_table, betas=ch.array(np_betas), weights=model.weights, posedirs=model.posedirs, shapedirs=model.shapedirs, bs_type='lrotmin', bs_style='lbs', f=model.f) return shape_model, compute
def optimize_on_joints(j2d, model, cam, img, prior, try_both_orient, body_orient, n_betas=10, regs=None, conf=None, viz=False): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: cids = range(12) + [13] # joint ids for SMPL # SMPL does not have a joint for head, instead we use a vertex for the head # and append it later. smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # the vertex id for the joint corresponding to the head head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) if try_both_orient: flipped_orient = cv2.Rodrigues(body_orient)[0].dot( cv2.Rodrigues(np.array([0., np.pi, 0]))[0]) flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel() orientations = [body_orient, flipped_orient] else: orientations = [body_orient] if try_both_orient: # store here the final error for both orientations, # and pick the orientation resulting in the lowest error errors = [] svs = [] cams = [] for o_id, orient in enumerate(orientations): # initialize the shape to the mean shape in the SMPL training set betas = ch.zeros(n_betas) # initialize the pose by using the optimized body orientation and the # pose prior init_pose = np.hstack((orient, prior.weights.dot(prior.means))) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint, corresponding to a vertex... Jtr = ch.vstack((Jtr, sv[head_id])) # ... and add the joint id to the list if o_id == 0: smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[ cids] if conf is not None else base_weights # project SMPL joints on the image plane using the estimated camera cam.v = Jtr # data term: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) if viz: import matplotlib.pyplot as plt plt.ion() def on_step(_): """Create visualization.""" plt.figure(1, figsize=(10, 10)) plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = img.copy() for coord, target_coord in zip( np.around(cam.r[smpl_ids]).astype(int), np.around(j2d[cids]).astype(int)): if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and coord[1] < tmp_img.shape[0] and coord[1] >= 0): cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255]) if (target_coord[0] < tmp_img.shape[1] and target_coord[0] >= 0 and target_coord[1] < tmp_img.shape[0] and target_coord[1] >= 0): cv2.circle(tmp_img, tuple(target_coord), 3, [0, 255, 0]) plt.imshow(tmp_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-2) on_step(_) else: on_step = None if regs is not None: # interpenetration term sp = SphereCollisions(pose=sv.pose, betas=sv.betas, model=model, regs=regs) sp.no_hands = True # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas if regs is not None: objs['sph_coll'] = 1e3 * sp ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if try_both_orient: errors.append((objs['j2d'].r**2).sum()) svs.append(sv) cams.append(cam) if try_both_orient and errors[0] > errors[1]: choose_id = 1 else: choose_id = 0 if viz: plt.ioff() return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r, Jtr)
def optimize_smal(fposes, ftrans, fbetas, model, cams, segs, imgs, landmarks, landmarks_names, key_vids, symIdx=None, frameId=0, opt_model_dir=None, save_name=None, COMPUTE_OPT=True, img_paths=None, img_offset=None, img_scales=None): mesh_v_opt_save_path = join(opt_model_dir, 'mesh_v_opt_no_mc_' + str(frameId) + '.ply') mesh_v_opt_mc_save_path = join(opt_model_dir, 'mesh_v_opt_' + str(frameId) + '.ply') mesh_init_save_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') nViews = len(fposes) if not COMPUTE_OPT: if not exists(opt_model_dir): makedirs(opt_model_dir) dv = 0 compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_init_save_path, fposes, ftrans, fbetas, '_no_refine', cams, imgs, segs, img_paths, img_offset, img_scales) return # Write the initial mesh np_betas = np.zeros_like(model.betas) np_betas[:len(fbetas[0])] = fbetas[0] tmp = verts_decorated(v_template=model.v_template, pose=ch.zeros_like(model.pose.r), trans=ch.zeros_like(model.trans), J=model.J_regressor, kintree_table=model.kintree_table, betas=ch.array(np_betas), weights=model.weights, posedirs=model.posedirs, shapedirs=model.shapedirs, bs_type='lrotmin', bs_style='lbs', f=model.f) tmp_mesh = Mesh(v=tmp.r, f=tmp.f) tmp_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') tmp_mesh.write_ply(tmp_path) del tmp assert (nViews == len(cams)) assert (nViews == len(segs)) assert (nViews == len(imgs)) # Define a displacement vector. We set a small non zero displacement as initialization dv = ch.array(np.random.rand(model.r.shape[0], 3) / 1000.) # Cell structure for ARAP f = model.f _, A3, A = edgesIdx(nV=dv.shape[0], f=f, save_dir='.', name='smal') wedge = wedges(A3, dv) s = np.zeros_like(dv) arap = ARAP(reg_e=MatVecMult(A3.T, model.ravel() + dv.ravel()).reshape(-1, 3), model_e=MatVecMult(A3.T, model.ravel()).reshape(-1, 3), w=wedge, A=A) k_arap = settings['ref_k_arap_per_view'] * nViews for weight, part in zip(settings['ref_W_arap_values'], settings['ref_W_arap_parts']): k_arap, W_per_vertex = get_arap_part_weights( A, k_arap, [part], [weight]) #, animal_name) # was only Head W = np.zeros((W_per_vertex.shape[0], 3)) for i in range(3): W[:, i] = W_per_vertex k_lap = settings['ref_k_lap'] * nViews * W k_sym = settings['ref_k_sym'] * nViews k_keyp = settings['ref_k_keyp_weight'] * nViews # Load already computed mesh if not exists(opt_model_dir): makedirs(opt_model_dir) shape_model, compute = load_shape_models(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas) mv = None # Remove inside mouth faces ''' if settings['ref_remove_inside_mouth']: # Giraffe faces_orig = shape_model[0].f.copy() im_v = im_up_v + im_down_v idx = [np.where(model.f == ix)[0] for ix in im_v] idx = np.concatenate(idx).ravel() for i in range(nViews): shape_model[i].f = np.delete(shape_model[i].f, idx, 0) ''' if compute: objs = {} FIX_CAM = True free_variables = [] kp_weights = k_keyp * np.ones((landmarks[0].shape[0], 1)) print('removing shoulders, often bad annotated') kp_weights[landmarks_names.index('leftShoulder'), :] *= 0 kp_weights[landmarks_names.index('rightShoulder'), :] *= 0 objs_pose = None j2d = None #k_silh_term = settings['ref_k_silh_term'] k_m2s = settings['ref_k_m2s'] k_s2m = settings['ref_k_s2m'] objs, params_, j2d = set_pose_objs(shape_model, cams, landmarks, key_vids, kp_weights=kp_weights, FIX_CAM=FIX_CAM, ONLY_KEYP=True, OPT_SHAPE=False) if np.any(k_arap) != 0: objs['arap'] = k_arap * arap if k_sym != 0: objs['sym_0'] = k_sym * (ch.abs(dv[:, 0] - dv[symIdx, 0])) objs['sym_1'] = k_sym * (ch.abs(dv[:, 1] + dv[symIdx, 1] - 0.00014954)) objs['sym_2'] = k_sym * (ch.abs(dv[:, 2] - dv[symIdx, 2])) if np.any(k_lap) != 0: lap_op = np.asarray( laplacian(Mesh(v=dv, f=shape_model[0].f)).todense()) objs['lap'] = k_lap * ch.dot(lap_op, dv) mv = None mv2 = MeshViewers(shape=(1, nViews)) #None vc = np.ones_like(dv) dv_r = fit_silhouettes_pyramid_opt(objs, shape_model, dv, segs, cams, j2d=j2d, weights=1., mv=mv, imgs=imgs, s2m_weights=k_s2m, m2s_weights=k_m2s, max_iter=100, free_variables=free_variables, vc=vc, symIdx=symIdx, mv2=mv2, objs_pose=objs_pose) # Save result image for i in range(nViews): img_res = render_mesh(Mesh(shape_model[i].r, shape_model[i].f), imgs[i].shape[1], imgs[i].shape[0], cams[i], img=imgs[i], world_frame=True) img_result = np.hstack((imgs[i], img_res * 255.)) save_img_path = save_name[i].replace('.pkl', '_v_opt.png') cv2.imwrite(save_img_path, img_result) shape_model[0].pose[:] = 0 shape_model[0].trans[:] = 0 V = shape_model[0].r.copy() vm = V[symIdx, :].copy() vm[:, 1] = -1 * vm[:, 1] V2 = (V + vm) / 2.0 mesh_out = Mesh(v=V2, f=shape_model[0].f) mesh_out.show() mesh_out.write_ply(mesh_v_opt_save_path) save_dv_data_path = mesh_v_opt_save_path.replace('.ply', '_dv.pkl') dv_data = {'betas': shape_model[0].betas.r, 'dv': dv_r} pkl.dump(dv_data, open(save_dv_data_path, 'wb')) compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas, '_non_opt', cams, imgs, segs, img_paths, img_offset, img_scales) return
def optimize_on_joints_and_silhouette(j2d, sil, model, cam, img, prior, init_pose, init_shape, n_betas=10, conf=None): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param sil: h x w silhouette with soft boundaries (np.float32, range(-1, 1)) :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param init_pose: 72D vector, pose prediction results provided by HMR :param init_shape: 10D vector, shape prediction results provided by HMR :param n_betas: number of shape coefficients considered during optimization :param conf: 14D vector storing the confidence values from the CNN :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ # define the mapping LSP joints -> SMPL joints cids = range(12) + [13] smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) betas = ch.array(init_shape) # instantiate the model: sv = verts_decorated(trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint Jtr = ch.vstack((Jtr, sv[head_id])) smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[cids] if conf is not None else base_weights # project SMPL joints and vertex on the image plane using the estimated camera cam.v = ch.vstack([Jtr, sv]) # obtain a gradient map of the soft silhouette grad_x = cv2.Sobel(sil, cv2.CV_32FC1, 1, 0) * 0.125 grad_y = cv2.Sobel(sil, cv2.CV_32FC1, 0, 1) * 0.125 # data term #1: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # data term #2: distance between the observed and projected boundaries obj_s2d = lambda w, sigma, flag, target_pose: (w * flag * GMOf( (target_pose - cam[len(Jtr):(len(Jtr) + 6890)]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors print('****** Optimization on joints') curr_pose = sv.pose.r opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas objs['thetas'] = wbetas * (sv.pose - curr_pose ) # constrain theta changes ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=None, options={ 'maxiter': 100, 'e_3': .001, 'disp': 0 }) curr_pose = sv.pose.r # cam.v = ch.vstack([Jtr, sv.r]) # run the optimization in 2 stages, progressively decreasing the # weights for the priors print('****** Optimization on silhouette and joints') opt_weights = zip([57.4, 4.78], [2e2, 1e2]) for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) # find the boundary vertices and estimate their expected location smpl_vs = cam.r[len(Jtr):, :] boundary_flag = np.zeros((smpl_vs.shape[0], 1)) expected_pos = np.zeros((smpl_vs.shape[0], 2)) for vi, v in enumerate(smpl_vs): r, c = int(v[1]), int(v[0]) if r < 0 or r >= sil.shape[0] or c < 0 or c >= sil.shape[1]: continue sil_v = sil[r, c] grad = np.array([grad_x[r, c], grad_y[r, c]]) grad_n = np.linalg.norm(grad) if grad_n > 1e-1 and sil_v < 0.4: # vertex on or out of the boundaries boundary_flag[vi] = 1.0 step = (grad / grad_n) * (sil_v / grad_n) expected_pos[vi] = np.array([c - step[0], r - step[1]]) # run optimization objs = {} objs['j2d'] = obj_j2d(1., 100) objs['s2d'] = obj_s2d(5., 100, boundary_flag, expected_pos) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas # constrain beta changes objs['thetas'] = wbetas * (sv.pose - curr_pose ) # constrain theta changes ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=None, options={ 'maxiter': 100, 'e_3': .001, 'disp': 0 }) return sv, cam.r, cam.t.r
def optimize_on_DensePose(ori_img, tar_img, dp_iuv, op_j2d, w_j2d, model, cam, cam_old, prior, init_trans, init_pose, init_betas, n_betas=10, viz=False, imageid=1): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ outPath = '/home/xiul/databag/net_images/smplify/vid_{:06d}_puredense'.format( imageid) if not os.path.isdir(outPath): os.mkdir(outPath) dp_iuv_tar = dp_iuv.copy() width = dp_iuv_tar.shape[1] height = dp_iuv_tar.shape[0] dp_iuv_weight = np.zeros(dp_iuv_tar.shape) + 10 dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 0] = 10 dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 1] = 10 dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 2] = 15 dp_iuv_weight = ch.array(dp_iuv_weight) t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: tarIUV = ch.array(dp_iuv) # initialize the shape to the mean shape in the SMPL training set betas = ch.array(init_betas) # initialize the pose by using the optimized body orientation and the # pose prior pose = ch.array(init_pose) #init_pose = np.hstack((body_orient, body_init)) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.array(init_trans), pose=pose, v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=None) J_tmpx = MatVecMult(J_reg, sv[:, 0]) J_tmpy = MatVecMult(J_reg, sv[:, 1]) J_tmpz = MatVecMult(J_reg, sv[:, 2]) Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T #Jtr = J_reg.dot(sv) cam.v = Jtr reIUV = render_model(sv, model.f, width, height, cam, near=0.5, far=25, vc=dp_colors, img=None) reModel = render_model(sv, model.f, width, height, cam, near=0.5, far=25, vc=None, img=None) fullModel = render_model(sv, model.f, ori_img.shape[1], ori_img.shape[0], cam_old, near=0.5, far=25, vc=None, img=None) #gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None): def obj_j2d(w, sigma): return (w * w_j2d.reshape((-1, 1)) * GMOf((op_j2d - cam), sigma)) #input_objective, imshape, normalization, n_levels, as_list err = (tarIUV - reIUV) * dp_iuv_weight obj_dense = 100 * gaussian_pyramid(err, n_levels=4, normalization='SSE') #obj_dense = gaussian_pyramid(err, n_levels=4, normalization=SSE) #obj_dense = 10000*err # data term: distance between observed and estimated joints in 2D # obj_dense = lambda w, sigma: ( # w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma)) # mixture of gaussians pose prior def pprior(w): return w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 def my_exp(x): return alpha * ch.exp(x) def obj_angle(w): return w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) def viz_func(stage_num): plt.figure(1, figsize=(10, 10)) plt.clf() plt.subplot(2, 3, 1) # show optimized joints in 2D tmp_img = reIUV.r.copy() tmp_tar = tarIUV.copy() tmp_model = reModel.r.copy() full_model = fullModel.r.copy() # w = tmp_tar.shape[1] # h = tmp_tar.shape[0] plt.imshow(tar_img) plt.imshow(tmp_img, alpha=0.5) for j1, j2, w_ts in zip(cam.r, op_j2d, w_j2d): if (w_ts > 0): plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r') plt.xlim([0, width]) plt.ylim([height, 0]) plt.subplot(2, 3, 2) plt.imshow(tar_img) plt.imshow(tmp_tar, alpha=0.5) plt.xlim([0, width]) plt.ylim([height, 0]) plt.subplot(2, 3, 3) plt.imshow(tar_img) tmp_model_alpha = np.ones((tmp_model.shape[0], tmp_model.shape[1])) tmp_model_alpha[tmp_model[:, :, 0] < 1e-2] = 0 tmp_model = np.dstack((tmp_model, tmp_model_alpha)) plt.imshow(tmp_model) plt.xlim([0, width]) plt.ylim([height, 0]) plt.subplot(2, 1, 2) plt.imshow(full_model) plt.draw() plt.savefig(os.path.join(outPath, 'stage-{}.png'.format(stage_num)), bbox_inches='tight') full_model_int = full_model.copy() full_model_int *= 255 full_model_int = full_model_int.astype(np.uint8) cv2.imwrite(os.path.join(outPath, 'model-{}.png'.format(stage_num)), full_model_int) out_params = { 'pose': sv.pose.r, 'shape': sv.betas.r, 'trans': sv.trans.r } with open(os.path.join(outPath, 'param-{}.pkl'.format(stage_num)), 'w') as fio: pickle.dump(out_params, fio, pickle.HIGHEST_PROTOCOL) viz_func(0) #if viz: if viz and False: def on_step(cstep): """Create visualization.""" # TODO this function is in vis_func #plt.savefig(os.path.join(outPath,'{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),bbox_inches='tight') on_step(stepI) else: on_step = None # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior # opt_weights = zip([4.78, 3.78, 2.78, 1.78], # [5, 5, 5, 5], # [1, 0.25, 0.10, 0]) opt_weights = zip([4.78, 4.78, 4.78, 4.78, 4.78], [50, 50, 50, 50, 50], [0.1, 0.1, 0.5, 0.25, 0.1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas, w_joints) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} #if stage >= 1: objs['dense'] = obj_dense objs['j2d'] = obj_j2d(w_joints, 50) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas ch.minimize(objs, x0=[sv.betas, sv.pose, sv.trans], method='dogleg', callback=on_step, options={ 'maxiter': 10000, 'e_3': .0001, 'disp': 1 }) viz_func(stage + 1) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if viz and False: plt.ioff()
def optimize_on_joints(j2d, model, cam, img, prior, init_pose, init_shape, n_betas=10, conf=None): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param init_pose: 72D vector, pose prediction results provided by HMR :param init_shape: 10D vector, shape prediction results provided by HMR :param n_betas: number of shape coefficients considered during optimization :param conf: 14D vector storing the confidence values from the CNN :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: cids = range(12) + [13] # joint ids for SMPL # SMPL does not have a joint for head, instead we use a vertex for the head # and append it later. smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # the vertex id for the joint corresponding to the head head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array([1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) # initialize the shape to the mean shape in the SMPL training set betas = ch.array(init_shape) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([ model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas)) ]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation(sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint, corresponding to a vertex... Jtr = ch.vstack((Jtr, sv[head_id])) # ... and add the joint id to the list smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[cids] if conf is not None else base_weights # project SMPL joints on the image plane using the estimated camera cam.v = Jtr # data term: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: (w * weights.reshape((-1, 1)) * GMOf( (j2d[cids] - cam[smpl_ids]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=None, options={ 'maxiter': 100, 'e_3': .0001, 'disp': 0 }) return sv, cam.r, cam.t.r
def main(args): """Set up paths to image and joint data, saves results. #Fit SMPL to a target 3D skeleton """ modelpath = (abspath(dirname(__file__))) + '/smplify/models/' # Note that rendering many views can take a while. model = load_model(modelpath + args.model) n_betas = args.n_betas fig = plt.figure(figsize=(15, 15)) ax = fig.add_subplot(111, projection='3d') out_dir = args.out_dir out_dir = None for idx in range(args.max_num): joint_file = 'body3DScene_{:08d}.json'.format( (idx + args.start_number) * 3) SMPL_file = 'bodySMPL_{:08d}.pkl'.format(idx + args.start_number) DP_file = 'DensePose_{:08d}.mat'.format(idx + args.start_number) #create visualization ax.clear() ax.set_aspect('equal') #load DensePose data #ax.set_xlim((x_max+x_min)/2.0 - plot_radius,(x_max+x_min)/2.0+plot_radius) #ax.set_ylim((y_max+y_min)/2.0 - plot_radius,(y_max+y_min)/2.0+plot_radius) #ax.set_zlim((z_max+z_min)/2.0 - plot_radius,(z_max+z_min)/2.0+plot_radius) DP_mat = sio.loadmat(join(args.DP, DP_file)) DP_Points = DP_mat['cPoints'] DP_Err = DP_mat['cErr'] DP_valid = DP_Points[:, 3] > 0 ax.scatter(DP_Points[DP_valid, 0] / 100., DP_Points[DP_valid, 1] / 100., DP_Points[DP_valid, 2] / 100., s=2, c=DP_Err[DP_valid, 0]) #Load SMPL and Joints if (os.path.isfile(join(args.joints, joint_file))): with open(join(args.joints, joint_file)) as f: rawData = json.load(f) Joint15 = rawData["bodies"][0]["joints15"] Joint15 = np.reshape(Joint15, (15, 4)) joints = Joint15[dome2lsp, 0:3] / 100. plot_pose3d(joints, ax) #load SMPl with open(join(args.SMPL, SMPL_file), 'rb') as f: rawSMPL = pickle.load(f) sv = verts_decorated(trans=ch.array(rawSMPL['trans']), pose=ch.array(rawSMPL['pose']), v_template=model.v_template, J=model.J_regressor, betas=ch.array(rawSMPL['betas']), shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # Visualize the SMPL vertices ax.scatter(sv.r[::2, 0], sv.r[::2, 1], sv.r[::2, 2], s=0.1, c='k') #plot skeleton ax.set_xlim(-1.5, 1.5) ax.set_ylim(-3, 0) ax.set_zlim(-1.5, 1.5) ax.view_init(elev=110., azim=90.) if out_dir is not None: figureName = 'Figure_{:08d}.png'.format(idx + args.start_number) plt.savefig(join(out_dir, figureName), bbox_inches='tight') else: plt.show() raw_input('Press any key to continue...')
def optimize_on_DensePose(dp_iuv, op_j2d, w_j2d, model, cam, prior, body_orient, body_trans, body_init, n_betas=10, viz=False, imageid=1): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ outPath = '/home/xiul/databag/dbfusion/record0/smplify/vid{}'.format( imageid) if not os.path.isdir(outPath): os.mkdir(outPath) t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: tarIUV = ch.array(dp_iuv) # initialize the shape to the mean shape in the SMPL training set betas = ch.zeros(n_betas) # initialize the pose by using the optimized body orientation and the # pose prior init_pose = np.hstack((body_orient, prior.weights.dot(prior.means))) #init_pose = np.hstack((body_orient, body_init)) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.array(body_trans), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) J_tmpx = MatVecMult(J_reg, sv[:, 0]) J_tmpy = MatVecMult(J_reg, sv[:, 1]) J_tmpz = MatVecMult(J_reg, sv[:, 2]) Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T #Jtr = J_reg.dot(sv) cam.v = Jtr op_j2d = op_j2d * 1280 / 1920 w = 1280 h = 720 reIUV = render_model(sv, model.f, w, h, cam, near=0.5, far=25, vc=dp_colors, img=None) #gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None): obj_j2d = lambda w, sigma: (w * w_j2d.reshape((-1, 1)) * GMOf( (op_j2d - cam), sigma)) #input_objective, imshape, normalization, n_levels, as_list err = tarIUV - reIUV obj_dense = 100 * gaussian_pyramid(err, n_levels=6, normalization='SSE') # data term: distance between observed and estimated joints in 2D # obj_dense = lambda w, sigma: ( # w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) if viz: import matplotlib.pyplot as plt from time import gmtime, strftime def on_step(cstep): """Create visualization.""" plt.figure(1, figsize=(10, 10)) plt.clf() plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = reIUV.r.copy() tmp_tar = tarIUV.copy() plt.imshow(tmp_img) for j1, j2 in zip(cam.r, op_j2d): plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r') plt.subplot(1, 2, 2) plt.imshow(tmp_tar) plt.draw() plt.savefig(os.path.join( outPath, '{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))), bbox_inches='tight') on_step(stepI) else: on_step = None # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78, 4.78, 4.78, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1, 5, 5, 5]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} if stage >= 4: objs['dense'] = obj_dense if stage <= 4: objs['j2d'] = obj_j2d(1, 100) else: objs['j2d'] = obj_j2d(0.1, 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={ 'maxiter': 10000, 'e_3': .001, 'disp': 1 }) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if viz: plt.ioff()
def test_interplote(): import cv2 pkl_file = '/home/xiul/databag/net_images/hmr/kobe.pkl' im = cv2.imread('/home/xiul/databag/net_images/images/kobe.jpg') data_all = parse_hmr_data(pkl_file) #only use 1:4 MODEL_DIR = join(abspath(dirname(__file__)), '../../models') # Model paths: MODEL_NEUTRAL_PATH = join(MODEL_DIR, 'basicModel_neutral_lbs_10_207_0_v1.0.0.pkl') MODEL_FEMALE_PATH = join(MODEL_DIR, 'basicModel_f_lbs_10_207_0_v1.0.0.pkl') MODEL_MALE_PATH = join(MODEL_DIR, 'basicmodel_m_lbs_10_207_0_v1.0.0.pkl') model = load_model(MODEL_MALE_PATH) sv = [] f = [] v_num = model.v_template.r.shape[0] fx = im.shape[1] fy = im.shape[1] rt = ch.zeros(3) t = ch.zeros(3) cx = im.shape[1] / 2 cy = im.shape[0] / 2 data_all = data_all[1:5] alpha_val = np.linspace(0, 1, num=10) cam = ProjectPoints(f=np.array([fx, fy]), rt=rt, t=t, k=np.zeros(5), c=[cx, cy]) for alpha in alpha_val: c_pose = data_all[2]['pose'] * alpha + data_all[3]['pose'] * (1 - alpha) c_pose[:3] = 0 c_pose[0] = np.pi c_sv = verts_decorated(trans=ch.array(data_all[1]['trans']), pose=ch.array(c_pose), v_template=model.v_template, J=model.J_regressor, betas=ch.array(data_all[1]['betas']), shapedirs=model.shapedirs[:, :, :10], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) #fullModel = render_model(sv, model.f, ori_img.shape[1], ori_img.shape[0], # cam_old, near=0.5, far=25, vc=None, img=None) im_render = render_model(c_sv, model.f, im.shape[1], im.shape[0], cam, near=0.5, far=25, vc=None, img=None) im_render = im_render.r * 255 im_render = im_render.astype(np.uint8) im_out = model_im_overlay(im_render, im) print im_out.shape print im_out.dtype cv2.imshow('im', im_out) cv2.waitKey(-1)
def main(): import cv2 pkl_file = '/home/xiul/databag/net_images/hmr/kobe.pkl' im = cv2.imread('/home/xiul/databag/net_images/images/kobe.jpg') data_all = parse_hmr_data(pkl_file) #only use 1:4 MODEL_DIR = join(abspath(dirname(__file__)), '../../models') # Model paths: MODEL_NEUTRAL_PATH = join(MODEL_DIR, 'basicModel_neutral_lbs_10_207_0_v1.0.0.pkl') MODEL_FEMALE_PATH = join(MODEL_DIR, 'basicModel_f_lbs_10_207_0_v1.0.0.pkl') MODEL_MALE_PATH = join(MODEL_DIR, 'basicmodel_m_lbs_10_207_0_v1.0.0.pkl') model = load_model(MODEL_MALE_PATH) sv = [] f = [] v_num = model.v_template.r.shape[0] fx = im.shape[1] fy = im.shape[1] rt = ch.zeros(3) t = ch.zeros(3) cx = im.shape[1] / 2 cy = im.shape[0] / 2 data_all = data_all[1:5] for idx, c_data_old in enumerate(data_all): c_data = normalize_cam_single(c_data_old['cam'], c_data_old, cx, cy, fx) c_sv = verts_decorated(trans=ch.array(c_data['trans']), pose=ch.array(c_data['pose']), v_template=model.v_template, J=model.J_regressor, betas=ch.array(c_data['betas']), shapedirs=model.shapedirs[:, :, :10], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) sv.append(c_sv.r.copy()) c_f = model.f + v_num * idx f.append(c_f.copy()) sv = np.array(sv) f = np.array(f) sv_vec = np.reshape(sv, (-1, 3)) f_vec = np.reshape(f, (-1, 3)) # initialize the camera cam = ProjectPoints(f=np.array([fx, fy]), rt=rt, t=t, k=np.zeros(5), c=[cx, cy]) #fullModel = render_model(sv, model.f, ori_img.shape[1], ori_img.shape[0], # cam_old, near=0.5, far=25, vc=None, img=None) im_render = render_model(sv_vec, f_vec, im.shape[1], im.shape[0], cam, near=0.5, far=25, vc=None, img=None) im_render = im_render.r * 255 im_render = im_render.astype(np.uint8) im_out = model_im_overlay(im_render, im) print im_out.shape print im_out.dtype cv2.imshow('im', im_out) cv2.waitKey(-1)
def vis_param(): import cv2 import matplotlib.pyplot as plt hsv_map = plt.get_cmap('hsv') from os.path import join, abspath, dirname seq = 'run' ori_pkl = '/home/xiul/databag/net_images/hmr/{}.pkl'.format(seq) im = cv2.imread('/home/xiul/databag/net_images/images/{}.jpg'.format(seq)) data_all = parse_hmr_data(ori_pkl) #only use 1:4 MODEL_DIR = join(abspath(dirname(__file__)), '../../models') # Model paths: MODEL_NEUTRAL_PATH = join(MODEL_DIR, 'basicModel_neutral_lbs_10_207_0_v1.0.0.pkl') MODEL_FEMALE_PATH = join(MODEL_DIR, 'basicModel_f_lbs_10_207_0_v1.0.0.pkl') MODEL_MALE_PATH = join(MODEL_DIR, 'basicmodel_m_lbs_10_207_0_v1.0.0.pkl') model = load_model(MODEL_MALE_PATH) sv = [] f_vec = [] v_num = model.v_template.r.shape[0] rt = ch.zeros(3) t = ch.zeros(3) cx = im.shape[1] / 2 cy = im.shape[0] / 2 f = 2000 for idx, c_data_old in enumerate(data_all): opt_param = normalize_cam_single(c_data_old, cx, cy, f) c_sv = verts_decorated(trans=ch.array(opt_param['trans']), pose=ch.array(opt_param['pose']), v_template=model.v_template, J=model.J_regressor, betas=ch.array(opt_param['betas']), shapedirs=model.shapedirs[:, :, :10], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) sv.append(copy.deepcopy(c_sv)) cf = model.f + idx * v_num f_vec.append(copy.deepcopy(cf)) print idx # initialize the camera print 'model_loaded' cam = ProjectPoints(f=np.array([f, f]), rt=rt, t=t, k=np.zeros(5), c=[cx, cy]) sv = reduce(lambda x, y: np.concatenate((x, y)), sv) f_vec = reduce(lambda x, y: np.concatenate((x, y)), f_vec) import scipy.io smpldpPath = '/home/xiul/workspace/PanopticDome/models/SMPL_DP.mat' smpldpFile = scipy.io.loadmat(smpldpPath) smpldp = smpldpFile['SMPL_IUV'] dp_colors = smpldp[:, [2, 1, 3]] dp_colors = dp_colors dp_colors[:, 2] = dp_colors[:, 2] / 255.0 reg_colors = np.ones(dp_colors.shape, dp_colors.dtype) with open( '/home/xiul/workspace/PanopticDome/models/neutral_smpl_with_cocoplus_reg.pkl' ) as fio: cPkl = pickle.load(fio) J_reg = model.J_regressor #J_reg = cPkl['cocoplus_regressor'] J_reg_vts = [] for reg_row in J_reg: J_reg_vts.append(scipy.sparse.coo_matrix(reg_row).col.tolist()) import matplotlib.pyplot as plt c_map = plt.get_cmap('hsv') for J_id, J_vts in enumerate(J_reg_vts): for cj in J_vts: reg_colors[cj, :] = c_map(J_id * 1.0 / len(J_reg_vts))[0:3] supp_vc = np.zeros(sv.shape) all_num = len(data_all) for cidx in range(all_num): c_color = hsv_map(cidx) supp_vc[cidx * v_num:cidx * v_num + v_num, :] = c_color[:3] #supp_vc[cidx*v_num:cidx*v_num+v_num,:] = (cidx+8)*1.0/(all_num+8) #supp_vc[cidx*v_num:cidx*v_num+v_num,:] = dp_colors #supp_vc[cidx*v_num:cidx*v_num+v_num,:] = reg_colors #f_vec = np.reshape(f_vec,(-1,3)) #fullModel = render_model(sv, model.f, ori_img.shape[1], ori_img.shape[0], # cam_old, near=0.5, far=25, vc=None, img=None) print 'start_render' im_render = render_model(sv, f_vec, im.shape[1], im.shape[0], cam, near=0.5, far=25, vc=supp_vc, img=None, lighting=True) im_render = im_render.r * 255 im_render = im_render.astype(np.uint8) im_bg = np.zeros(im_render.shape, im_render.dtype) * 255 im_out = model_im_overlay(im_render, im_bg) cv2.imshow('im', im_out) cv2.waitKey(-1)
def genMeshInstance(self, meshType): self.render_lock.lock() if len(self.smpl_Lists) > 0: cfile = self.smpl_Lists[self.frame_id] with open(cfile) as f: SMPLParams = pickle.load(f) if not type(SMPLParams) == list: smplParam = SMPLParams SMPLParams = [] SMPLParams.append(smplParam) self.vts_buffer = [] self.vns_buffer = [] self.inds_buffer = [] self.face_num = [] for smplParam in SMPLParams: if meshType == 0: #SMPL sv = verts_decorated( trans=chumpy.array(smplParam['trans']), pose=chumpy.array(smplParam['pose']), v_template=self.smpl.v_template, J=self.smpl.J_regressor, betas=chumpy.array(smplParam['betas']), shapedirs=self.smpl.shapedirs, weights=self.smpl.weights, kintree_table=self.smpl.kintree_table, bs_style=self.smpl.bs_style, f=self.smpl.f, bs_type=self.smpl.bs_type, posedirs=self.smpl.posedirs) SMPL_vts = sv.r * 100.0 SMPL_inds = sv.f vts_num = SMPL_vts.shape[0] SMPL_vns = numpy.zeros((vts_num, 3)) U = SMPL_vts[SMPL_inds[:, 1]] - SMPL_vts[SMPL_inds[:, 0]] V = SMPL_vts[SMPL_inds[:, 2]] - SMPL_vts[SMPL_inds[:, 0]] Nor = numpy.cross(U, V) Nor = sklearn.preprocessing.normalize(Nor) SMPL_vns[SMPL_inds[:, 0]] += Nor SMPL_vns[SMPL_inds[:, 1]] += Nor SMPL_vns[SMPL_inds[:, 2]] += Nor SMPL_vns = sklearn.preprocessing.normalize(SMPL_vns) vns = SMPL_vns.flatten() vts = SMPL_vts.flatten() inds = SMPL_inds.flatten() else: pose = 0 - smplParam['pose'] vts, vns, inds = self.meshlib.adam_vertsdecorate( pose, smplParam['betas'], smplParam['trans'], smplParam['faces'], 2) vns = vns.flatten() vts = vts.flatten() inds = inds.flatten() vn_buffer = glGenBuffers(1) vts_buffer = glGenBuffers(1) inds_buffer = glGenBuffers(1) face_num = len(inds) / 3 glBindBuffer(GL_ARRAY_BUFFER, vn_buffer) glBufferData(GL_ARRAY_BUFFER, len(vns) * sizeof(ctypes.c_float), (ctypes.c_float * len(vns))(*vns), GL_STATIC_DRAW) glBindBuffer(GL_ARRAY_BUFFER, vts_buffer) glBufferData(GL_ARRAY_BUFFER, len(vts) * sizeof(ctypes.c_float), (ctypes.c_float * len(vts))(*vts), GL_STATIC_DRAW) glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, inds_buffer) glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof(ctypes.c_uint) * len(inds), (ctypes.c_uint * len(inds))(*inds), GL_STATIC_DRAW) self.vns_buffer.append(vn_buffer) self.vts_buffer.append(vts_buffer) self.inds_buffer.append(inds_buffer) self.face_num.append(face_num) self.render_lock.unlock()
def optimize_on_joints(j2d, model, cam, img, prior, try_both_orient, body_orient, n_betas=10, regs=None, conf=None, viz=False): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: cids = range(12) + [13] # joint ids for SMPL # SMPL does not have a joint for head, instead we use a vertex for the head # and append it later. smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20] # the vertex id for the joint corresponding to the head head_id = 411 # weights assigned to each joint during optimization; # the definition of hips in SMPL and LSP is significantly different so set # their weights to zero base_weights = np.array( [1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64) if try_both_orient: flipped_orient = cv2.Rodrigues(body_orient)[0].dot( cv2.Rodrigues(np.array([0., np.pi, 0]))[0]) flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel() orientations = [body_orient, flipped_orient] else: orientations = [body_orient] if try_both_orient: # store here the final error for both orientations, # and pick the orientation resulting in the lowest error errors = [] svs = [] cams = [] for o_id, orient in enumerate(orientations): # initialize the shape to the mean shape in the SMPL training set betas = ch.zeros(n_betas) # initialize the pose by using the optimized body orientation and the # pose prior init_pose = np.hstack((orient, prior.weights.dot(prior.means))) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated( trans=ch.zeros(3), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) # make the SMPL joints depend on betas Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i]) for i in range(len(betas))]) J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot( model.v_template.r) # get joint positions as a function of model pose, betas and trans (_, A_global) = global_rigid_transformation( sv.pose, J_onbetas, model.kintree_table, xp=ch) Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans # add the head joint, corresponding to a vertex... Jtr = ch.vstack((Jtr, sv[head_id])) # ... and add the joint id to the list if o_id == 0: smpl_ids.append(len(Jtr) - 1) # update the weights using confidence values weights = base_weights * conf[ cids] if conf is not None else base_weights # project SMPL joints on the image plane using the estimated camera cam.v = Jtr # data term: distance between observed and estimated joints in 2D obj_j2d = lambda w, sigma: ( w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[ 58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])]) if viz: import matplotlib.pyplot as plt plt.ion() def on_step(_): """Create visualization.""" plt.figure(1, figsize=(10, 10)) plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = img.copy() for coord, target_coord in zip( np.around(cam.r[smpl_ids]).astype(int), np.around(j2d[cids]).astype(int)): if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and coord[1] < tmp_img.shape[0] and coord[1] >= 0): cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255]) if (target_coord[0] < tmp_img.shape[1] and target_coord[0] >= 0 and target_coord[1] < tmp_img.shape[0] and target_coord[1] >= 0): cv2.circle(tmp_img, tuple(target_coord), 3, [0, 255, 0]) plt.imshow(tmp_img[:, :, ::-1]) plt.draw() plt.show() plt.pause(1e-2) on_step(_) else: on_step = None if regs is not None: # interpenetration term sp = SphereCollisions( pose=sv.pose, betas=sv.betas, model=model, regs=regs) sp.no_hands = True # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} objs['j2d'] = obj_j2d(1., 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas if regs is not None: objs['sph_coll'] = 1e3 * sp ch.minimize( objs, x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={'maxiter': 100, 'e_3': .0001, 'disp': 0}) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if try_both_orient: errors.append((objs['j2d'].r**2).sum()) svs.append(sv) cams.append(cam) if try_both_orient and errors[0] > errors[1]: choose_id = 1 else: choose_id = 0 if viz: plt.ioff() return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)