def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False): options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]} options['disp'] = 1.0 options['delta_0'] = 0.1 options['e_3'] = 1e-4 s = ch.ones(1) r = ch.zeros(3) R = Rodrigues(r) t = ch.zeros(3) trafo_mesh = s*(R.dot(mesh.v.T)).T + t sampler = sample_from_mesh(scan, sample_type='vertices') s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False) if visualize: #Visualization code mv = MeshViewer() mv.set_static_meshes([scan]) tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) def on_show(_): tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) else: def on_show(_): pass ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options) return s,Rodrigues(r),t
def __init__(self, t, rod, rad, length): self.t = t # translation of the axis self.rod = rod # rotation of the axis in Rodrigues form self.rad = rad # radious of the capsule self.length = length # length of the axis axis0 = ch.vstack([0, ch.abs(self.length), 0]) self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T)) v0 = ch.hstack([v[:26].T*rad, (v[26:].T*rad)+ axis0]) self.v = ((t + Rodrigues(rod).dot(v0)).T) self.set_sphere_centers()
def __init__(self, t, rod, rad, length): assert (hasattr(t, 'dterms')) # the translation should be a chumpy object (differentiable wrt shape) self.t = t # translation of the axis self.rod = rod # rotation of the axis in Rodrigues form # the radius should be a chumpy object (differentiable wrt shape) assert (hasattr(rad, 'dterms')) self.rad = rad # radius of the capsule # the length should be a chumpy object (differentiable wrt shape) assert (hasattr(length, 'dterms')) self.length = length # length of the axis axis0 = ch.vstack([0, ch.abs(self.length), 0]) self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T)) v0 = ch.hstack([v[:26].T * rad, (v[26:].T * rad) + axis0]) self.v = ((t + Rodrigues(rod).dot(v0)).T) self.set_sphere_centers()
def volume(v, f): # Construct a 3D matrix which is of size (nfaces x 3 x 3) # Each row corresponds to a face; the third dimension indicates # which triangle in that face is being referred to vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :])) v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0] v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0] v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1] v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1] v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2] v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2] volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1.0 / 6.0) return ch.abs(ch.sum(volumes))
def volume(v, f): # Construct a 3D matrix which is of size (nfaces x 3 x 3) # Each row corresponds to a face; the third dimension indicates # which triangle in that face is being referred to vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :])) v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0] v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0] v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1] v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1] v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2] v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2] volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1. / 6.) return ch.abs(ch.sum(volumes))
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None): from opendr.geometry import Rodrigues if length_regs is not None: n_shape_dofs = length_regs.shape[0] - 1 else: n_shape_dofs = model.betas.r.size segm = np.argmax(model.weights_prior, axis=1) J_off = ch.zeros((len(joint2name), 3)) rots = rots0.copy() mujoco_t_mid = [0, 3, 6, 9] if wrt_betas is not None: # if we want to differentiate wrt betas (shape), we must have the # regressors... assert (length_regs is not None and rad_regs is not None) # ... and betas must be a chumpy object assert (hasattr(wrt_betas, 'dterms')) pad = ch.concatenate( (wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1))) lengths = pad.dot(length_regs) rads = pad.dot(rad_regs) else: lengths = ch.ones(len(joint2name)) rads = ch.ones(len(joint2name)) betas = wrt_betas if wrt_betas is not None else model.betas n_betas = len(betas) # the joint regressors are the original, pre-optimized ones # (middle of the part frontier) myJ_regressor = model.J_regressor_prior myJ0 = ch.vstack( (ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 0] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]), ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 1] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]), ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 2] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T # with small adjustments for hips, spine and feet myJ = ch.vstack( [ch.concatenate([myJ0[0, 0], ( .6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2]]), ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate( [myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]), ch.concatenate( [myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]), ch.concatenate( [myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]), ch.vstack([myJ0[i] for i in range(10, 24)])]) capsules = [] # create one capsule per mujoco joint for ijoint, segms in enumerate(mujoco2segm): if wrt_betas is None: vidxs = np.asarray([segm == k for k in segms]).any(axis=0) verts = model.v_template[vidxs].r dims = (verts.max(axis=0) - verts.min(axis=0)) rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[( np.argmax(dims) + 2) % 3]) / 4.) lengths[ijoint] = max(dims) - 2. * rads[ijoint].r # the core joints are different, since the capsule is not in the joint # but in the middle if ijoint in mujoco_t_mid: len_offset = ch.vstack([ch.zeros(1), ch.abs(lengths[ijoint]) / 2., ch.zeros(1)]).reshape(3, 1) caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape( 3, 1) - Rodrigues(rots[ijoint]).dot(len_offset), rots[ijoint], rads[ijoint], lengths[ijoint]) else: caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1), rots[ijoint], rads[ijoint], lengths[ijoint]) caps.id = ijoint capsules.append(caps) return capsules
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None): from opendr.geometry import Rodrigues if length_regs is not None: n_shape_dofs = length_regs.shape[0] - 1 else: n_shape_dofs = model.betas.r.size segm = np.argmax(model.weights_prior, axis=1) J_off = ch.zeros((len(joint2name), 3)) rots = rots0.copy() mujoco_t_mid = [0, 3, 6, 9] if wrt_betas is not None: # if we want to differentiate wrt betas (shape), we must have the # regressors... assert (length_regs is not None and rad_regs is not None) # ... and betas must be a chumpy object assert (hasattr(wrt_betas, 'dterms')) pad = ch.concatenate( (wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1))) lengths = pad.dot(length_regs) rads = pad.dot(rad_regs) else: lengths = ch.ones(len(joint2name)) rads = ch.ones(len(joint2name)) betas = wrt_betas if wrt_betas is not None else model.betas n_betas = len(betas) # the joint regressors are the original, pre-optimized ones # (middle of the part frontier) myJ_regressor = model.J_regressor_prior myJ0 = ch.vstack((ch.ch.MatVecMult( myJ_regressor, model.v_template[:, 0] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]), ch.ch.MatVecMult( myJ_regressor, model.v_template[:, 1] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]), ch.ch.MatVecMult( myJ_regressor, model.v_template[:, 2] + model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T # with small adjustments for hips, spine and feet myJ = ch.vstack([ ch.concatenate([ myJ0[0, 0], (.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2] ]), ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate( [myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]), ch.concatenate( [myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]), ch.concatenate( [myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]), ch.vstack([myJ0[i] for i in range(10, 24)]) ]) capsules = [] # create one capsule per mujoco joint for ijoint, segms in enumerate(mujoco2segm): if wrt_betas is None: vidxs = np.asarray([segm == k for k in segms]).any(axis=0) verts = model.v_template[vidxs].r dims = (verts.max(axis=0) - verts.min(axis=0)) rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[ (np.argmax(dims) + 2) % 3]) / 4.) lengths[ijoint] = max(dims) - 2. * rads[ijoint].r # the core joints are different, since the capsule is not in the joint # but in the middle if ijoint in mujoco_t_mid: len_offset = ch.vstack( [ch.zeros(1), ch.abs(lengths[ijoint]) / 2., ch.zeros(1)]).reshape(3, 1) caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1) - Rodrigues(rots[ijoint]).dot(len_offset), rots[ijoint], rads[ijoint], lengths[ijoint]) else: caps = Capsule( (J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1), rots[ijoint], rads[ijoint], lengths[ijoint]) caps.id = ijoint capsules.append(caps) return capsules
def diffHog(image, drconv=None, numOrient=9, cwidth=8, cheight=8): imagegray = 0.3 * image[:, :, 0] + 0.59 * image[:, :, 1] + 0.11 * image[:, :, 2] sy, sx = imagegray.shape # gx = ch.empty(imagegray.shape, dtype=np.double) gx = imagegray[:, 2:] - imagegray[:, :-2] gx = ch.hstack([np.zeros([sy, 1]), gx, np.zeros([sy, 1])]) gy = imagegray[2:, :] - imagegray[:-2, :] # gy = imagegray[:, 2:] - imagegray[:, :-2] gy = ch.vstack([np.zeros([1, sx]), gy, np.zeros([1, sx])]) gx += 1e-5 # gy = imagegray[:-2,1:-1] - imagegray[2:,1:-1] + 0.00001 # gx = imagegray[1:-1,:-2] - imagegray[1:-1, 2:] + 0.00001 distFilter = np.ones([2 * cheight, 2 * cwidth], dtype=np.uint8) distFilter[np.int(2 * cheight / 2), np.int(2 * cwidth / 2)] = 0 distFilter = (cv2.distanceTransform(distFilter, cv2.DIST_L2, 3) - np.max( cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) / ( -np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) magn = ch.sqrt(gy**2 + gx**2) * 180 / np.sqrt(2) angles = ch.arctan(gy / gx) * 180 / np.pi + 90 # meanOrient = np.linspace(0, 180, numOrient) orientations_arr = np.arange(numOrient) meanOrient = orientations_arr / numOrient * 180 fb_resttmp = 1 - ch.abs( ch.expand_dims(angles[:, :], 2) - meanOrient[1:].reshape([1, 1, numOrient - 1])) * numOrient / 180 zeros_rest = np.zeros([sy, sx, numOrient - 1, 1]) fb_rest = ch.max(ch.concatenate([fb_resttmp[:, :, :, None], zeros_rest], axis=3), axis=3) chMinOrient0 = ch.min(ch.concatenate([ ch.abs( ch.expand_dims(angles[:, :], 2) - meanOrient[0].reshape([1, 1, 1]))[:, :, :, None], ch.abs(180 - ch.expand_dims(angles[:, :], 2) - meanOrient[0].reshape([1, 1, 1]))[:, :, :, None] ], axis=3), axis=3) zeros_fb0 = np.zeros([sy, sx, 1]) fb0_tmp = ch.concatenate( [1 - chMinOrient0[:, :] * numOrient / 180, zeros_fb0], axis=2) fb_0 = ch.max(fb0_tmp, axis=2) fb = ch.concatenate([fb_0[:, :, None], fb_rest], axis=2) # fb[:,:,0] = ch.max(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180,0) # fb = 1./(1. + ch.exp(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180)) Fb = ch.expand_dims(magn, 2) * fb if drconv is None: drconv = dr_wrt_convolution(Fb[:, :, 0], distFilter) Fs_list = [ convolve2D(x=Fb[:, :, Fbi], filter=distFilter, convolve2DDr=drconv).reshape([Fb.shape[0], Fb.shape[1], 1]) for Fbi in range(numOrient) ] # Fs_list = [scipy.signal.convolve2d(Fb[:,:,Fbi], distFilter).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)] Fs = ch.concatenate(Fs_list, axis=2) # cellCols = np.arange(start=cwidth/2, stop=Fs.shape[1]-cwidth/2 , step=cwidth) # cellRows = np.arange(start=cheight/2, stop=Fs.shape[0]-cheight/2 , step=cheight) Fcells = Fs[0:Fs.shape[0]:cheight, 0:Fs.shape[1]:cwidth, :] epsilon = 1e-5 v = Fcells / ch.sqrt(ch.sum(Fcells**2) + epsilon) # v = Fcells # hog, hogim = skimage.feature.hog(imagegray, orientations=numOrient, pixels_per_cell=(cheight, cwidth), visualise=True) hog_image = HogImage(image=image, hog=Fcells, numOrient=numOrient, cwidth=cwidth, cheight=cheight) # plt.imshow(hog_image) # plt.figure() # plt.imshow(hogim) # ipdb.set_trace() return v, hog_image, drconv
c_pre={}; if (W_Joints): k=a['Joints_Target']; j_to_consider = range(0,24) J_distances = J_reposed[j_to_consider,:] - (k[j_to_consider,:]+trans); c_pre['Joints']= J_distances*W_Joints; if(W_FMP2P): c_pre['FMP2P']= distances*W_FMP2P; if(W_Norm_B): c_pre['Norm_B']= ((result.betas)**2)*W_Norm_B; if(W_Norm_T): pose_res=result.pose.reshape(-1,3); angles=ch.sum(ch.abs(pose_res)**2,axis=-1)**(1./2) pesi = np.ones(24)*8/18 pesi[[0]] = np.ones(1)*[2] pesi[[10, 11, 22, 23, 15]] = np.ones(5)*[2./18] pesi[[6,3, 7,8]] = np.ones(4)*[5./18] pesi[[12, 15]] = np.ones(2)*[1./36] costo_T= (angles/(ch.pi*pesi))**12 c_pre['Norm_T']=costo_T*W_Norm_T; if(W_Landmarks): Tar_Land = Tar_shift[a['landmarks1'].reshape(5)-1]; SMPL_Land=result[a['landmarks2'].reshape(5)-1]; c_pre['Landmarks']= (SMPL_Land-Tar_Land)*W_Landmarks; (r,b,t)=ch.minimize(c_pre, x0 = [result.pose, result.betas,trans], method = 'dogleg', callback = None,
def set_pose_objs(sv, cam, landmarks, key_vids, animal=None, shape_data_name=None, nbetas=0, kp_weights=None, fix_rot=False, SOLVE_FLATER=True, FIX_CAM=False, ONLY_KEYP=False, OPT_SHAPE=True, DO_FREE_SHAPE=False): nCameras = len(cam) nClips = nCameras params = set_params(SOLVE_FLATER, nCameras) pose_prior = [set_pose_prior(len(sv[0].pose.r)) for _ in range(nClips)] pose_prior_tail = [ set_pose_prior_tail(len(sv[0].pose.r)) for _ in range(nClips) ] if OPT_SHAPE: shape_prior = [ set_shape_prior(DO_FREE_SHAPE, animal, shape_data_name) for _ in range(nClips) ] # indices with no prior noprior_ind = ~pose_prior[0].use_ind noprior_ind[:3] = False limit_prior = [set_limit_prior(len(sv[0].pose.r)) for _ in range(nClips)] init_rot = [sv[ic].pose[:3].r.copy() for ic in range(nClips)] init_trans = [sv[ic].trans.r.copy() for ic in range(nClips)] init_pose = [sv[ic].pose.r.copy() for ic in range(nClips)] # Setup keypoint projection error with multi verts: j2d = [None] * nCameras assignments = [None] * nCameras num_points = [None] * nCameras use_ids = [None] * nCameras visible_vids = [None] * nCameras all_vids = [None] * nCameras for i in range(nCameras): visible = landmarks[i][:, 2].astype(bool) use_ids[i] = [ id for id in np.arange(landmarks[i].shape[0]) if visible[id] ] visible_vids[i] = np.hstack([key_vids[i][id] for id in use_ids[i]]) group = np.hstack([ index * np.ones(len(key_vids[i][row_id])) for index, row_id in enumerate(use_ids[i]) ]) assignments[i] = np.vstack( [group == j for j in np.arange(group[-1] + 1)]) num_points[i] = len(use_ids[i]) all_vids[i] = visible_vids[i] cam[i].v = sv[i][all_vids[i], :] j2d[i] = landmarks[i][use_ids[i], :2] if kp_weights is None: kp_weights = np.ones((landmarks[i].shape[0], 1)) def kp_proj_error(i, w, sigma): return w * kp_weights[use_ids[i]] * GMOf( ch.vstack([ cam[i][choice] if np.sum(choice) == 1 else cam[i][choice].mean( axis=0) for choice in assignments[i] ]) - j2d[i], sigma) / np.sqrt(num_points[i]) objs = {} for i in range(nCameras): objs['kp_proj_' + str(i)] = kp_proj_error(i, params['k_kp_term'], params['k_robust_sig']) if not ONLY_KEYP: objs['trans_init_' + str(i)] = params['k_trans_term'] * (sv[i].trans - init_trans[i]) if not ONLY_KEYP: if fix_rot: objs['fix_rot_' + str(i)] = params['k_rot_term'] * (sv[i].pose[:3] - init_rot[i]) if OPT_SHAPE: if (i > 0): objs['betas_var_' + str(i)] = params['betas_var'] * ch.abs(sv[i - 1].betas - sv[i].betas) objs['shape_prior_' + str(i)] = shape_prior[i](sv[i].betas) / np.sqrt(nbetas) if not FIX_CAM: for i in range(nCameras): objs['feq_' + str(i)] = 1e3 * (cam[i].f[0] - cam[i].f[1]) objs['fpos_' + str(i)] = 1e3 * ch.maximum(0, 500 - cam[i].f[0]) if not SOLVE_FLATER: for i in range(nCameras): objs['freg_' + str(i)] = 9 * 1e2 * (cam[i].f[0] - 3000) / 1000. objs['cam_t_pos_' + str(i)] = 1e3 * ch.maximum(0, 0.01 - sv[i].trans[2]) del objs['trans_init_' + str(i)] num_pose_prior = len(pose_prior[0](sv[0].pose)) num_limit_prior = len(limit_prior[0](sv[0].pose)) if not ONLY_KEYP: if np.sum(noprior_ind) > 0: objs['rest_poseprior_' + str(i)] = params['k_rest_pose_term'] * ( sv[i].pose[noprior_ind] - init_pose[noprior_ind]) / np.sqrt( len(sv[i].pose[noprior_ind])) for i in range(nClips): objs['pose_limit_' + str(i)] = params['k_limit_term'] * limit_prior[i]( sv[i].pose) / np.sqrt(num_limit_prior) objs['pose_prior_' + str(i)] = pose_prior[i](sv[i].pose) / np.sqrt(num_pose_prior) objs['pose_prior_tail_' + str(i)] = 2.0 * pose_prior_tail[i]( sv[i].pose) / np.sqrt(num_pose_prior) return objs, params, j2d
def optimize_smal(fposes, ftrans, fbetas, model, cams, segs, imgs, landmarks, landmarks_names, key_vids, symIdx=None, frameId=0, opt_model_dir=None, save_name=None, COMPUTE_OPT=True, img_paths=None, img_offset=None, img_scales=None): mesh_v_opt_save_path = join(opt_model_dir, 'mesh_v_opt_no_mc_' + str(frameId) + '.ply') mesh_v_opt_mc_save_path = join(opt_model_dir, 'mesh_v_opt_' + str(frameId) + '.ply') mesh_init_save_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') nViews = len(fposes) if not COMPUTE_OPT: if not exists(opt_model_dir): makedirs(opt_model_dir) dv = 0 compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_init_save_path, fposes, ftrans, fbetas, '_no_refine', cams, imgs, segs, img_paths, img_offset, img_scales) return # Write the initial mesh np_betas = np.zeros_like(model.betas) np_betas[:len(fbetas[0])] = fbetas[0] tmp = verts_decorated(v_template=model.v_template, pose=ch.zeros_like(model.pose.r), trans=ch.zeros_like(model.trans), J=model.J_regressor, kintree_table=model.kintree_table, betas=ch.array(np_betas), weights=model.weights, posedirs=model.posedirs, shapedirs=model.shapedirs, bs_type='lrotmin', bs_style='lbs', f=model.f) tmp_mesh = Mesh(v=tmp.r, f=tmp.f) tmp_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') tmp_mesh.write_ply(tmp_path) del tmp assert (nViews == len(cams)) assert (nViews == len(segs)) assert (nViews == len(imgs)) # Define a displacement vector. We set a small non zero displacement as initialization dv = ch.array(np.random.rand(model.r.shape[0], 3) / 1000.) # Cell structure for ARAP f = model.f _, A3, A = edgesIdx(nV=dv.shape[0], f=f, save_dir='.', name='smal') wedge = wedges(A3, dv) s = np.zeros_like(dv) arap = ARAP(reg_e=MatVecMult(A3.T, model.ravel() + dv.ravel()).reshape(-1, 3), model_e=MatVecMult(A3.T, model.ravel()).reshape(-1, 3), w=wedge, A=A) k_arap = settings['ref_k_arap_per_view'] * nViews for weight, part in zip(settings['ref_W_arap_values'], settings['ref_W_arap_parts']): k_arap, W_per_vertex = get_arap_part_weights( A, k_arap, [part], [weight]) #, animal_name) # was only Head W = np.zeros((W_per_vertex.shape[0], 3)) for i in range(3): W[:, i] = W_per_vertex k_lap = settings['ref_k_lap'] * nViews * W k_sym = settings['ref_k_sym'] * nViews k_keyp = settings['ref_k_keyp_weight'] * nViews # Load already computed mesh if not exists(opt_model_dir): makedirs(opt_model_dir) shape_model, compute = load_shape_models(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas) mv = None # Remove inside mouth faces ''' if settings['ref_remove_inside_mouth']: # Giraffe faces_orig = shape_model[0].f.copy() im_v = im_up_v + im_down_v idx = [np.where(model.f == ix)[0] for ix in im_v] idx = np.concatenate(idx).ravel() for i in range(nViews): shape_model[i].f = np.delete(shape_model[i].f, idx, 0) ''' if compute: objs = {} FIX_CAM = True free_variables = [] kp_weights = k_keyp * np.ones((landmarks[0].shape[0], 1)) print('removing shoulders, often bad annotated') kp_weights[landmarks_names.index('leftShoulder'), :] *= 0 kp_weights[landmarks_names.index('rightShoulder'), :] *= 0 objs_pose = None j2d = None #k_silh_term = settings['ref_k_silh_term'] k_m2s = settings['ref_k_m2s'] k_s2m = settings['ref_k_s2m'] objs, params_, j2d = set_pose_objs(shape_model, cams, landmarks, key_vids, kp_weights=kp_weights, FIX_CAM=FIX_CAM, ONLY_KEYP=True, OPT_SHAPE=False) if np.any(k_arap) != 0: objs['arap'] = k_arap * arap if k_sym != 0: objs['sym_0'] = k_sym * (ch.abs(dv[:, 0] - dv[symIdx, 0])) objs['sym_1'] = k_sym * (ch.abs(dv[:, 1] + dv[symIdx, 1] - 0.00014954)) objs['sym_2'] = k_sym * (ch.abs(dv[:, 2] - dv[symIdx, 2])) if np.any(k_lap) != 0: lap_op = np.asarray( laplacian(Mesh(v=dv, f=shape_model[0].f)).todense()) objs['lap'] = k_lap * ch.dot(lap_op, dv) mv = None mv2 = MeshViewers(shape=(1, nViews)) #None vc = np.ones_like(dv) dv_r = fit_silhouettes_pyramid_opt(objs, shape_model, dv, segs, cams, j2d=j2d, weights=1., mv=mv, imgs=imgs, s2m_weights=k_s2m, m2s_weights=k_m2s, max_iter=100, free_variables=free_variables, vc=vc, symIdx=symIdx, mv2=mv2, objs_pose=objs_pose) # Save result image for i in range(nViews): img_res = render_mesh(Mesh(shape_model[i].r, shape_model[i].f), imgs[i].shape[1], imgs[i].shape[0], cams[i], img=imgs[i], world_frame=True) img_result = np.hstack((imgs[i], img_res * 255.)) save_img_path = save_name[i].replace('.pkl', '_v_opt.png') cv2.imwrite(save_img_path, img_result) shape_model[0].pose[:] = 0 shape_model[0].trans[:] = 0 V = shape_model[0].r.copy() vm = V[symIdx, :].copy() vm[:, 1] = -1 * vm[:, 1] V2 = (V + vm) / 2.0 mesh_out = Mesh(v=V2, f=shape_model[0].f) mesh_out.show() mesh_out.write_ply(mesh_v_opt_save_path) save_dv_data_path = mesh_v_opt_save_path.replace('.ply', '_dv.pkl') dv_data = {'betas': shape_model[0].betas.r, 'dv': dv_r} pkl.dump(dv_data, open(save_dv_data_path, 'wb')) compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas, '_non_opt', cams, imgs, segs, img_paths, img_offset, img_scales) return