def transformObject(v, vn, chScale, chObjAz, chObjDisplacement, chObjRotation, targetPosition): scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3, 0:3] chRotAzMat = geometry.RotateZ(a=-chObjAz)[0:3, 0:3] transformation = ch.dot(chRotAzMat, scaleMat) invTranspModel = ch.transpose(ch.inv(transformation)) objDisplacementMat = computeHemisphereTransformation( chObjRotation, 0, chObjDisplacement, np.array([0, 0, 0])) newPos = objDisplacementMat[0:3, 3] vtransf = [] vntransf = [] for mesh_i, mesh in enumerate(v): vtransf = vtransf + [ ch.dot(v[mesh_i], transformation) + newPos + targetPosition ] # ipdb.set_trace() # vtransf = vtransf + [v[mesh_i] + chPosition] vndot = ch.dot(vn[mesh_i], invTranspModel) vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None] vntransf = vntransf + [vndot] return vtransf, vntransf, newPos
def transformObject(v, vn, chScale, chObjAz, chPosition): #print ('utils.py:16 transformObject') #print ('v', type(v)) #import ipdb #ipdb.set_trace() if chScale.size == 1: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[0])[0:3, 0:3] elif chScale.size == 2: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[1])[0:3, 0:3] else: scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3, 0:3] chRotAzMat = geometry.RotateZ(a=chObjAz)[0:3, 0:3] chRotAzMatX = geometry.RotateX(a=0)[0:3, 0:3] # transformation = scaleMat transformation = ch.dot(ch.dot(chRotAzMat, chRotAzMatX), scaleMat) invTranspModel = ch.transpose(ch.inv(transformation)) vtransf = [] vntransf = [] for mesh_i, mesh in enumerate(v): vtransf = vtransf + [ch.dot(v[mesh_i], transformation) + chPosition] vndot = ch.dot(vn[mesh_i], invTranspModel) vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None] vntransf = vntransf + [vndot] return vtransf, vntransf
def transformObjectFull(v, vn, chScale, chObjAz, chObjAx, chObjAz2, chPosition): if chScale.size == 1: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[0])[0:3, 0:3] elif chScale.size == 2: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[1])[0:3, 0:3] else: scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3, 0:3] chRotAzMat = geometry.RotateZ(a=chObjAz)[0:3, 0:3] chRotAxMat = geometry.RotateX(a=-chObjAx)[0:3, 0:3] chRotAzMat2 = geometry.RotateZ(a=chObjAz2)[0:3, 0:3] transformation = ch.dot( ch.dot(ch.dot(chRotAzMat, chRotAxMat), chRotAzMat2), scaleMat) invTranspModel = ch.transpose(ch.inv(transformation)) vtransf = [] vntransf = [] for mesh_i, mesh in enumerate(v): vtransf = vtransf + [ch.dot(v[mesh_i], transformation) + chPosition] vndot = ch.dot(vn[mesh_i], invTranspModel) vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None] vntransf = vntransf + [vndot] return vtransf, vntransf
def computeHemisphereTransformation(chAz, chEl, chDist, objCenter): chDistMat = geometry.Translate(x=ch.Ch(0), y=-chDist, z=ch.Ch(0)) chToObjectTranslate = geometry.Translate(x=objCenter[0], y=objCenter[1], z=objCenter[2]) chRotAzMat = geometry.RotateZ(a=chAz) chRotElMat = geometry.RotateX(a=-chEl) chCamModelWorld = ch.dot(chToObjectTranslate, ch.dot(chRotAzMat, ch.dot(chRotElMat, chDistMat))) return chCamModelWorld
def computeGlobalAndDirectionalLighting(vn, vc, chLightAzimuth, chLightElevation, chLightIntensity, chGlobalConstant): # Construct point light source rangeMeshes = range(len(vn)) vc_list = [] chRotAzMat = geometry.RotateZ(a=chLightAzimuth)[0:3, 0:3] chRotElMat = geometry.RotateX(a=chLightElevation)[0:3, 0:3] chLightVector = -ch.dot(chRotAzMat, ch.dot(chRotElMat, np.array([0, 0, -1 ]))) for mesh in rangeMeshes: l1 = ch.maximum(ch.dot(vn[mesh], chLightVector).reshape((-1, 1)), 0.) vcmesh = vc[mesh] * (chLightIntensity * l1 + chGlobalConstant) vc_list = vc_list + [vcmesh] return vc_list
def setupCamera(v, chAz, chEl, chDist, objCenter, width, height): chCamModelWorld = computeHemisphereTransformation(chAz, chEl, chDist, objCenter) chMVMat = ch.dot(chCamModelWorld, np.array(mathutils.Matrix.Rotation(radians(270), 4, 'X'))) chInvCam = ch.inv(chMVMat) modelRotation = chInvCam[0:3, 0:3] chRod = opendr.geometry.Rodrigues(rt=modelRotation).reshape(3) chTranslation = chInvCam[0:3, 3] translation, rotation = (chTranslation, chRod) camera = ProjectPoints(v=v, rt=rotation, t=translation, f=1.12 * ch.array([width, width]), c=ch.array([width, height]) / 2.0, k=ch.zeros(5)) camera.openglMat = np.array(mathutils.Matrix.Rotation( radians(180), 4, 'X')) return camera, modelRotation, chMVMat
def chShapeParamsToNormals(N, landmarks, linT): T = ch.dot(linT, landmarks) invT = [] nLandmarks = landmarks.r.shape[0] for i in range(nLandmarks): R = T[4 * i:4 * i + 3, :3].T invR = ch.inv(R.T) invT = invT + [invR] invT = ch.vstack(invT) newNormals = ch.dot(N, invT) import opendr.geometry n = opendr.geometry.NormalizedNx3(newNormals) return newNormals
def setupCamera(v, cameraParams): chDistMat = geometry.Translate(x=0, y=cameraParams['Zshift'], z=cameraParams['chCamHeight']) chRotElMat = geometry.RotateX(a=-cameraParams['chCamEl']) chCamModelWorld = ch.dot(chDistMat, chRotElMat) flipZYRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0, 1.0, 0.0], [0.0, -1.0, 0, 0.0], [0.0, 0.0, 0.0, 1.0]]) chMVMat = ch.dot(chCamModelWorld, flipZYRotation) chInvCam = ch.inv(chMVMat) modelRotation = chInvCam[0:3, 0:3] chRod = opendr.geometry.Rodrigues(rt=modelRotation).reshape(3) chTranslation = chInvCam[0:3, 3] translation, rotation = (chTranslation, chRod) camera = ProjectPoints(v=v, rt=rotation, t=translation, f=1000 * cameraParams['chCamFocalLength'] * cameraParams['a'], c=cameraParams['c'], k=ch.zeros(5)) flipXRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0., 0.0], [0.0, 0., -1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) camera.openglMat = flipXRotation #Needed to match OpenGL flipped axis. return camera, modelRotation, chMVMat
def test_matmatmult(self): from chumpy import dot mtx1 = ch.Ch(np.arange(6).reshape((3, 2))) mtx2 = ch.Ch(np.arange(8).reshape((2, 4)) * 10) mtx3 = dot(mtx1, mtx2) #print(mtx1.r) #print(mtx2.r) #print(mtx3.r) #print(mtx3.dr_wrt(mtx1).todense()) #print(mtx3.dr_wrt(mtx2).todense()) for mtx in [mtx1, mtx2]: oldval = mtx3.r.copy() mtxd = mtx3.dr_wrt(mtx).copy() mtx_diff = np.random.rand(mtx.r.size).reshape(mtx.r.shape) mtx.x = mtx.r + mtx_diff mtx_emp = mtx3.r - oldval mtx_pred = mtxd.dot(mtx_diff.ravel()).reshape(mtx_emp.shape) self.assertTrue(np.max(np.abs(mtx_emp - mtx_pred)) < 1e-11)
def setupCamera(v, cameraParams, is_ycb=False): chDistMat = geometry.Translate(x=0, y=cameraParams['Zshift'], z=cameraParams['chCamHeight']) #print ('chDistMat', chDistMat) chRotElMat = geometry.RotateX(a=-cameraParams['chCamEl']) chCamModelWorld = ch.dot(chDistMat, chRotElMat) flipZYRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, 0, 1.0, 0.0], [0.0, -1.0, 0, 0.0], [0.0, 0.0, 0.0, 1.0]]) chMVMat = ch.dot(chCamModelWorld, flipZYRotation) if is_ycb: chMVMat = ch.Ch(np.eye(4)) # np.save('extrinsics.npy', chMVMat, allow_pickle=False) chInvCam = ch.inv(chMVMat) modelRotation = chInvCam[0:3, 0:3] chRod = opendr.geometry.Rodrigues(rt=modelRotation).reshape(3) chTranslation = chInvCam[0:3, 3] translation, rotation = (chTranslation, chRod) # camera parameters format suitable for YCB video dataset if 'a' in cameraParams.keys(): # NOTE: Focal lenght is represented in mm and a is no.of pixels per mm _f = 1000 * cameraParams['chCamFocalLength'] * cameraParams['a'] else: # NOTE: Focal length is already in terms of pixels if np.any(cameraParams['chCamFocalLength'] < 1): import sys sys.exit( "Camera Focal length 'chCamFocalLength' is represented in number of pixels." ) _f = cameraParams['chCamFocalLength'] if 'k' in cameraParams.keys(): _k = cameraParams['k'] else: _k = ch.zeros(5) print('Using k', _k) camera = ProjectPoints(v=v, rt=rotation, t=translation, f=_f, c=cameraParams['c'], k=_k) # _f = 1000 * cameraParams['chCamFocalLength'] * cameraParams['a'] # _c = cameraParams['c'] #np.save('intrinsics.npy', camera.camera_mtx, allow_pickle=False) ##print ('camera shape', camera.shape) ##print ('camera ', camera) print('camera.camera_mtx', camera.camera_mtx) np.save('projection_matrix', camera.camera_mtx) #import ipdb #ipdb.set_trace() flipXRotation = np.array([[1.0, 0.0, 0.0, 0.0], [0.0, -1.0, 0., 0.0], [0.0, 0., -1.0, 0.0], [0.0, 0.0, 0.0, 1.0]]) camera.openglMat = flipXRotation #Needed to match OpenGL flipped axis. return camera, modelRotation, chMVMat
def final_fit( opt, part_mesh, v, v_offset, dist_o, dist_i, smpl_h_ref, rn_m, debug_rn, dif_mask, v_ids_template, faces_template, v_ids_side, faces_side, max_y, proj_cam, ref_joint_list_coup, ): if opt.disp_mesh_side: mv = MeshViewer() else: mv = None if opt.disp_mesh_whl: mv2 = MeshViewer() else: mv2 = None import scipy.sparse as sp sparse_solver = lambda A, x: sp.linalg.cg(A, x, maxiter=500)[0] tgt_pose = get_pose_prior(init_pose_path=opt.init_pose_path, gar_type=opt.gar_type) E = { 'mask': gaussian_pyramid(rn_m * dist_o * opt.ref_wt_dist_o + (1 - rn_m) * dist_i, n_levels=4, normalization='size') * opt.ref_wt_mask } x0 = [v_offset] if opt.ref_wt_coup: # x0 = [smpl_h_ref.trans, smpl_h_ref.betas, v_offset] E['coupling'] = (v + v_offset - smpl_h_ref[v_ids_template]) * opt.ref_wt_coup if opt.ref_wt_shp: E['beta_prior'] = ch.linalg.norm(smpl_h_ref.betas) * opt.ref_wt_shp if opt.ref_wt_pose: E['pose'] = (smpl_h_ref.pose - tgt_pose) * opt.ref_wt_pose if ref_joint_list_coup != None: range_joint = [] for item in ref_joint_list_coup: range_joint.append(3 * int(item)) range_joint.append(3 * int(item) + 1) range_joint.append(3 * int(item) + 2) x0 = x0 + [smpl_h_ref.pose[range_joint]] if opt.ref_use_betas: x0 = x0 + [smpl_h_ref.betas] if opt.ref_wt_proj: error_bd = get_rings_error(proj_cam, max_y) E['proj_bd'] = error_bd * opt.ref_wt_proj if opt.ref_wt_bd: gar_rings = compute_boundaries(v + v_offset, faces_template) error = smooth_rings(gar_rings, v + v_offset) E['boundary'] = error * opt.ref_wt_bd if opt.ref_wt_lap: lap_op = np.asarray(laplacian(part_mesh).todense()) lap_err = ch.dot(lap_op, v + v_offset) E['laplacian'] = lap_err * opt.ref_wt_lap ch.minimize(E, x0, method='dogleg', options={ 'e_3': .000001, 'sparse_solver': sparse_solver }, callback=get_callback_ref(rend=debug_rn, mask=dif_mask, vertices=v + v_offset, display=opt.display, v_ids_sides=v_ids_side, faces_template=faces_template, faces_side=faces_side, disp_mesh_side=opt.disp_mesh_side, disp_mesh_whl=opt.disp_mesh_whl, save_dir=opt.save_opt_images, mv=mv, mv2=mv2, show=opt.show)) final_mask = rn_m.r mask = dif_mask.copy() mask[dif_mask == 0.5] = 1 final_iou = compute_iou(mask, final_mask) return v + v_offset, final_iou
chComponentGT = ch.Ch(np.array([2, 0.25, 0.25, 0.12,-0.17,0.36,0.1,0.,0.])) chComponent = ch.Ch(np.array([2, 0.25, 0.25, 0.12,-0.17,0.36,0.1,0.,0.])) chPointLightIntensity = ch.Ch([1]) chPointLightIntensityGT = ch.Ch([1]) chLightAz = ch.Ch([0.0]) chLightEl = ch.Ch([np.pi/2]) chLightDist = ch.Ch([0.5]) chLightDistGT = ch.Ch([0.5]) chLightAzGT = ch.Ch([0.0]) chLightElGT = ch.Ch([np.pi/4]) ligthTransf = computeHemisphereTransformation(chLightAz, chLightEl, chLightDist, targetPosition) ligthTransfGT = computeHemisphereTransformation(chLightAzGT, chLightElGT, chLightDistGT, targetPosition) lightPos = ch.dot(ligthTransf, ch.Ch([0.,0.,0.,1.]))[0:3] lightPos = ch.Ch([targetPosition[0]+0.5,targetPosition[1],targetPosition[2] + 0.5]) lightPosGT = ch.dot(ligthTransfGT, ch.Ch([0.,0.,0.,1.]))[0:3] chGlobalConstant = ch.Ch([0.5]) chGlobalConstantGT = ch.Ch([0.5]) light_color = ch.ones(3)*chPointLightIntensity light_colorGT = ch.ones(3)*chPointLightIntensityGT chVColors = ch.Ch([0.8,0.8,0.8]) chVColorsGT = ch.Ch([0.8,0.8,0.8]) shCoefficientsFile = 'data/sceneSH' + str(sceneIdx) + '.pickle' chAmbientIntensityGT = ch.Ch([0.025]) clampedCosCoeffs = clampedCosineCoefficients() chAmbientSHGT = ch.zeros([9])
def optimize_smal(fposes, ftrans, fbetas, model, cams, segs, imgs, landmarks, landmarks_names, key_vids, symIdx=None, frameId=0, opt_model_dir=None, save_name=None, COMPUTE_OPT=True, img_paths=None, img_offset=None, img_scales=None): mesh_v_opt_save_path = join(opt_model_dir, 'mesh_v_opt_no_mc_' + str(frameId) + '.ply') mesh_v_opt_mc_save_path = join(opt_model_dir, 'mesh_v_opt_' + str(frameId) + '.ply') mesh_init_save_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') nViews = len(fposes) if not COMPUTE_OPT: if not exists(opt_model_dir): makedirs(opt_model_dir) dv = 0 compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_init_save_path, fposes, ftrans, fbetas, '_no_refine', cams, imgs, segs, img_paths, img_offset, img_scales) return # Write the initial mesh np_betas = np.zeros_like(model.betas) np_betas[:len(fbetas[0])] = fbetas[0] tmp = verts_decorated(v_template=model.v_template, pose=ch.zeros_like(model.pose.r), trans=ch.zeros_like(model.trans), J=model.J_regressor, kintree_table=model.kintree_table, betas=ch.array(np_betas), weights=model.weights, posedirs=model.posedirs, shapedirs=model.shapedirs, bs_type='lrotmin', bs_style='lbs', f=model.f) tmp_mesh = Mesh(v=tmp.r, f=tmp.f) tmp_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') tmp_mesh.write_ply(tmp_path) del tmp assert (nViews == len(cams)) assert (nViews == len(segs)) assert (nViews == len(imgs)) # Define a displacement vector. We set a small non zero displacement as initialization dv = ch.array(np.random.rand(model.r.shape[0], 3) / 1000.) # Cell structure for ARAP f = model.f _, A3, A = edgesIdx(nV=dv.shape[0], f=f, save_dir='.', name='smal') wedge = wedges(A3, dv) s = np.zeros_like(dv) arap = ARAP(reg_e=MatVecMult(A3.T, model.ravel() + dv.ravel()).reshape(-1, 3), model_e=MatVecMult(A3.T, model.ravel()).reshape(-1, 3), w=wedge, A=A) k_arap = settings['ref_k_arap_per_view'] * nViews for weight, part in zip(settings['ref_W_arap_values'], settings['ref_W_arap_parts']): k_arap, W_per_vertex = get_arap_part_weights( A, k_arap, [part], [weight]) #, animal_name) # was only Head W = np.zeros((W_per_vertex.shape[0], 3)) for i in range(3): W[:, i] = W_per_vertex k_lap = settings['ref_k_lap'] * nViews * W k_sym = settings['ref_k_sym'] * nViews k_keyp = settings['ref_k_keyp_weight'] * nViews # Load already computed mesh if not exists(opt_model_dir): makedirs(opt_model_dir) shape_model, compute = load_shape_models(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas) mv = None # Remove inside mouth faces ''' if settings['ref_remove_inside_mouth']: # Giraffe faces_orig = shape_model[0].f.copy() im_v = im_up_v + im_down_v idx = [np.where(model.f == ix)[0] for ix in im_v] idx = np.concatenate(idx).ravel() for i in range(nViews): shape_model[i].f = np.delete(shape_model[i].f, idx, 0) ''' if compute: objs = {} FIX_CAM = True free_variables = [] kp_weights = k_keyp * np.ones((landmarks[0].shape[0], 1)) print('removing shoulders, often bad annotated') kp_weights[landmarks_names.index('leftShoulder'), :] *= 0 kp_weights[landmarks_names.index('rightShoulder'), :] *= 0 objs_pose = None j2d = None #k_silh_term = settings['ref_k_silh_term'] k_m2s = settings['ref_k_m2s'] k_s2m = settings['ref_k_s2m'] objs, params_, j2d = set_pose_objs(shape_model, cams, landmarks, key_vids, kp_weights=kp_weights, FIX_CAM=FIX_CAM, ONLY_KEYP=True, OPT_SHAPE=False) if np.any(k_arap) != 0: objs['arap'] = k_arap * arap if k_sym != 0: objs['sym_0'] = k_sym * (ch.abs(dv[:, 0] - dv[symIdx, 0])) objs['sym_1'] = k_sym * (ch.abs(dv[:, 1] + dv[symIdx, 1] - 0.00014954)) objs['sym_2'] = k_sym * (ch.abs(dv[:, 2] - dv[symIdx, 2])) if np.any(k_lap) != 0: lap_op = np.asarray( laplacian(Mesh(v=dv, f=shape_model[0].f)).todense()) objs['lap'] = k_lap * ch.dot(lap_op, dv) mv = None mv2 = MeshViewers(shape=(1, nViews)) #None vc = np.ones_like(dv) dv_r = fit_silhouettes_pyramid_opt(objs, shape_model, dv, segs, cams, j2d=j2d, weights=1., mv=mv, imgs=imgs, s2m_weights=k_s2m, m2s_weights=k_m2s, max_iter=100, free_variables=free_variables, vc=vc, symIdx=symIdx, mv2=mv2, objs_pose=objs_pose) # Save result image for i in range(nViews): img_res = render_mesh(Mesh(shape_model[i].r, shape_model[i].f), imgs[i].shape[1], imgs[i].shape[0], cams[i], img=imgs[i], world_frame=True) img_result = np.hstack((imgs[i], img_res * 255.)) save_img_path = save_name[i].replace('.pkl', '_v_opt.png') cv2.imwrite(save_img_path, img_result) shape_model[0].pose[:] = 0 shape_model[0].trans[:] = 0 V = shape_model[0].r.copy() vm = V[symIdx, :].copy() vm[:, 1] = -1 * vm[:, 1] V2 = (V + vm) / 2.0 mesh_out = Mesh(v=V2, f=shape_model[0].f) mesh_out.show() mesh_out.write_ply(mesh_v_opt_save_path) save_dv_data_path = mesh_v_opt_save_path.replace('.ply', '_dv.pkl') dv_data = {'betas': shape_model[0].betas.r, 'dv': dv_r} pkl.dump(dv_data, open(save_dv_data_path, 'wb')) compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas, '_non_opt', cams, imgs, segs, img_paths, img_offset, img_scales) return
gmm = mixture.GMM(n_components=nComps, covariance_type='spherical') win = 40 colors = image[image.shape[0] / 2 - win:image.shape[0] / 2 + win, image.shape[1] / 2 - win:image.shape[1] / 2 + win, :].reshape( [4 * win * win, 3]) gmm.fit(colors) imshape = [win * 2, win * 2, 3] numPixels = win * 2 * win * 2 chInput = ch.Ch(colors) numVars = chInput.size recSoftmaxW = ch.Ch(np.random.uniform(0, 1, [nRecComps, numVars]) / numVars) chRecLogistic = ch.exp(ch.dot(recSoftmaxW, chInput.reshape([numVars, 1]))) chRecSoftmax = chRecLogistic.ravel() / ch.sum(chRecLogistic) chZRecComps = ch.zeros([numVars, nRecComps]) chZ = ch.zeros([numVars]) recMeans = ch.Ch(np.random.uniform(0, 1, [3, nRecComps])) recCovars = 0.2 chRecLogLikelihoods = -0.5 * (chZ.reshape([numPixels, 3, 1]) - ch.tile( recMeans, [numPixels, 1, 1]))**2 - ch.log( (2 * recCovars) * (1 / (ch.sqrt(recCovars) * np.sqrt(2 * np.pi)))) genZCovars = 0.2 chGenComponentsProbs = ch.Ch(gmm.weights_) chCompMeans = ch.zeros([nComps, 3])
def chShapeParamsToVerts(landmarks, meshLinearTransform): vertices = ch.dot(meshLinearTransform, landmarks) return vertices