def lambertian_spotlight(v, vn, pos, dir, spot_exponent, camcoord=False, camera_t=None, camera_rt=None): """ :param v: vertices :param vn: vertex normals :param light_pos: light position :param light_dir: light direction :param spot_exponent: spot exponent (a la opengl) :param camcoord: if True, then pos and dir are wrt the camera :param camera_t: 3-vector indicating translation of camera :param camera_rt: 3-vector indicating direction of camera :return: Vx1 array of brightness """ if camcoord: # Transform pos and dir from camera to world coordinate system assert(camera_t is not None and camera_rt is not None) from opendr.geometry import Rodrigues rot = Rodrigues(rt=camera_rt) pos = rot.T.dot(pos-camera_t) dir = rot.T.dot(dir) dir = dir / ch.sqrt(ch.sum(dir**2.)) v_minus_light = v - pos.reshape((1,3)) v_distances = ch.sqrt(ch.sum(v_minus_light**2, axis=1)) v_minus_light_normed = v_minus_light / v_distances.reshape((-1,1)) cosangle = v_minus_light_normed.dot(dir.reshape((3,1))) light_dot_normal = ch.sum(vn*v_minus_light_normed, axis=1) light_dot_normal.label = 'light_dot_normal' cosangle.label = 'cosangle' result = light_dot_normal.ravel() * cosangle.ravel()**spot_exponent result = result / v_distances ** 2. result = maximum(result, 0.0) return result
def _set_up(self): self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template body_height = (self.v_shaped[2802, 1] + self.v_shaped[6262, 1]) - ( self.v_shaped[2237, 1] + self.v_shaped[6728, 1]) # print('111111111111111111111111111111111111111') # print(np.array(body_height)[0]) # print(type(body_height)) # print('22222222222222222222222222222222222222222') self.scale = 1.66 / np.array(body_height)[0] self.v_shaped_personal = self.scale * self.v_shaped + self.v_personal if sp.issparse(self.J_regressor): self.J = self.scale * sp_dot(self.J_regressor, self.v_shaped) else: self.J = self.scale * ch.sum(self.J_regressor.T.reshape(-1, 1, 24) * self.v_shaped.reshape(-1, 3, 1), axis=0).T self.v_posevariation = self.posedirs.dot( posemap(self.bs_type)(self.pose)) self.v_poseshaped = self.v_shaped_personal + self.v_posevariation self.A, A_global = self._global_rigid_transformation() self.Jtr = ch.vstack([g[:3, 3] for g in A_global]) self.J_transformed = self.Jtr + self.trans.reshape((1, 3)) self.V = self.A.dot(self.weights.T) rest_shape_h = ch.hstack( (self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1)))) self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3] self.v = self.v_posed + self.trans
def LightDotNormal(num_verts): normalize_rows = lambda v : v / col(ch.sqrt(ch.sum(v.reshape((-1,3))**2, axis=1))) sum_rows = lambda v : ch.sum(v.reshape((-1,3)), axis=1) return Ch(lambda light_pos, v, vn : sum_rows(normalize_rows(light_pos.reshape((1,3)) - v.reshape((-1,3))) * vn.reshape((-1,3))))
def _set_up(self): self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template self.v_shaped_personal = self.v_shaped + self.v_personal if sp.issparse(self.J_regressor): self.J = sp_dot(self.J_regressor, self.v_shaped) else: self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) * self.v_shaped.reshape(-1, 3, 1), axis=0).T self.v_posevariation = self.posedirs.dot( posemap(self.bs_type)(self.pose)) self.v_poseshaped = self.v_shaped_personal + self.v_posevariation self.A, A_global = self._global_rigid_transformation() self.Jtr = ch.vstack([g[:3, 3] for g in A_global]) self.J_transformed = self.Jtr + self.trans.reshape((1, 3)) self.V = self.A.dot(self.weights.T) rest_shape_h = ch.hstack( (self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1)))) self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1), axis=1)[:, :3] self.v = self.v_posed + self.trans
def LightDotNormal(num_verts): normalize_rows = lambda v : v / ch.sqrt(ch.sum(v.reshape((-1,3))**2, axis=1)).reshape((-1,1)) sum_rows = lambda v : ch.sum(v.reshape((-1,3)), axis=1) return Ch(lambda light_pos, v, vn : sum_rows(normalize_rows(light_pos.reshape((1,3)) - v.reshape((-1,3))) * vn.reshape((-1,3))))
def chZernikeProjection(image, numCoeffs=20, win=50): croppedImage = image[image.shape[0] / 2 - win:image.shape[0] / 2 + win, image.shape[1] / 2 - win:image.shape[1] / 2 + win, :] zpolys = zernikePolynomials(imageSize=(croppedImage.shape[0], croppedImage.shape[1]), numCoeffs=numCoeffs) imageProjections = croppedImage[:, :, :, None] * zpolys.reshape( [zpolys.shape[0], zpolys.shape[1], 1, -1]) coeffs = ch.sum(ch.sum(imageProjections, axis=0), axis=0) return coeffs, imageProjections
def transformObject(v, vn, chScale, chObjAz, chObjDisplacement, chObjRotation, targetPosition): scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3, 0:3] chRotAzMat = geometry.RotateZ(a=-chObjAz)[0:3, 0:3] transformation = ch.dot(chRotAzMat, scaleMat) invTranspModel = ch.transpose(ch.inv(transformation)) objDisplacementMat = computeHemisphereTransformation( chObjRotation, 0, chObjDisplacement, np.array([0, 0, 0])) newPos = objDisplacementMat[0:3, 3] vtransf = [] vntransf = [] for mesh_i, mesh in enumerate(v): vtransf = vtransf + [ ch.dot(v[mesh_i], transformation) + newPos + targetPosition ] # ipdb.set_trace() # vtransf = vtransf + [v[mesh_i] + chPosition] vndot = ch.dot(vn[mesh_i], invTranspModel) vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None] vntransf = vntransf + [vndot] return vtransf, vntransf, newPos
def run(self, beta=None, theta=None, garment_d=None, garment_class=None): """Outputs body and garment of specified garment class given theta, beta and displacements.""" if beta is not None: self.smpl_base.betas[:beta.shape[0]] = beta else: self.smpl_base.betas[:] = 0 if theta is not None: self.smpl_base.pose[:] = theta else: self.smpl_base.pose[:] = 0 self.smpl_base.v_personal[:] = 0 if garment_d is not None and garment_class is not None: if 'skirt' not in garment_class: vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] self.smpl_base.v_personal[vert_indices] = garment_d garment_m = Mesh(v=self.smpl_base.r[vert_indices], f=f) else: # vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] A = self.smpl_base.A.reshape((16, 24)).T skirt_V = self.skirt_skinning.dot(A).reshape((-1, 4, 4)) verts = self.skirt_weight.dot(self.smpl_base.v_poseshaped) verts = verts + garment_d verts_h = ch.hstack((verts, ch.ones((verts.shape[0], 1)))) verts = ch.sum(skirt_V * verts_h.reshape(-1, 1, 4), axis=-1)[:, :3] garment_m = Mesh(v=verts, f=f) else: garment_m = None self.smpl_base.v_personal[:] = 0 body_m = Mesh(v=self.smpl_base.r, f=self.smpl_base.f) return body_m, garment_m
def transformObjectFull(v, vn, chScale, chObjAz, chObjAx, chObjAz2, chPosition): if chScale.size == 1: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[0])[0:3, 0:3] elif chScale.size == 2: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[1])[0:3, 0:3] else: scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3, 0:3] chRotAzMat = geometry.RotateZ(a=chObjAz)[0:3, 0:3] chRotAxMat = geometry.RotateX(a=-chObjAx)[0:3, 0:3] chRotAzMat2 = geometry.RotateZ(a=chObjAz2)[0:3, 0:3] transformation = ch.dot( ch.dot(ch.dot(chRotAzMat, chRotAxMat), chRotAzMat2), scaleMat) invTranspModel = ch.transpose(ch.inv(transformation)) vtransf = [] vntransf = [] for mesh_i, mesh in enumerate(v): vtransf = vtransf + [ch.dot(v[mesh_i], transformation) + chPosition] vndot = ch.dot(vn[mesh_i], invTranspModel) vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None] vntransf = vntransf + [vndot] return vtransf, vntransf
def joints_coco(smpl): J = smpl.J_transformed nose = smpl[VERT_NOSE] ear_l = smpl[VERT_EAR_L] ear_r = smpl[VERT_EAR_R] eye_l = smpl[VERT_EYE_L] eye_r = smpl[VERT_EYE_R] shoulders_m = ch.sum(J[[14, 13]], axis=0) / 2. neck = J[12] - 0.55 * (J[12] - shoulders_m) return ch.vstack(( nose, neck, 2.1 * (J[14] - shoulders_m) + neck, J[[19, 21]], 2.1 * (J[13] - shoulders_m) + neck, J[[18, 20]], J[2] + 0.38 * (J[2] - J[1]), J[[5, 8]], J[1] + 0.38 * (J[1] - J[2]), J[[4, 7]], eye_r, eye_l, ear_r, ear_l, ))
def on_changed(self, which): if not hasattr(self, 'normalized'): self.normalized = True if hasattr(self, 'v') and hasattr(self, 'f'): if 'f' not in which and hasattr(self, 'faces_by_vertex') and self.faces_by_vertex.shape[0]/3 == self.v.shape[0]: self.tns.v = self.v else: # change in f or in size of v. shouldn't happen often. f = self.f IS = f.ravel() JS = np.array([range(f.shape[0])]*3).T.ravel() data = np.ones(len(JS)) IS = np.concatenate((IS*3, IS*3+1, IS*3+2)) JS = np.concatenate((JS*3, JS*3+1, JS*3+2)) data = np.concatenate((data, data, data)) sz = self.v.size self.faces_by_vertex = sp.csc_matrix((data, (IS, JS)), shape=(sz, f.size)) self.tns = Ch(lambda v : CrossProduct(TriEdges(f,1,0,v), TriEdges(f,2,0, v))) self.tns.v = self.v if self.normalized: tmp = MatVecMult(self.faces_by_vertex, self.tns) self.normals = NormalizedNx3(tmp) else: test = self.faces_by_vertex.dot(np.ones(self.faces_by_vertex.shape[1])) faces_by_vertex = sp.diags([1. / test], [0]).dot(self.faces_by_vertex).tocsc() normals = MatVecMult(faces_by_vertex, self.tns).reshape((-1,3)) normals = normals / (ch.sum(normals**2, axis=1) ** .25).reshape((-1,1)) self.normals = normals
def setup_silhouette_obj(silhs, rends, f): n_model = [ch.sum(rend[:, :, 0] > 0) for rend in rends] dist_tsf = [cv2.distanceTransform(np.uint8(1 - silh), cv2.DIST_L2, cv2.DIST_MASK_PRECISE) for silh in silhs] # Make sigma proportional to image area. # This gives radius 400 for 960 x 1920 image. sigma_ratio = 0.2727 sigma = [np.sqrt(sigma_ratio * (silh.shape[0] * silh.shape[1]) / np.pi) for silh in silhs] # Model2silhouette ==> Consistency (contraction) def obj_m2s(w, i): return w * (GMOf(rends[i][:, :, 0] * dist_tsf[i], sigma[i]) / np.sqrt(n_model[i])) # silhouette error term (scan-to-model) ==> Coverage (expansion) coords = [np.fliplr(np.array(np.where(silh > 0)).T) + 0.5 for silh in silhs]# is this offset needed? scan_flat_v = [np.hstack((coord, ch.zeros(len(coord)).reshape((-1, 1)))) for coord in coords] scan_flat = [Mesh(v=sflat_v, f=[]) for sflat_v in scan_flat_v] # 2d + all 0. sv_flat = [ch.hstack((rend.camera, ch.zeros(len(rend.v)).reshape((-1, 1)))) for rend in rends] for i in range(len(rends)): sv_flat[i].f = f def obj_s2m(w, i): from sbody.mesh_distance import ScanToMesh return w * ch.sqrt(GMOf(ScanToMesh(scan_flat[i], sv_flat[i], f), sigma[i])) # For vis for i in range(len(rends)): scan_flat[i].vc = np.tile(np.array([0, 0, 1]), (len(scan_flat[i].v), 1)) return obj_m2s, obj_s2m, dist_tsf
def transformObject(v, vn, chScale, chObjAz, chPosition): #print ('utils.py:16 transformObject') #print ('v', type(v)) #import ipdb #ipdb.set_trace() if chScale.size == 1: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[0])[0:3, 0:3] elif chScale.size == 2: scaleMat = geometry.Scale(x=chScale[0], y=chScale[0], z=chScale[1])[0:3, 0:3] else: scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3, 0:3] chRotAzMat = geometry.RotateZ(a=chObjAz)[0:3, 0:3] chRotAzMatX = geometry.RotateX(a=0)[0:3, 0:3] # transformation = scaleMat transformation = ch.dot(ch.dot(chRotAzMat, chRotAzMatX), scaleMat) invTranspModel = ch.transpose(ch.inv(transformation)) vtransf = [] vntransf = [] for mesh_i, mesh in enumerate(v): vtransf = vtransf + [ch.dot(v[mesh_i], transformation) + chPosition] vndot = ch.dot(vn[mesh_i], invTranspModel) vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None] vntransf = vntransf + [vndot] return vtransf, vntransf
def modelLogLikelihoodRobustRegionCh(image, template, testMask, backgroundModel, layerPriors, variances): likelihood = pixelLikelihoodRobustRegionCh(image, template, testMask, backgroundModel, layerPriors, variances) liksum = ch.sum(ch.log(likelihood)) return liksum
def test_sum_mean_std_var(self): for fn in [ch.sum, ch.mean, ch.var, ch.std]: # Create fake input and differences in input space data1 = ch.ones((3, 4, 7, 2)) data2 = ch.array(data1.r + .1 * np.random.rand(data1.size).reshape(data1.shape)) diff = data2.r - data1.r # Compute outputs result1 = fn(data1, axis=2) result2 = fn(data2, axis=2) # Empirical and predicted derivatives gt = result2.r - result1.r pred = result1.dr_wrt(data1).dot(diff.ravel()).reshape(gt.shape) #print(np.max(np.abs(gt - pred))) if fn in [ch.std, ch.var]: self.assertTrue(1e-2 > np.max(np.abs(gt - pred))) else: self.assertTrue(1e-14 > np.max(np.abs(gt - pred))) # test caching dr0 = result1.dr_wrt(data1) data1[:] = np.random.randn(data1.size).reshape(data1.shape) self.assertTrue( result1.dr_wrt(data1) is dr0) # changing values shouldn't force recompute result1.axis = 1 self.assertTrue(result1.dr_wrt(data1) is not dr0) self.assertEqual( ch.mean(ch.eye(3), axis=1).ndim, np.mean(np.eye(3), axis=1).ndim) self.assertEqual( ch.mean(ch.eye(3), axis=0).ndim, np.mean(np.eye(3), axis=0).ndim) self.assertEqual( ch.sum(ch.eye(3), axis=1).ndim, np.sum(np.eye(3), axis=1).ndim) self.assertEqual( ch.sum(ch.eye(3), axis=0).ndim, np.sum(np.eye(3), axis=0).ndim)
def face_bases(v, f): t1 = TriEdges(f, 1, 0, v).reshape((-1,3)) t2 = TriEdges(f, 2, 0, v).reshape((-1,3)) #t3 = NormalizedNx3(CrossProduct(t1, t2)).reshape((-1,3)) #t3 = CrossProduct(t1, t2).reshape((-1,3)) # Problem: cross-product is proportional in length to len(t1)*len(t2) # Solution: divide by sqrt(sqrt(len(cross-product))) t3 = CrossProduct(t1, t2).reshape((-1,3)); t3 = t3 / col(ch.sum(t3**2., axis=1)**.25) result = ch.hstack((t1, t2, t3)).reshape((-1,3,3)) return result
def axis2quat(p): angle = ch.sqrt(ch.clip(ch.sum(ch.square(p), 1), 1e-16, 1e16)) norm_p = p / angle[:, np.newaxis] cos_angle = ch.cos(angle / 2) sin_angle = ch.sin(angle / 2) qx = norm_p[:, 0] * sin_angle qy = norm_p[:, 1] * sin_angle qz = norm_p[:, 2] * sin_angle qw = cos_angle - 1 return ch.concatenate([ qx[:, np.newaxis], qy[:, np.newaxis], qz[:, np.newaxis], qw[:, np.newaxis] ], axis=1)
def LightDotNormal(num_verts): def normalize_rows(v): b = ch.sqrt(ch.sum(v.reshape((-1, 3))**2, axis=1)).reshape((-1, 1)) return v / b.compute_r() sum_rows = lambda v: ch.sum(v.reshape((-1, 3)), axis=1) def f(light_pos, v, vn): light_pos = light_pos[0, ] a = np.array([light_pos, light_pos, light_pos]) v = v[0, ] b = np.array([v, v, v]) return sum_rows(normalize_rows(a - b) * vn.reshape((-1, 3))) return Ch(f)
def volume(v, f): # Construct a 3D matrix which is of size (nfaces x 3 x 3) # Each row corresponds to a face; the third dimension indicates # which triangle in that face is being referred to vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :])) v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0] v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0] v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1] v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1] v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2] v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2] volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1.0 / 6.0) return ch.abs(ch.sum(volumes))
def volume(v, f): # Construct a 3D matrix which is of size (nfaces x 3 x 3) # Each row corresponds to a face; the third dimension indicates # which triangle in that face is being referred to vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :])) v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0] v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0] v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1] v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1] v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2] v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2] volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1. / 6.) return ch.abs(ch.sum(volumes))
def on_changed(self, which): if not hasattr(self, "normalized"): self.normalized = True for w in which: if w not in ("v", "f"): raise Exception("VertNormals has only v and f now, and you specified %s." % (w)) if hasattr(self, "v") and hasattr(self, "f"): if ( "f" not in which and hasattr(self, "faces_by_vertex") and self.faces_by_vertex.shape[0] / 3 == self.v.shape[0] ): self.tns.v = self.v else: # change in f or in size of v. shouldn't happen often. f = self.f IS = f.ravel() JS = np.array([range(f.shape[0])] * 3).T.ravel() data = np.ones(len(JS)) IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2)) JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2)) data = np.concatenate((data, data, data)) sz = self.v.size self.faces_by_vertex = sp.csc_matrix((data, (IS, JS)), shape=(sz, f.size)) self.tns = Ch(lambda v: CrossProduct(TriEdges(f, 1, 0, v), TriEdges(f, 2, 0, v))) self.tns.v = self.v if self.normalized: tmp = MatVecMult(self.faces_by_vertex, self.tns) self.normals = NormalizedNx3(tmp) else: test = self.faces_by_vertex.dot(np.ones(self.faces_by_vertex.shape[1])) faces_by_vertex = sp.diags([1.0 / test], [0]).dot(self.faces_by_vertex).tocsc() normals = MatVecMult(faces_by_vertex, self.tns).reshape((-1, 3)) normals = normals / (ch.sum(normals ** 2, axis=1) ** 0.25).reshape((-1, 1)) self.normals = normals
def run(self, beta=None, theta=None, garment_d=None, garment_class=None): """Outputs body and garment of specified garment class given theta, beta and displacements.""" if beta is not None: self.smpl_base.betas[:beta.shape[0]] = beta else: self.smpl_base.betas[:] = 0 if theta is not None: self.smpl_base.pose[:] = theta else: self.smpl_base.pose[:] = 0 self.smpl_base.v_personal[:] = 0 if garment_d is not None and garment_class is not None: if 'skirt' not in garment_class: vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] self.smpl_base.v_personal[vert_indices] = garment_d garment_m = Mesh(v=self.smpl_base.r[vert_indices], f=f) else: vert_indices = self.class_info[garment_class]['vert_indices'] f = self.class_info[garment_class]['f'] verts = self.smpl_base.v_poseshaped[vert_indices] + garment_d verts_h = ch.hstack((verts, ch.ones((verts.shape[0], 1)))) verts = ch.sum(self.smpl_base.V.T[vert_indices] * verts_h.reshape(-1, 4, 1), axis=1)[:, :3] # if theta is not None: # rotmat = self.smpl_base.A.r[:, :, 0] # verts_homo = np.hstack( # (verts, np.ones((verts.shape[0], 1)))) # verts = verts_homo.dot(rotmat.T)[:, :3] garment_m = Mesh(v=verts, f=f) else: garment_m = None self.smpl_base.v_personal[:] = 0 body_m = Mesh(v=self.smpl_base.r, f=self.smpl_base.f) return body_m, garment_m
win = 40 colors = image[image.shape[0] / 2 - win:image.shape[0] / 2 + win, image.shape[1] / 2 - win:image.shape[1] / 2 + win, :].reshape( [4 * win * win, 3]) gmm.fit(colors) imshape = [win * 2, win * 2, 3] numPixels = win * 2 * win * 2 chInput = ch.Ch(colors) numVars = chInput.size recSoftmaxW = ch.Ch(np.random.uniform(0, 1, [nRecComps, numVars]) / numVars) chRecLogistic = ch.exp(ch.dot(recSoftmaxW, chInput.reshape([numVars, 1]))) chRecSoftmax = chRecLogistic.ravel() / ch.sum(chRecLogistic) chZRecComps = ch.zeros([numVars, nRecComps]) chZ = ch.zeros([numVars]) recMeans = ch.Ch(np.random.uniform(0, 1, [3, nRecComps])) recCovars = 0.2 chRecLogLikelihoods = -0.5 * (chZ.reshape([numPixels, 3, 1]) - ch.tile( recMeans, [numPixels, 1, 1]))**2 - ch.log( (2 * recCovars) * (1 / (ch.sqrt(recCovars) * np.sqrt(2 * np.pi)))) genZCovars = 0.2 chGenComponentsProbs = ch.Ch(gmm.weights_) chCompMeans = ch.zeros([nComps, 3])
def mano_inverse_kinematics(mano_hand, desired_zimmerman_coords, plot_progress=False): joint_idxs_mano = np.asarray(range(16)) joint_idxs_zimmerman = MANO_TO_ZIMMERMAN_INDICES[joint_idxs_mano] assert (joint_idxs_zimmerman.shape == joint_idxs_mano.shape) joint_coords_mano = mano_hand.J_transformed[joint_idxs_mano] joint_coords_zimmerman = desired_zimmerman_coords[joint_idxs_zimmerman] mano_coords_centered = joint_coords_mano - joint_coords_mano[0] zimmerman_coords_centered = joint_coords_zimmerman - joint_coords_zimmerman[ 0] zimmerman_coords_centered *= 0.05 if plot_progress: plot_mano_skeleton(mano_coords_centered, zimmerman_coords_centered) tip_idxs_mano = np.asarray([1, 4, 10, 7]) # knuckles (align palms) scale = ch.var(1) mano_coords_scaled = mano_coords_centered * scale position_loss = (mano_coords_scaled[tip_idxs_mano] - zimmerman_coords_centered[tip_idxs_mano])**2 loss = position_loss inputs = [mano_hand.pose[:3], scale] objective = {'loss': loss} opt = {'maxiter': 10} print("Rotating and scaling palms together...") ch.minimize(objective, x0=inputs, method='dogleg', options=opt) bone_idxs = [] for zb0, zb1 in BONES: if zb0 in ZIMMERMAN_TO_MANO_DICT and zb1 in ZIMMERMAN_TO_MANO_DICT: b0 = ZIMMERMAN_TO_MANO_DICT[zb0] b1 = ZIMMERMAN_TO_MANO_DICT[zb1] bone_idxs.append([b0, b1]) bone_idxs = np.asarray(bone_idxs) zimmerman_displacements = zimmerman_coords_centered[ bone_idxs[:, 1]] - zimmerman_coords_centered[bone_idxs[:, 0]] mano_displacements = mano_coords_scaled[ bone_idxs[:, 1]] - mano_coords_scaled[bone_idxs[:, 0]] displacement_loss = ch.sum( ch.multiply(zimmerman_displacements, mano_displacements)) regularization_loss = 1e-5 * ch.sum(mano_hand.pose[3:]**2) # MAGIC HAPPENING RIGHT HERE: # This huge number is important. # Apparently, ch.minimize will attempt to push the loss to zero, # not towards minus infinity. # Here we're trying to maximize the dot product of all bones # in both hands (thereby making them all point in the same # direction). The dot product needs to be maximally positive, # and should not be zero, hence the huge number below. loss = (1000000 - displacement_loss) + regularization_loss inputs = [mano_hand.pose[3:]] opt = {'maxiter': 10} objective = {'loss': loss} print("Aligning all bones to be co-linear...") ch.minimize(objective, x0=inputs, method='dogleg', options=opt) if plot_progress: plot_mano_skeleton(mano_coords_scaled, zimmerman_coords_centered)
def diffHog(image, drconv=None, numOrient=9, cwidth=8, cheight=8): imagegray = 0.3 * image[:, :, 0] + 0.59 * image[:, :, 1] + 0.11 * image[:, :, 2] sy, sx = imagegray.shape # gx = ch.empty(imagegray.shape, dtype=np.double) gx = imagegray[:, 2:] - imagegray[:, :-2] gx = ch.hstack([np.zeros([sy, 1]), gx, np.zeros([sy, 1])]) gy = imagegray[2:, :] - imagegray[:-2, :] # gy = imagegray[:, 2:] - imagegray[:, :-2] gy = ch.vstack([np.zeros([1, sx]), gy, np.zeros([1, sx])]) gx += 1e-5 # gy = imagegray[:-2,1:-1] - imagegray[2:,1:-1] + 0.00001 # gx = imagegray[1:-1,:-2] - imagegray[1:-1, 2:] + 0.00001 distFilter = np.ones([2 * cheight, 2 * cwidth], dtype=np.uint8) distFilter[np.int(2 * cheight / 2), np.int(2 * cwidth / 2)] = 0 distFilter = (cv2.distanceTransform(distFilter, cv2.DIST_L2, 3) - np.max( cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) / ( -np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) magn = ch.sqrt(gy**2 + gx**2) * 180 / np.sqrt(2) angles = ch.arctan(gy / gx) * 180 / np.pi + 90 # meanOrient = np.linspace(0, 180, numOrient) orientations_arr = np.arange(numOrient) meanOrient = orientations_arr / numOrient * 180 fb_resttmp = 1 - ch.abs( ch.expand_dims(angles[:, :], 2) - meanOrient[1:].reshape([1, 1, numOrient - 1])) * numOrient / 180 zeros_rest = np.zeros([sy, sx, numOrient - 1, 1]) fb_rest = ch.max(ch.concatenate([fb_resttmp[:, :, :, None], zeros_rest], axis=3), axis=3) chMinOrient0 = ch.min(ch.concatenate([ ch.abs( ch.expand_dims(angles[:, :], 2) - meanOrient[0].reshape([1, 1, 1]))[:, :, :, None], ch.abs(180 - ch.expand_dims(angles[:, :], 2) - meanOrient[0].reshape([1, 1, 1]))[:, :, :, None] ], axis=3), axis=3) zeros_fb0 = np.zeros([sy, sx, 1]) fb0_tmp = ch.concatenate( [1 - chMinOrient0[:, :] * numOrient / 180, zeros_fb0], axis=2) fb_0 = ch.max(fb0_tmp, axis=2) fb = ch.concatenate([fb_0[:, :, None], fb_rest], axis=2) # fb[:,:,0] = ch.max(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180,0) # fb = 1./(1. + ch.exp(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180)) Fb = ch.expand_dims(magn, 2) * fb if drconv is None: drconv = dr_wrt_convolution(Fb[:, :, 0], distFilter) Fs_list = [ convolve2D(x=Fb[:, :, Fbi], filter=distFilter, convolve2DDr=drconv).reshape([Fb.shape[0], Fb.shape[1], 1]) for Fbi in range(numOrient) ] # Fs_list = [scipy.signal.convolve2d(Fb[:,:,Fbi], distFilter).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)] Fs = ch.concatenate(Fs_list, axis=2) # cellCols = np.arange(start=cwidth/2, stop=Fs.shape[1]-cwidth/2 , step=cwidth) # cellRows = np.arange(start=cheight/2, stop=Fs.shape[0]-cheight/2 , step=cheight) Fcells = Fs[0:Fs.shape[0]:cheight, 0:Fs.shape[1]:cwidth, :] epsilon = 1e-5 v = Fcells / ch.sqrt(ch.sum(Fcells**2) + epsilon) # v = Fcells # hog, hogim = skimage.feature.hog(imagegray, orientations=numOrient, pixels_per_cell=(cheight, cwidth), visualise=True) hog_image = HogImage(image=image, hog=Fcells, numOrient=numOrient, cwidth=cwidth, cheight=cheight) # plt.imshow(hog_image) # plt.figure() # plt.imshow(hogim) # ipdb.set_trace() return v, hog_image, drconv
def modelLogLikelihoodCh(image, template, testMask, backgroundModel, variances): logLikelihood = logPixelLikelihoodCh(image, template, testMask, backgroundModel, variances) return ch.sum(logLikelihood)
c_pre={}; if (W_Joints): k=a['Joints_Target']; j_to_consider = range(0,24) J_distances = J_reposed[j_to_consider,:] - (k[j_to_consider,:]+trans); c_pre['Joints']= J_distances*W_Joints; if(W_FMP2P): c_pre['FMP2P']= distances*W_FMP2P; if(W_Norm_B): c_pre['Norm_B']= ((result.betas)**2)*W_Norm_B; if(W_Norm_T): pose_res=result.pose.reshape(-1,3); angles=ch.sum(ch.abs(pose_res)**2,axis=-1)**(1./2) pesi = np.ones(24)*8/18 pesi[[0]] = np.ones(1)*[2] pesi[[10, 11, 22, 23, 15]] = np.ones(5)*[2./18] pesi[[6,3, 7,8]] = np.ones(4)*[5./18] pesi[[12, 15]] = np.ones(2)*[1./36] costo_T= (angles/(ch.pi*pesi))**12 c_pre['Norm_T']=costo_T*W_Norm_T; if(W_Landmarks): Tar_Land = Tar_shift[a['landmarks1'].reshape(5)-1]; SMPL_Land=result[a['landmarks2'].reshape(5)-1]; c_pre['Landmarks']= (SMPL_Land-Tar_Land)*W_Landmarks; (r,b,t)=ch.minimize(c_pre, x0 = [result.pose, result.betas,trans], method = 'dogleg', callback = None,
def normalize_rows(v): b = ch.sqrt(ch.sum(v.reshape((-1, 3))**2, axis=1)).reshape((-1, 1)) return v / b.compute_r()
def lift2Dto3D(projPtsGT, camMat, filename, img, JVis=np.ones((21, ), dtype=np.float32), trans=None, beta=None, wrist3D=None, withPoseCoeff=True, weights=1.0, relDepGT=None, rel3DCoordGT=None, rel3DCoordNormGT=None, img2DGT=None, outDir=None, poseCoeffInit=None, transInit=None, betaInit=None): loss = {} if withPoseCoeff: numComp = 30 m, poseCoeffCh, betaCh, transCh, fullposeCh = getHandModelPoseCoeffs( numComp) if poseCoeffInit is not None: poseCoeffCh[:] = poseCoeffInit if transInit is not None: transCh[:] = transInit if betaInit is not None: betaCh[:] = betaInit freeVars = [poseCoeffCh] if beta is None: freeVars = freeVars + [betaCh] loss['shape'] = 1e2 * betaCh else: betaCh[:] = beta if trans is None: freeVars = freeVars + [transCh] else: transCh[:] = trans # loss['pose'] = 0.5e2 * poseCoeffCh[3:]/stdPCACoeff[:numComp] thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints( fullposeCh[3:]) loss['constMin'] = 5e2 * thetaConstMin loss['constMax'] = 5e2 * thetaConstMax loss['invalidTheta'] = 1e3 * fullposeCh[Constraints().invalidThetaIDs] else: m, rotCh, jointsCh, transCh, betaCh = getHandModel() thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints( jointsCh) loss['constMin'] = 5e3 * thetaConstMin loss['constMax'] = 5e3 * thetaConstMax validTheta = jointsCh[Constraints().validThetaIDs[3:] - 3] freeVars = [validTheta, rotCh] if beta is None: freeVars = freeVars + [betaCh] loss['shape'] = 0.5e2 * betaCh else: betaCh[:] = beta if trans is None: freeVars = freeVars + [transCh] else: transCh[:] = trans if relDepGT is not None: relDepPred = m.J_transformed[:, 2] - m.J_transformed[0, 2] loss['relDep'] = (relDepPred - relDepGT) * weights[:, 0] * 5e1 if rel3DCoordGT is not None: rel3DCoordPred = m.J_transformed - m.J_transformed[0:1, :] loss['rel3DCoord'] = (rel3DCoordPred - rel3DCoordGT) * np.tile( weights[:, 0:1], [1, 3]) * 5e1 if rel3DCoordNormGT is not None: rel3DCoordPred = m.J_transformed[jointsMap][ 1:, :] - m.J_transformed[jointsMap][0:1, :] rel3DCoordPred = rel3DCoordPred / ch.expand_dims( ch.sqrt(ch.sum(ch.square(rel3DCoordPred), axis=1)), axis=1) loss['rel3DCoordNorm'] = ( 1. - ch.sum(rel3DCoordPred * rel3DCoordNormGT, axis=1)) * 1e4 # loss['rel3DCoordNorm'] = \ # (rel3DCoordNormGT*ch.expand_dims(ch.sum(rel3DCoordPred*rel3DCoordNormGT, axis=1), axis=1) - rel3DCoordPred) * 1e2#5e2 projPts = utilsEval.chProjectPoints(m.J_transformed, camMat, False)[jointsMap] JVis = np.tile(np.expand_dims(JVis, 1), [1, 2]) loss['joints2D'] = (projPts - projPtsGT) * JVis * weights * 1e0 loss['joints2DClip'] = clipIden(projPts - projPtsGT) * JVis * weights * 1e1 if wrist3D is not None: dep = wrist3D[2] if dep < 0: dep = -dep loss['wristDep'] = (m.J_transformed[0, 2] - dep) * 1e2 # vis_mesh(m) render = False def cbPass(_): pass # print(loss['joints'].r) print(filename) warnings.simplefilter('ignore') loss['joints2D'] = loss[ 'joints2D'] * 1e1 / weights # dont want to use confidence now if True: ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cbPass if render else cbPass, method='dogleg', options={'maxiter': 50}) else: manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) freeVars = [poseCoeffCh[:3], transCh] ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cbPass, method='dogleg', options={'maxiter': 20}) manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) freeVars = [poseCoeffCh[3:]] ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cb if render else cbPass, method='dogleg', options={'maxiter': 20}) manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) freeVars = [poseCoeffCh, transCh] if beta is None: freeVars = freeVars + [betaCh] ch.minimize({k: loss[k] for k in loss.keys() if k != 'joints2DClip'}, x0=freeVars, callback=cb if render else cbPass, method='dogleg', options={'maxiter': 20}) if False: open3dVisualize(m) else: manoVis.dump3DModel2DKpsHand(img, m, filename, camMat, est2DJoints=projPtsGT, gt2DJoints=img2DGT, outDir=outDir) # vis_mesh(m) joints3D = m.J_transformed.r[jointsMap] # print(betaCh.r) # print((relDepPred.r - relDepGT)) return joints3D, poseCoeffCh.r.copy(), betaCh.r.copy(), transCh.r.copy( ), loss['joints2D'].r.copy(), m.r.copy()
def objFun(vs): vs = np.array(vs) res = [] for vs_it, vs_i in enumerate(vs): changevars(vs_i, free_variables) import densecrf_model vis_im = np.array(renderer.indices_image == 1).copy().astype( np.bool) bound_im = renderer.boundarybool_image.astype(np.bool) segmentation, Q = densecrf_model.crfInference( rendererGT.r, vis_im, bound_im, [0.75, 0.25, 0.01], resultDir + 'imgs/crf/Q_' + str(test_i) + '_it' + str(vs_it)) vColor = color if updateColor: if np.sum(segmentation == 0) > 5: segmentRegion = segmentation == 0 vColor = np.median(rendererGT.reshape( [-1, 3])[segmentRegion.ravel()], axis=0) * 1.4 vColor = vColor / max(np.max(vColor), 1.) chVColors[:] = vColor chSHLightCoeffs[:] = lightCoeffs variances = stds**2 fgProb = ch.exp(-(renderer - rendererGT)**2 / (2 * variances)) * (1. / (stds * np.sqrt(2 * np.pi))) h = renderer.r.shape[0] w = renderer.r.shape[1] occProb = np.ones([h, w]) bgProb = np.ones([h, w]) errorFun = -ch.sum( ch.log(vis_im[:, :, None] * ((Q[0].reshape([h, w, 1]) * fgProb) + (Q[1].reshape([h, w]) * occProb + Q[2].reshape([h, w]) * bgProb)[:, :, None]) + (1 - vis_im[:, :, None]))) / (h * w) if minAppLight: options = {'disp': False, 'maxiter': 10} def cb(_): print("Error: " + str(errorFun.r)) ch.minimize({'raw': errorFun}, bounds=None, method=method, x0=free_variables_app_light, callback=cb, options=options) res = res + [errorFun.r.reshape([1, 1])] return np.vstack(res)
def get_objective(self): return ch.sum((ch.exp(-((ch.sum( (self.sph_v[self.ids0] - self.sph_v[self.ids1])**2, axis=1)) / (self.radiuss)) / 2.))**2)**.5