def laplacian_pyramid(input_objective, imshape, normalization, n_levels, as_list): if normalization is None: norm2 = lambda x : x elif normalization is 'SSE': norm2 = lambda x : x / np.sqrt(np.sum(x.r**2.)) elif normalization is 'size': norm2 = lambda x : x / x.r.size else: norm2 = normalization output_objs = [] for level in range(n_levels): blur_mtx = filter_for(imshape[0], imshape[1], imshape[2] if len(imshape)>2 else 1, kernel = GaussianKernel2D(3, 1)) blurred = MatVecMult(blur_mtx, input_objective).reshape(imshape) output_objs.append(norm2(input_objective - blurred)) halfsampler_mtx, imshape = halfsampler_for(imshape) input_objective = MatVecMult(halfsampler_mtx, blurred.ravel()).reshape(imshape) output_objs.append(norm2(input_objective).reshape(imshape)) return output_objs if as_list else reduce(lambda x, y : ch.concatenate((x.ravel(), y.ravel())), output_objs)
def ready_arguments(fname_or_dict): if not isinstance(fname_or_dict, dict): dd = pickle.load(open(fname_or_dict, 'rb'), encoding="latin1") else: dd = fname_or_dict backwards_compatibility_replacements(dd) want_shapemodel = 'shapedirs' in dd nposeparms = dd['kintree_table'].shape[1]*3 if 'trans' not in dd: dd['trans'] = np.zeros(3) if 'pose' not in dd: dd['pose'] = np.zeros(nposeparms) if 'shapedirs' in dd and 'betas' not in dd: dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']: if (s in dd) and not hasattr(dd[s], 'dterms'): dd[s] = ch.array(dd[s]) if want_shapemodel: dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template'] v_shaped = dd['v_shaped'] J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0]) J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1]) J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2]) dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose'])) else: dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose'])) return dd
def on_changed(self, which): if not hasattr(self, 'normalized'): self.normalized = True if hasattr(self, 'v') and hasattr(self, 'f'): if 'f' not in which and hasattr(self, 'faces_by_vertex') and self.faces_by_vertex.shape[0]/3 == self.v.shape[0]: self.tns.v = self.v else: # change in f or in size of v. shouldn't happen often. f = self.f IS = f.ravel() JS = np.array([range(f.shape[0])]*3).T.ravel() data = np.ones(len(JS)) IS = np.concatenate((IS*3, IS*3+1, IS*3+2)) JS = np.concatenate((JS*3, JS*3+1, JS*3+2)) data = np.concatenate((data, data, data)) sz = self.v.size self.faces_by_vertex = sp.csc_matrix((data, (IS, JS)), shape=(sz, f.size)) self.tns = Ch(lambda v : CrossProduct(TriEdges(f,1,0,v), TriEdges(f,2,0, v))) self.tns.v = self.v if self.normalized: tmp = MatVecMult(self.faces_by_vertex, self.tns) self.normals = NormalizedNx3(tmp) else: test = self.faces_by_vertex.dot(np.ones(self.faces_by_vertex.shape[1])) faces_by_vertex = sp.diags([1. / test], [0]).dot(self.faces_by_vertex).tocsc() normals = MatVecMult(faces_by_vertex, self.tns).reshape((-1,3)) normals = normals / (ch.sum(normals**2, axis=1) ** .25).reshape((-1,1)) self.normals = normals
def pixelLikelihoodRobustRegionCh(image, template, testMask, backgroundModel, layerPrior, variances): sigma = ch.sqrt(variances) mask = testMask if backgroundModel == 'FULL': mask = np.ones(image.shape[0:2]) # mask = np.repeat(mask[..., np.newaxis], 3, 2) repPriors = ch.tile(layerPrior, image.shape[0:2]) # sum = np.sum(np.log(layerPrior * scipy.stats.norm.pdf(image, location = template, scale=np.sqrt(variances) ) + (1 - repPriors))) # uniformProbs = np.ones(image.shape) imshape = image.shape from opendr.filters import filter_for from opendr.filters import GaussianKernel2D blur_mtx = filter_for(imshape[0], imshape[1], imshape[2] if len(imshape) > 2 else 1, kernel=GaussianKernel2D(3, 1)) blurred_image = MatVecMult(blur_mtx, image).reshape(imshape) blurred_template = MatVecMult(blur_mtx, template).reshape(imshape) probs = ch.exp(-(blurred_image - template)**2 / (2 * variances)) * (1. / (sigma * np.sqrt(2 * np.pi))) foregroundProbs = (probs[:, :, 0] * probs[:, :, 1] * probs[:, :, 2]) * layerPrior + (1 - repPriors) return foregroundProbs * mask + (1 - mask)
def ready_arguments(fname_or_dict, highRes): import numpy as np import pickle import chumpy as ch from chumpy.ch import MatVecMult from dataset.smpl_layer.posemapper import posemap import scipy.sparse as sp if not isinstance(fname_or_dict, dict): dd = pickle.load(open(fname_or_dict, 'rb'), encoding='latin1') # dd = pickle.load(open(fname_or_dict, 'rb')) else: dd = fname_or_dict want_shapemodel = 'shapedirs' in dd nposeparms = dd['kintree_table'].shape[1] * 3 if 'trans' not in dd: dd['trans'] = np.zeros(3) if 'pose' not in dd: dd['pose'] = np.zeros(nposeparms) if 'shapedirs' in dd and 'betas' not in dd: dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']: if (s in dd) and isinstance(dd[s], ch.ch.Ch): dd[s] = dd[s].r if want_shapemodel: dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template'] v_shaped = dd['v_shaped'] J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0]) J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1]) J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2]) dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose'])) else: dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose'])) if highRes is not None: with open(highRes, 'rb') as f: mapping, hf = pickle.load(f, encoding='latin1') num_betas = dd['shapedirs'].shape[-1] hv = mapping.dot(dd['v_template'].ravel()).reshape(-1, 3) J_reg = dd['J_regressor'].asformat('csr') dd['f'] = hf dd['v_template'] = hv dd['weights'] = np.hstack([ np.expand_dims( np.mean( mapping.dot(np.repeat(np.expand_dims(dd['weights'][:, i], -1), 3)).reshape(-1, 3) , axis=1), axis=-1) for i in range(24) ]) dd['posedirs'] = mapping.dot(dd['posedirs'].reshape((-1, 207))).reshape(-1, 3, 207) dd['shapedirs'] = mapping.dot(dd['shapedirs'].reshape((-1, num_betas))).reshape(-1, 3, num_betas) dd['J_regressor'] = sp.csr_matrix((J_reg.data, J_reg.indices, J_reg.indptr), shape=(24, hv.shape[0])) return dd
def ready_arguments(fname_or_dict): if not isinstance(fname_or_dict, dict): dd = pickle.load(open(fname_or_dict)) else: dd = fname_or_dict backwards_compatibility_replacements(dd) #insert or change the name of some index want_shapemodel = 'shapedirs' in dd nposeparms = dd['kintree_table'].shape[1]*3 #shape[1] = 24, shape[0] = 2, why shape[0] = 2? # *3 is to sca.e the kintree_table to QUATERNION if 'trans' not in dd: dd['trans'] = np.zeros(3) # why trans = (0,0,0) what trans? if 'pose' not in dd: dd['pose'] = np.zeros(nposeparms) # dim of pose is 24*3 = 72 if 'shapedirs' in dd and 'betas' not in dd: # dim of shapedirs 6890 * 3 * 10 dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) # dim of beta 10 for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']: if (s in dd) and not hasattr(dd[s], 'dterms'): dd[s] = ch.array(dd[s]) if want_shapemodel: dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template'] v_shaped = dd['v_shaped'] J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0]) J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1]) J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2]) dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose'])) else: dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose'])) return dd
def verts_decorated(trans, pose, v_template, J, weights, kintree_table, bs_style, f, bs_type=None, posedirs=None, betas=None, shapedirs=None, want_Jtr=False): for which in [trans, pose, v_template, weights, posedirs, betas, shapedirs]: if which is not None: assert ischumpy(which) v = v_template if shapedirs is not None: if betas is None: betas = chumpy.zeros(shapedirs.shape[-1]) v_shaped = v + shapedirs.dot(betas) else: v_shaped = v if posedirs is not None: v_posed = v_shaped + posedirs.dot(posemap(bs_type)(pose)) else: v_posed = v_shaped v = v_posed if sp.issparse(J): regressor = J J_tmpx = MatVecMult(regressor, v_shaped[:,0]) J_tmpy = MatVecMult(regressor, v_shaped[:,1]) J_tmpz = MatVecMult(regressor, v_shaped[:,2]) J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T else: assert(ischumpy(J)) assert(bs_style=='lbs') result, Jtr = verts_core(pose, v, J, weights, kintree_table, want_Jtr=True, xp=chumpy) tr = trans.reshape((1,3)) result = result + tr Jtr = Jtr + tr result.trans = trans result.f = f result.pose = pose result.v_template = v_template result.J = J result.weights = weights result.kintree_table = kintree_table result.bs_style = bs_style result.bs_type =bs_type if posedirs is not None: result.posedirs = posedirs result.v_posed = v_posed if shapedirs is not None: result.shapedirs = shapedirs result.betas = betas result.v_shaped = v_shaped if want_Jtr: result.J_transformed = Jtr return result
def verts_decorated_quat(trans, pose, v_template, J_regressor, weights, kintree_table, f, posedirs=None, betas=None, add_shape=True, shapedirs=None, want_Jtr=False): for which in [ trans, pose, v_template, weights, posedirs, betas, shapedirs ]: if which is not None: assert ischumpy(which) v = v_template v_shaped = v + shapedirs.dot(betas) #Add Shape of the model. quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1)[4:] shape_feat = betas[1] feat = ch.concatenate([quaternion_angles, shape_feat], axis=0) poseblends = posedirs.dot(feat) v_posed = v_shaped + poseblends J_tmpx = MatVecMult(J_regressor, v_shaped[:, 0]) J_tmpy = MatVecMult(J_regressor, v_shaped[:, 1]) J_tmpz = MatVecMult(J_regressor, v_shaped[:, 2]) J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T result, meta = verts_core(pose, v, J, weights, kintree_table, want_Jtr=True) Jtr = meta.Jtr if meta is not None else None tr = trans.reshape((1, 3)) result = result + tr Jtr = Jtr + tr result.trans = trans result.f = f result.pose = pose result.v_template = v_template result.J = J result.weights = weights result.posedirs = posedirs result.v_posed = v_posed result.shapedirs = shapedirs result.betas = betas result.v_shaped = v_shaped result.J_transformed = Jtr return result
def ready_arguments(fname_or_dict, posekey4vposed='pose', shared_args=None, chTrans=None, chBetas=None): import numpy as np import pickle import chumpy as ch from chumpy.ch import MatVecMult from mano.webuser.posemapper import posemap if not isinstance(fname_or_dict, dict): dd = pickle.load(open(fname_or_dict)) else: dd = fname_or_dict want_shapemodel = 'shapedirs' in dd nposeparms = dd['kintree_table'].shape[1] * 3 if 'trans' not in dd: dd['trans'] = np.zeros(3) if 'pose' not in dd: dd['pose'] = np.zeros(nposeparms) if 'shapedirs' in dd and 'betas' not in dd: dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) if chTrans is not None: dd['trans'] = chTrans if chTrans is not None: dd['betas'] = chBetas for s in [ 'v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J', 'fullpose', 'pose_dof' ]: if (s in dd) and not hasattr(dd[s], 'dterms'): if shared_args is not None and s in shared_args: dd[s] = shared_args[s] else: dd[s] = ch.array(dd[s]) assert (posekey4vposed in dd) if want_shapemodel: dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template'] v_shaped = dd['v_shaped'] J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0]) J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1]) J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2]) dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T dd['v_posed'] = v_shaped + dd['posedirs'].dot( posemap(dd['bs_type'])(dd[posekey4vposed])) else: dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot( posemap(dd['bs_type'])(dd[posekey4vposed])) return dd
def flow_to(self, v_next, cam_next): from chumpy.ch import MatVecMult color_image = self.r visibility = self.visibility_image pxpos = np.zeros_like(self.color_image) pxpos[:, :, 0] = np.tile(row(np.arange(self.color_image.shape[1])), (self.color_image.shape[0], 1)) pxpos[:, :, 2] = np.tile(col(np.arange(self.color_image.shape[0])), (1, self.color_image.shape[1])) visible = np.nonzero(visibility.ravel() != 4294967295)[0] num_visible = len(visible) barycentric = self.barycentric_image # map 3d to 3d JS = col(self.f[visibility.ravel()[visible]]).ravel() IS = np.tile(col(np.arange(JS.size / 3)), (1, 3)).ravel() data = barycentric.reshape((-1, 3))[visible].ravel() # replicate to xyz IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2)) JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2)) data = np.concatenate((data, data, data)) verts_to_visible = sp.csc_matrix((data, (IS, JS)), shape=(np.max(IS) + 1, self.v.r.size)) v_old = self.camera.v cam_old = self.camera if cam_next is None: cam_next = self.camera self.camera.v = MatVecMult(verts_to_visible, self.v.r) r1 = self.camera.r.copy() self.camera = cam_next self.camera.v = MatVecMult(verts_to_visible, v_next) r2 = self.camera.r.copy() n_channels = self.camera.shape[1] flow = r2 - r1 flow_im = np.zeros( (self.frustum['height'], self.frustum['width'], n_channels)).reshape( (-1, n_channels)) flow_im[visible] = flow flow_im = flow_im.reshape( (self.frustum['height'], self.frustum['width'], n_channels)) self.camera = cam_old self.camera.v = v_old return flow_im
def SecondFundamentalForm(v, f): from chumpy import hstack, vstack from chumpy.linalg import Pinv nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape( (-1, 3)) b0 = VertNormals(f=f, v=v) b1 = NormalizedNx3(CrossProduct(b0, nbrs - v)).reshape((-1, 3)) b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1, 3)) cnct = get_vert_connectivity(np.asarray(v), f) ffs = [] for i in range(v.size / 3): nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row( v[i]) us = nbrs.dot(b2[i]) vs = nbrs.dot(b1[i]) hs = nbrs.dot(b0[i]) coeffs = Pinv( hstack((col((us * .5)**2), col(us * vs), col( (vs * .5)**2)))).dot(hs) ffs.append(row(coeffs)) # if i == 3586: # import pdb; pdb.set_trace() ffs = vstack(ffs) return ffs
def ready_arguments(fname_or_dict, posekey4vposed='pose'): import numpy as np # import cPickle as pickle import pickle import chumpy as ch from chumpy.ch import MatVecMult from posemapper import posemap if not isinstance(fname_or_dict, dict): # dd = pickle.load(open(fname_or_dict)) dd = pickle.load(open(fname_or_dict, 'rb'), encoding='latin1') else: dd = fname_or_dict want_shapemodel = 'shapedirs' in dd nposeparms = dd['kintree_table'].shape[1] * 3 if 'trans' not in dd: dd['trans'] = np.zeros(3) if 'pose' not in dd: dd['pose'] = np.zeros(nposeparms) if 'shapedirs' in dd and 'betas' not in dd: dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) for s in [ 'v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J' ]: if (s in dd) and not hasattr(dd[s], 'dterms'): dd[s] = ch.array(dd[s]) assert (posekey4vposed in dd) if want_shapemodel: dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template'] v_shaped = dd['v_shaped'] J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0]) J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1]) J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2]) dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T dd['v_posed'] = v_shaped + dd['posedirs'].dot( posemap(dd['bs_type'])(dd[posekey4vposed])) else: dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot( posemap(dd['bs_type'])(dd[posekey4vposed])) return dd
def on_changed(self, which): if not hasattr(self, 'kernel'): self.kernel = self.default_kernel.copy() if 'im_shape' in which: sh = self.im_shape self.transf_mtx, self._output_shape = filter_for_nopadding(sh, self.kernel) if True: # self.want_downsampling: <-- setting want_downsampling to "False" is broken halfsampler, self._output_shape = halfsampler_for(self._output_shape) self.transf_mtx = halfsampler.dot(self.transf_mtx) self.add_dterm('transform', MatVecMult(mtx=self.transf_mtx)) if 'px' in which: self.transform.vec = self.px
def blurredDiff(self): edges = self.renderer.boundarybool_image imshape = self.renderer.shape rgbEdges = np.tile(edges.reshape([imshape[0], imshape[1], 1]), [1, 1, 3]).astype(np.bool) from opendr.filters import filter_for from opendr.filters import GaussianKernel2D blur_mtx = filter_for(imshape[0], imshape[1], imshape[2] if len(imshape) > 2 else 1, kernel=GaussianKernel2D(3, 1)) blurred_diff = MatVecMult(blur_mtx, self.renderer - self.rendererGT).reshape(imshape) return blurred_diff[rgbEdges]
def on_changed(self, which): if 'w' in which or 'A' in which: A = np.abs(self.A.tocoo()) self.wmat = A.tocsc().dot(sp.diags([self.w], [0])) A9_data = np.tile(A.data / 2., 9) A9_row = np.hstack([A.row * 9 + i for i in range(9)]) A9_col = np.hstack([A.col * 9 + i for i in range(9)]) self.A9 = sp.csc_matrix((A9_data, (A9_row, A9_col)), shape=(A.shape[0] * 9, A.shape[1] * 9)) #wmatcoo = self.wmat.tocoo() #wmat3_data = np.tile(wmatcoo.data, 3) #wmat3_row = np.hstack([wmatcoo.row*3 + i for i in range(3)]) #wmat3_col = np.hstack([wmatcoo.col*3 + i for i in range(3)]) #self.wmat3 = sp.csc_matrix((wmat3_data, (wmat3_row, wmat3_col)), # shape=(wmatcoo.shape[0]*3, # wmatcoo.shape[1]*3)) if 'reg_e' in which or 'model_e' in which: T = np.einsum('ij,jk->ikj', self.model_e.T, self.reg_e).reshape(9, -1) # compute rotation per vertex rv through SVD T_per_v = (self.wmat.dot(T.T)).reshape(-1, 3, 3) u_s_v = np.linalg.svd(T_per_v) col3 = np.asarray([1., 1., -1.]) # negate third column of u if det is negative # The real rotation, according to Sorkine, would be v.T.dot(u.T), # because numpy returns usv instead of usv.T. However, we're applying # e.dot(rot), so we need rot.T which is u.dot(v) self.rv = np.asarray([ u.dot(v) if np.linalg.det(u.dot(v)) > 0 # np.prod(s) > 0# else (u * col3).dot(v) for u, s, v in zip(*u_s_v) ]) # average vertex rotations for each edge self.rf = MatVecMult(mtx=self.A9.T, vec=self.rv.ravel()).reshape(-1, 3) w3 = ch.tile(self.w, (3, 1)).T.ravel() self.err = (LoopProd(rot=self.rf, edge=self.model_e).ravel() - self.reg_e.ravel()) * w3
def verts_decorated(trans, pose, v_template, J, weights, kintree_table, bs_style, f, bs_type=None, posedirs=None, betas=None, shapedirs=None, want_Jtr=False): for which in [ trans, pose, v_template, weights, posedirs, betas, shapedirs ]: if which is not None: assert ischumpy(which) v = v_template print "********" print "betas : " + str(betas.shape) print "posedirs : " + str(posedirs.shape) print "shapedirs : " + str(shapedirs.shape) if shapedirs is not None: if betas is None: betas = chumpy.zeros(shapedirs.shape[-1]) v_shaped = v + shapedirs.dot(betas) #print shapedirs.dot(betas).shape #v_shaped = np.copy(v) #for i in range(0,shapedirs.shape[0]): # v_shaped[i,:] = v[i,:] + shapedirs[i,:,:].dot(betas) #print v_shaped[1,:] #xx = np.random.rand(4,4,4) #print xx #print xx[0,:,1] #print (np.array(xx)).tolist() #print (np.array(xx)).tolist()[1][1][1] else: v_shaped = v if posedirs is not None: print pose.shape print posemap(bs_type)(pose).shape v_posed = v_shaped + posedirs.dot(posemap(bs_type)(pose)) #v_posed = v_shaped else: v_posed = v_shaped v = v_posed # IS this needed if sp.issparse(J): regressor = J print regressor.shape J_tmpx = MatVecMult(regressor, v_shaped[:, 0]) J_tmpy = MatVecMult(regressor, v_shaped[:, 1]) J_tmpz = MatVecMult(regressor, v_shaped[:, 2]) J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T #XX = np.array(regressor) * np.array(v_shaped[:,0]) #print "--" #print XX.shape #print XX #print v_shaped[200,2] #print XX[1] else: assert (ischumpy(J)) assert (bs_style == 'lbs') result, Jtr = lbs.verts_core(pose, v, J, weights, kintree_table, want_Jtr=True, xp=chumpy) tr = trans.reshape((1, 3)) result = result + tr Jtr = Jtr + tr result.trans = trans result.f = f result.pose = pose result.v_template = v_template result.J = J result.weights = weights result.kintree_table = kintree_table result.bs_style = bs_style result.bs_type = bs_type if posedirs is not None: result.posedirs = posedirs result.v_posed = v_posed if shapedirs is not None: result.shapedirs = shapedirs result.betas = betas result.v_shaped = v_shaped if want_Jtr: result.J_transformed = Jtr #print Jtr return result
nposeparms = dd['kintree_table'].shape[1]*3 dd['trans'] = np.zeros(3) dd['pose'] = np.zeros(nposeparms) dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'betas', 'J']: if (s in dd) and not hasattr(dd[s], 'dterms'): dd[s] = ch.array(dd[s]) else: print type(dd[s]) dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template'] v_shaped = dd['v_shaped'] J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0]) J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1]) J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2]) dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T if dd['pose'].ndim != 2 or p.shape[1] != 3: p = dd['pose'].reshape((-1,3)) p = p[1:] c= ch.concatenate([(Rodrigues(pp)-ch.eye(3)).ravel() for pp in p]).ravel() dd['v_posed'] = v_shaped + dd['posedirs'].dot(c) args = { 'pose': dd['pose'], 'v': dd['v_posed'], 'J': dd['J'],
def optimize_on_DensePose(dp_iuv, op_j2d, w_j2d, model, cam, prior, body_orient, body_trans, body_init, n_betas=10, viz=False, imageid=1): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ outPath = '/home/xiul/databag/dbfusion/record0/smplify/vid{}'.format( imageid) if not os.path.isdir(outPath): os.mkdir(outPath) t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: tarIUV = ch.array(dp_iuv) # initialize the shape to the mean shape in the SMPL training set betas = ch.zeros(n_betas) # initialize the pose by using the optimized body orientation and the # pose prior init_pose = np.hstack((body_orient, prior.weights.dot(prior.means))) #init_pose = np.hstack((body_orient, body_init)) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.array(body_trans), pose=ch.array(init_pose), v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=model.posedirs) J_tmpx = MatVecMult(J_reg, sv[:, 0]) J_tmpy = MatVecMult(J_reg, sv[:, 1]) J_tmpz = MatVecMult(J_reg, sv[:, 2]) Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T #Jtr = J_reg.dot(sv) cam.v = Jtr op_j2d = op_j2d * 1280 / 1920 w = 1280 h = 720 reIUV = render_model(sv, model.f, w, h, cam, near=0.5, far=25, vc=dp_colors, img=None) #gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None): obj_j2d = lambda w, sigma: (w * w_j2d.reshape((-1, 1)) * GMOf( (op_j2d - cam), sigma)) #input_objective, imshape, normalization, n_levels, as_list err = tarIUV - reIUV obj_dense = 100 * gaussian_pyramid(err, n_levels=6, normalization='SSE') # data term: distance between observed and estimated joints in 2D # obj_dense = lambda w, sigma: ( # w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma)) # mixture of gaussians pose prior pprior = lambda w: w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 my_exp = lambda x: alpha * ch.exp(x) obj_angle = lambda w: w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) if viz: import matplotlib.pyplot as plt from time import gmtime, strftime def on_step(cstep): """Create visualization.""" plt.figure(1, figsize=(10, 10)) plt.clf() plt.subplot(1, 2, 1) # show optimized joints in 2D tmp_img = reIUV.r.copy() tmp_tar = tarIUV.copy() plt.imshow(tmp_img) for j1, j2 in zip(cam.r, op_j2d): plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r') plt.subplot(1, 2, 2) plt.imshow(tmp_tar) plt.draw() plt.savefig(os.path.join( outPath, '{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))), bbox_inches='tight') on_step(stepI) else: on_step = None # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78, 4.78, 4.78, 4.78], [1e2, 5 * 1e1, 1e1, .5 * 1e1, 5, 5, 5]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} if stage >= 4: objs['dense'] = obj_dense if stage <= 4: objs['j2d'] = obj_j2d(1, 100) else: objs['j2d'] = obj_j2d(0.1, 100) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas ch.minimize(objs, x0=[sv.betas, sv.pose], method='dogleg', callback=on_step, options={ 'maxiter': 10000, 'e_3': .001, 'disp': 1 }) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if viz: plt.ioff()
def optimize_on_DensePose(ori_img, tar_img, dp_iuv, op_j2d, w_j2d, model, cam, cam_old, prior, init_trans, init_pose, init_betas, n_betas=10, viz=False, imageid=1): """Fit the model to the given set of joints, given the estimated camera :param j2d: 14x2 array of CNN joints :param model: SMPL model :param cam: estimated camera :param img: h x w x 3 image :param prior: mixture of gaussians pose prior :param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit :param body_orient: 3D vector, initialization for the body orientation :param n_betas: number of shape coefficients considered during optimization :param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term :param conf: 14D vector storing the confidence values from the CNN :param viz: boolean, if True enables visualization during optimization :returns: a tuple containing the optimized model, its joints projected on image space, the camera translation """ outPath = '/home/xiul/databag/net_images/smplify/vid_{:06d}_puredense'.format( imageid) if not os.path.isdir(outPath): os.mkdir(outPath) dp_iuv_tar = dp_iuv.copy() width = dp_iuv_tar.shape[1] height = dp_iuv_tar.shape[0] dp_iuv_weight = np.zeros(dp_iuv_tar.shape) + 10 dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 0] = 10 dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 1] = 10 dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 2] = 15 dp_iuv_weight = ch.array(dp_iuv_weight) t0 = time() # define the mapping LSP joints -> SMPL joints # cids are joints ids for LSP: tarIUV = ch.array(dp_iuv) # initialize the shape to the mean shape in the SMPL training set betas = ch.array(init_betas) # initialize the pose by using the optimized body orientation and the # pose prior pose = ch.array(init_pose) #init_pose = np.hstack((body_orient, body_init)) # instantiate the model: # verts_decorated allows us to define how many # shape coefficients (directions) we want to consider (here, n_betas) sv = verts_decorated(trans=ch.array(init_trans), pose=pose, v_template=model.v_template, J=model.J_regressor, betas=betas, shapedirs=model.shapedirs[:, :, :n_betas], weights=model.weights, kintree_table=model.kintree_table, bs_style=model.bs_style, f=model.f, bs_type=model.bs_type, posedirs=None) J_tmpx = MatVecMult(J_reg, sv[:, 0]) J_tmpy = MatVecMult(J_reg, sv[:, 1]) J_tmpz = MatVecMult(J_reg, sv[:, 2]) Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T #Jtr = J_reg.dot(sv) cam.v = Jtr reIUV = render_model(sv, model.f, width, height, cam, near=0.5, far=25, vc=dp_colors, img=None) reModel = render_model(sv, model.f, width, height, cam, near=0.5, far=25, vc=None, img=None) fullModel = render_model(sv, model.f, ori_img.shape[1], ori_img.shape[0], cam_old, near=0.5, far=25, vc=None, img=None) #gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None): def obj_j2d(w, sigma): return (w * w_j2d.reshape((-1, 1)) * GMOf((op_j2d - cam), sigma)) #input_objective, imshape, normalization, n_levels, as_list err = (tarIUV - reIUV) * dp_iuv_weight obj_dense = 100 * gaussian_pyramid(err, n_levels=4, normalization='SSE') #obj_dense = gaussian_pyramid(err, n_levels=4, normalization=SSE) #obj_dense = 10000*err # data term: distance between observed and estimated joints in 2D # obj_dense = lambda w, sigma: ( # w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma)) # mixture of gaussians pose prior def pprior(w): return w * prior(sv.pose) # joint angles pose prior, defined over a subset of pose parameters: # 55: left elbow, 90deg bend at -np.pi/2 # 58: right elbow, 90deg bend at np.pi/2 # 12: left knee, 90deg bend at np.pi/2 # 15: right knee, 90deg bend at np.pi/2 alpha = 10 def my_exp(x): return alpha * ch.exp(x) def obj_angle(w): return w * ch.concatenate([ my_exp(sv.pose[55]), my_exp(-sv.pose[58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15]) ]) def viz_func(stage_num): plt.figure(1, figsize=(10, 10)) plt.clf() plt.subplot(2, 3, 1) # show optimized joints in 2D tmp_img = reIUV.r.copy() tmp_tar = tarIUV.copy() tmp_model = reModel.r.copy() full_model = fullModel.r.copy() # w = tmp_tar.shape[1] # h = tmp_tar.shape[0] plt.imshow(tar_img) plt.imshow(tmp_img, alpha=0.5) for j1, j2, w_ts in zip(cam.r, op_j2d, w_j2d): if (w_ts > 0): plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r') plt.xlim([0, width]) plt.ylim([height, 0]) plt.subplot(2, 3, 2) plt.imshow(tar_img) plt.imshow(tmp_tar, alpha=0.5) plt.xlim([0, width]) plt.ylim([height, 0]) plt.subplot(2, 3, 3) plt.imshow(tar_img) tmp_model_alpha = np.ones((tmp_model.shape[0], tmp_model.shape[1])) tmp_model_alpha[tmp_model[:, :, 0] < 1e-2] = 0 tmp_model = np.dstack((tmp_model, tmp_model_alpha)) plt.imshow(tmp_model) plt.xlim([0, width]) plt.ylim([height, 0]) plt.subplot(2, 1, 2) plt.imshow(full_model) plt.draw() plt.savefig(os.path.join(outPath, 'stage-{}.png'.format(stage_num)), bbox_inches='tight') full_model_int = full_model.copy() full_model_int *= 255 full_model_int = full_model_int.astype(np.uint8) cv2.imwrite(os.path.join(outPath, 'model-{}.png'.format(stage_num)), full_model_int) out_params = { 'pose': sv.pose.r, 'shape': sv.betas.r, 'trans': sv.trans.r } with open(os.path.join(outPath, 'param-{}.pkl'.format(stage_num)), 'w') as fio: pickle.dump(out_params, fio, pickle.HIGHEST_PROTOCOL) viz_func(0) #if viz: if viz and False: def on_step(cstep): """Create visualization.""" # TODO this function is in vis_func #plt.savefig(os.path.join(outPath,'{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),bbox_inches='tight') on_step(stepI) else: on_step = None # weight configuration used in the paper, with joints + confidence values from the CNN # (all the weights used in the code were obtained via grid search, see the paper for more details) # the first list contains the weights for the pose priors, # the second list contains the weights for the shape prior # opt_weights = zip([4.78, 3.78, 2.78, 1.78], # [5, 5, 5, 5], # [1, 0.25, 0.10, 0]) opt_weights = zip([4.78, 4.78, 4.78, 4.78, 4.78], [50, 50, 50, 50, 50], [0.1, 0.1, 0.5, 0.25, 0.1]) # run the optimization in 4 stages, progressively decreasing the # weights for the priors for stage, (w, wbetas, w_joints) in enumerate(opt_weights): _LOGGER.info('stage %01d', stage) objs = {} #if stage >= 1: objs['dense'] = obj_dense objs['j2d'] = obj_j2d(w_joints, 50) objs['pose'] = pprior(w) objs['pose_exp'] = obj_angle(0.317 * w) objs['betas'] = wbetas * betas ch.minimize(objs, x0=[sv.betas, sv.pose, sv.trans], method='dogleg', callback=on_step, options={ 'maxiter': 10000, 'e_3': .0001, 'disp': 1 }) viz_func(stage + 1) t1 = time() _LOGGER.info('elapsed %.05f', (t1 - t0)) if viz and False: plt.ioff()
def optimize_smal(fposes, ftrans, fbetas, model, cams, segs, imgs, landmarks, landmarks_names, key_vids, symIdx=None, frameId=0, opt_model_dir=None, save_name=None, COMPUTE_OPT=True, img_paths=None, img_offset=None, img_scales=None): mesh_v_opt_save_path = join(opt_model_dir, 'mesh_v_opt_no_mc_' + str(frameId) + '.ply') mesh_v_opt_mc_save_path = join(opt_model_dir, 'mesh_v_opt_' + str(frameId) + '.ply') mesh_init_save_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') nViews = len(fposes) if not COMPUTE_OPT: if not exists(opt_model_dir): makedirs(opt_model_dir) dv = 0 compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_init_save_path, fposes, ftrans, fbetas, '_no_refine', cams, imgs, segs, img_paths, img_offset, img_scales) return # Write the initial mesh np_betas = np.zeros_like(model.betas) np_betas[:len(fbetas[0])] = fbetas[0] tmp = verts_decorated(v_template=model.v_template, pose=ch.zeros_like(model.pose.r), trans=ch.zeros_like(model.trans), J=model.J_regressor, kintree_table=model.kintree_table, betas=ch.array(np_betas), weights=model.weights, posedirs=model.posedirs, shapedirs=model.shapedirs, bs_type='lrotmin', bs_style='lbs', f=model.f) tmp_mesh = Mesh(v=tmp.r, f=tmp.f) tmp_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply') tmp_mesh.write_ply(tmp_path) del tmp assert (nViews == len(cams)) assert (nViews == len(segs)) assert (nViews == len(imgs)) # Define a displacement vector. We set a small non zero displacement as initialization dv = ch.array(np.random.rand(model.r.shape[0], 3) / 1000.) # Cell structure for ARAP f = model.f _, A3, A = edgesIdx(nV=dv.shape[0], f=f, save_dir='.', name='smal') wedge = wedges(A3, dv) s = np.zeros_like(dv) arap = ARAP(reg_e=MatVecMult(A3.T, model.ravel() + dv.ravel()).reshape(-1, 3), model_e=MatVecMult(A3.T, model.ravel()).reshape(-1, 3), w=wedge, A=A) k_arap = settings['ref_k_arap_per_view'] * nViews for weight, part in zip(settings['ref_W_arap_values'], settings['ref_W_arap_parts']): k_arap, W_per_vertex = get_arap_part_weights( A, k_arap, [part], [weight]) #, animal_name) # was only Head W = np.zeros((W_per_vertex.shape[0], 3)) for i in range(3): W[:, i] = W_per_vertex k_lap = settings['ref_k_lap'] * nViews * W k_sym = settings['ref_k_sym'] * nViews k_keyp = settings['ref_k_keyp_weight'] * nViews # Load already computed mesh if not exists(opt_model_dir): makedirs(opt_model_dir) shape_model, compute = load_shape_models(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas) mv = None # Remove inside mouth faces ''' if settings['ref_remove_inside_mouth']: # Giraffe faces_orig = shape_model[0].f.copy() im_v = im_up_v + im_down_v idx = [np.where(model.f == ix)[0] for ix in im_v] idx = np.concatenate(idx).ravel() for i in range(nViews): shape_model[i].f = np.delete(shape_model[i].f, idx, 0) ''' if compute: objs = {} FIX_CAM = True free_variables = [] kp_weights = k_keyp * np.ones((landmarks[0].shape[0], 1)) print('removing shoulders, often bad annotated') kp_weights[landmarks_names.index('leftShoulder'), :] *= 0 kp_weights[landmarks_names.index('rightShoulder'), :] *= 0 objs_pose = None j2d = None #k_silh_term = settings['ref_k_silh_term'] k_m2s = settings['ref_k_m2s'] k_s2m = settings['ref_k_s2m'] objs, params_, j2d = set_pose_objs(shape_model, cams, landmarks, key_vids, kp_weights=kp_weights, FIX_CAM=FIX_CAM, ONLY_KEYP=True, OPT_SHAPE=False) if np.any(k_arap) != 0: objs['arap'] = k_arap * arap if k_sym != 0: objs['sym_0'] = k_sym * (ch.abs(dv[:, 0] - dv[symIdx, 0])) objs['sym_1'] = k_sym * (ch.abs(dv[:, 1] + dv[symIdx, 1] - 0.00014954)) objs['sym_2'] = k_sym * (ch.abs(dv[:, 2] - dv[symIdx, 2])) if np.any(k_lap) != 0: lap_op = np.asarray( laplacian(Mesh(v=dv, f=shape_model[0].f)).todense()) objs['lap'] = k_lap * ch.dot(lap_op, dv) mv = None mv2 = MeshViewers(shape=(1, nViews)) #None vc = np.ones_like(dv) dv_r = fit_silhouettes_pyramid_opt(objs, shape_model, dv, segs, cams, j2d=j2d, weights=1., mv=mv, imgs=imgs, s2m_weights=k_s2m, m2s_weights=k_m2s, max_iter=100, free_variables=free_variables, vc=vc, symIdx=symIdx, mv2=mv2, objs_pose=objs_pose) # Save result image for i in range(nViews): img_res = render_mesh(Mesh(shape_model[i].r, shape_model[i].f), imgs[i].shape[1], imgs[i].shape[0], cams[i], img=imgs[i], world_frame=True) img_result = np.hstack((imgs[i], img_res * 255.)) save_img_path = save_name[i].replace('.pkl', '_v_opt.png') cv2.imwrite(save_img_path, img_result) shape_model[0].pose[:] = 0 shape_model[0].trans[:] = 0 V = shape_model[0].r.copy() vm = V[symIdx, :].copy() vm[:, 1] = -1 * vm[:, 1] V2 = (V + vm) / 2.0 mesh_out = Mesh(v=V2, f=shape_model[0].f) mesh_out.show() mesh_out.write_ply(mesh_v_opt_save_path) save_dv_data_path = mesh_v_opt_save_path.replace('.ply', '_dv.pkl') dv_data = {'betas': shape_model[0].betas.r, 'dv': dv_r} pkl.dump(dv_data, open(save_dv_data_path, 'wb')) compute_texture(nViews, opt_model_dir, dv, model, frameId, mesh_v_opt_save_path, fposes, ftrans, fbetas, '_non_opt', cams, imgs, segs, img_paths, img_offset, img_scales) return
def verts_decorated_quat(trans, pose, v_template, J, weights, kintree_table, f, posedirs=None, betas=None, add_shape=True, shapedirs=None, want_Jtr=False): for which in [ trans, pose, v_template, weights, posedirs, betas, shapedirs ]: if which is not None: assert ischumpy(which) v = v_template if shapedirs is not None: if betas is None: betas = chumpy.zeros(shapedirs.shape[-1]) if add_shape: v_shaped = v + shapedirs.dot(betas) #Add Shape of the model. else: v_shaped = v else: v_shaped = v quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1) shape_feat = betas[1] feat = ch.concatenate([quaternion_angles, shape_feat], axis=0) feat = quaternion_angles poseblends = posedirs.dot(feat) v_posed = v_shaped + poseblends v = v_posed regressor = J J_tmpx = MatVecMult(regressor, v_shaped[:, 0]) J_tmpy = MatVecMult(regressor, v_shaped[:, 1]) J_tmpz = MatVecMult(regressor, v_shaped[:, 2]) J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T result, meta = verts_core(pose, v, J, weights, kintree_table, want_Jtr=True) Jtr = meta.Jtr if meta is not None else None tr = trans.reshape((1, 3)) result = result + tr Jtr = Jtr + tr result.trans = trans result.f = f result.pose = pose result.v_template = v_template result.J = J result.weights = weights result.kintree_table = kintree_table result.poseblends = poseblends result.quats = quaternion_angles if meta is not None: for field in ['Jtr', 'A', 'A_global', 'A_weighted']: if (hasattr(meta, field)): setattr(result, field, getattr(meta, field)) if posedirs is not None: result.posedirs = posedirs result.v_posed = v_posed if shapedirs is not None: result.shapedirs = shapedirs result.betas = betas result.v_shaped = v_shaped if want_Jtr: result.J_transformed = Jtr result.poseblends = poseblends return result