def SecondFundamentalForm(v, f):
from chumpy import hstack, vstack
from chumpy.linalg import Pinv
nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape(
(-1, 3))
b0 = VertNormals(f=f, v=v)
b1 = NormalizedNx3(CrossProduct(b0, nbrs - v)).reshape((-1, 3))
b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1, 3))
cnct = get_vert_connectivity(np.asarray(v), f)
ffs = []
for i in range(v.size / 3):
nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(
v[i])
us = nbrs.dot(b2[i])
vs = nbrs.dot(b1[i])
hs = nbrs.dot(b0[i])
coeffs = Pinv(
hstack((col((us * .5)**2), col(us * vs), col(
(vs * .5)**2)))).dot(hs)
ffs.append(row(coeffs))
# if i == 3586:
# import pdb; pdb.set_trace()
ffs = vstack(ffs)
return ffs
def ready_arguments(fname_or_dict):
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict))
else:
dd = fname_or_dict
backwards_compatibility_replacements(dd) #insert or change the name of some index
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1]*3 #shape[1] = 24, shape[0] = 2, why shape[0] = 2?
# *3 is to sca.e the kintree_table to QUATERNION
if 'trans' not in dd:
dd['trans'] = np.zeros(3) # why trans = (0,0,0) what trans?
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms) # dim of pose is 24*3 = 72
if 'shapedirs' in dd and 'betas' not in dd: # dim of shapedirs 6890 * 3 * 10
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) # dim of beta 10
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
return dd
def ready_arguments(fname_or_dict):
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict, 'rb'), encoding="latin1")
else:
dd = fname_or_dict
backwards_compatibility_replacements(dd)
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1]*3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
return dd
def ready_arguments(fname_or_dict, highRes):
import numpy as np
import pickle
import chumpy as ch
from chumpy.ch import MatVecMult
from dataset.smpl_layer.posemapper import posemap
import scipy.sparse as sp
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict, 'rb'), encoding='latin1')
# dd = pickle.load(open(fname_or_dict, 'rb'))
else:
dd = fname_or_dict
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1] * 3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and isinstance(dd[s], ch.ch.Ch):
dd[s] = dd[s].r
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
if highRes is not None:
with open(highRes, 'rb') as f:
mapping, hf = pickle.load(f, encoding='latin1')
num_betas = dd['shapedirs'].shape[-1]
hv = mapping.dot(dd['v_template'].ravel()).reshape(-1, 3)
J_reg = dd['J_regressor'].asformat('csr')
dd['f'] = hf
dd['v_template'] = hv
dd['weights'] = np.hstack([
np.expand_dims(
np.mean(
mapping.dot(np.repeat(np.expand_dims(dd['weights'][:, i], -1), 3)).reshape(-1, 3)
, axis=1),
axis=-1)
for i in range(24)
])
dd['posedirs'] = mapping.dot(dd['posedirs'].reshape((-1, 207))).reshape(-1, 3, 207)
dd['shapedirs'] = mapping.dot(dd['shapedirs'].reshape((-1, num_betas))).reshape(-1, 3, num_betas)
dd['J_regressor'] = sp.csr_matrix((J_reg.data, J_reg.indices, J_reg.indptr), shape=(24, hv.shape[0]))
return dd
def on_changed(self, which):
if not hasattr(self, 'normalized'):
self.normalized = True
if hasattr(self, 'v') and hasattr(self, 'f'):
if 'f' not in which and hasattr(self, 'faces_by_vertex') and self.faces_by_vertex.shape[0]/3 == self.v.shape[0]:
self.tns.v = self.v
else: # change in f or in size of v. shouldn't happen often.
f = self.f
IS = f.ravel()
JS = np.array([range(f.shape[0])]*3).T.ravel()
data = np.ones(len(JS))
IS = np.concatenate((IS*3, IS*3+1, IS*3+2))
JS = np.concatenate((JS*3, JS*3+1, JS*3+2))
data = np.concatenate((data, data, data))
sz = self.v.size
self.faces_by_vertex = sp.csc_matrix((data, (IS, JS)), shape=(sz, f.size))
self.tns = Ch(lambda v : CrossProduct(TriEdges(f,1,0,v), TriEdges(f,2,0, v)))
self.tns.v = self.v
if self.normalized:
tmp = MatVecMult(self.faces_by_vertex, self.tns)
self.normals = NormalizedNx3(tmp)
else:
test = self.faces_by_vertex.dot(np.ones(self.faces_by_vertex.shape[1]))
faces_by_vertex = sp.diags([1. / test], [0]).dot(self.faces_by_vertex).tocsc()
normals = MatVecMult(faces_by_vertex, self.tns).reshape((-1,3))
normals = normals / (ch.sum(normals**2, axis=1) ** .25).reshape((-1,1))
self.normals = normals
def pixelLikelihoodRobustRegionCh(image, template, testMask, backgroundModel,
layerPrior, variances):
sigma = ch.sqrt(variances)
mask = testMask
if backgroundModel == 'FULL':
mask = np.ones(image.shape[0:2])
# mask = np.repeat(mask[..., np.newaxis], 3, 2)
repPriors = ch.tile(layerPrior, image.shape[0:2])
# sum = np.sum(np.log(layerPrior * scipy.stats.norm.pdf(image, location = template, scale=np.sqrt(variances) ) + (1 - repPriors)))
# uniformProbs = np.ones(image.shape)
imshape = image.shape
from opendr.filters import filter_for
from opendr.filters import GaussianKernel2D
blur_mtx = filter_for(imshape[0],
imshape[1],
imshape[2] if len(imshape) > 2 else 1,
kernel=GaussianKernel2D(3, 1))
blurred_image = MatVecMult(blur_mtx, image).reshape(imshape)
blurred_template = MatVecMult(blur_mtx, template).reshape(imshape)
probs = ch.exp(-(blurred_image - template)**2 /
(2 * variances)) * (1. / (sigma * np.sqrt(2 * np.pi)))
foregroundProbs = (probs[:, :, 0] * probs[:, :, 1] *
probs[:, :, 2]) * layerPrior + (1 - repPriors)
return foregroundProbs * mask + (1 - mask)
def laplacian_pyramid(input_objective, imshape, normalization, n_levels, as_list):
if normalization is None:
norm2 = lambda x : x
elif normalization is 'SSE':
norm2 = lambda x : x / np.sqrt(np.sum(x.r**2.))
elif normalization is 'size':
norm2 = lambda x : x / x.r.size
else:
norm2 = normalization
output_objs = []
for level in range(n_levels):
blur_mtx = filter_for(imshape[0], imshape[1], imshape[2] if len(imshape)>2 else 1, kernel = GaussianKernel2D(3, 1))
blurred = MatVecMult(blur_mtx, input_objective).reshape(imshape)
output_objs.append(norm2(input_objective - blurred))
halfsampler_mtx, imshape = halfsampler_for(imshape)
input_objective = MatVecMult(halfsampler_mtx, blurred.ravel()).reshape(imshape)
output_objs.append(norm2(input_objective).reshape(imshape))
return output_objs if as_list else reduce(lambda x, y : ch.concatenate((x.ravel(), y.ravel())), output_objs)
def verts_decorated(trans, pose,
v_template, J, weights, kintree_table, bs_style, f,
bs_type=None, posedirs=None, betas=None, shapedirs=None, want_Jtr=False):
for which in [trans, pose, v_template, weights, posedirs, betas, shapedirs]:
if which is not None:
assert ischumpy(which)
v = v_template
if shapedirs is not None:
if betas is None:
betas = chumpy.zeros(shapedirs.shape[-1])
v_shaped = v + shapedirs.dot(betas)
else:
v_shaped = v
if posedirs is not None:
v_posed = v_shaped + posedirs.dot(posemap(bs_type)(pose))
else:
v_posed = v_shaped
v = v_posed
if sp.issparse(J):
regressor = J
J_tmpx = MatVecMult(regressor, v_shaped[:,0])
J_tmpy = MatVecMult(regressor, v_shaped[:,1])
J_tmpz = MatVecMult(regressor, v_shaped[:,2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
else:
assert(ischumpy(J))
assert(bs_style=='lbs')
result, Jtr = verts_core(pose, v, J, weights, kintree_table, want_Jtr=True, xp=chumpy)
tr = trans.reshape((1,3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.kintree_table = kintree_table
result.bs_style = bs_style
result.bs_type =bs_type
if posedirs is not None:
result.posedirs = posedirs
result.v_posed = v_posed
if shapedirs is not None:
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
if want_Jtr:
result.J_transformed = Jtr
return result
def verts_decorated_quat(trans,
pose,
v_template,
J_regressor,
weights,
kintree_table,
f,
posedirs=None,
betas=None,
add_shape=True,
shapedirs=None,
want_Jtr=False):
for which in [
trans, pose, v_template, weights, posedirs, betas, shapedirs
]:
if which is not None:
assert ischumpy(which)
v = v_template
v_shaped = v + shapedirs.dot(betas) #Add Shape of the model.
quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1)[4:]
shape_feat = betas[1]
feat = ch.concatenate([quaternion_angles, shape_feat], axis=0)
poseblends = posedirs.dot(feat)
v_posed = v_shaped + poseblends
J_tmpx = MatVecMult(J_regressor, v_shaped[:, 0])
J_tmpy = MatVecMult(J_regressor, v_shaped[:, 1])
J_tmpz = MatVecMult(J_regressor, v_shaped[:, 2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
result, meta = verts_core(pose,
v,
J,
weights,
kintree_table,
want_Jtr=True)
Jtr = meta.Jtr if meta is not None else None
tr = trans.reshape((1, 3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.posedirs = posedirs
result.v_posed = v_posed
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
result.J_transformed = Jtr
return result
def ready_arguments(fname_or_dict,
posekey4vposed='pose',
shared_args=None,
chTrans=None,
chBetas=None):
import numpy as np
import pickle
import chumpy as ch
from chumpy.ch import MatVecMult
from mano.webuser.posemapper import posemap
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict))
else:
dd = fname_or_dict
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1] * 3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
if chTrans is not None:
dd['trans'] = chTrans
if chTrans is not None:
dd['betas'] = chBetas
for s in [
'v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs',
'betas', 'J', 'fullpose', 'pose_dof'
]:
if (s in dd) and not hasattr(dd[s], 'dterms'):
if shared_args is not None and s in shared_args:
dd[s] = shared_args[s]
else:
dd[s] = ch.array(dd[s])
assert (posekey4vposed in dd)
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(
posemap(dd['bs_type'])(dd[posekey4vposed]))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(
posemap(dd['bs_type'])(dd[posekey4vposed]))
return dd
def flow_to(self, v_next, cam_next):
from chumpy.ch import MatVecMult
color_image = self.r
visibility = self.visibility_image
pxpos = np.zeros_like(self.color_image)
pxpos[:, :, 0] = np.tile(row(np.arange(self.color_image.shape[1])),
(self.color_image.shape[0], 1))
pxpos[:, :, 2] = np.tile(col(np.arange(self.color_image.shape[0])),
(1, self.color_image.shape[1]))
visible = np.nonzero(visibility.ravel() != 4294967295)[0]
num_visible = len(visible)
barycentric = self.barycentric_image
# map 3d to 3d
JS = col(self.f[visibility.ravel()[visible]]).ravel()
IS = np.tile(col(np.arange(JS.size / 3)), (1, 3)).ravel()
data = barycentric.reshape((-1, 3))[visible].ravel()
# replicate to xyz
IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2))
JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2))
data = np.concatenate((data, data, data))
verts_to_visible = sp.csc_matrix((data, (IS, JS)),
shape=(np.max(IS) + 1, self.v.r.size))
v_old = self.camera.v
cam_old = self.camera
if cam_next is None:
cam_next = self.camera
self.camera.v = MatVecMult(verts_to_visible, self.v.r)
r1 = self.camera.r.copy()
self.camera = cam_next
self.camera.v = MatVecMult(verts_to_visible, v_next)
r2 = self.camera.r.copy()
n_channels = self.camera.shape[1]
flow = r2 - r1
flow_im = np.zeros(
(self.frustum['height'], self.frustum['width'], n_channels)).reshape(
(-1, n_channels))
flow_im[visible] = flow
flow_im = flow_im.reshape(
(self.frustum['height'], self.frustum['width'], n_channels))
self.camera = cam_old
self.camera.v = v_old
return flow_im
def ready_arguments(fname_or_dict, posekey4vposed='pose'):
import numpy as np
# import cPickle as pickle
import pickle
import chumpy as ch
from chumpy.ch import MatVecMult
from posemapper import posemap
if not isinstance(fname_or_dict, dict):
# dd = pickle.load(open(fname_or_dict))
dd = pickle.load(open(fname_or_dict, 'rb'), encoding='latin1')
else:
dd = fname_or_dict
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1] * 3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in [
'v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs',
'betas', 'J'
]:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
assert (posekey4vposed in dd)
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas']) + dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:, 0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:, 1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:, 2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(
posemap(dd['bs_type'])(dd[posekey4vposed]))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(
posemap(dd['bs_type'])(dd[posekey4vposed]))
return dd
def on_changed(self, which):
if not hasattr(self, 'kernel'):
self.kernel = self.default_kernel.copy()
if 'im_shape' in which:
sh = self.im_shape
self.transf_mtx, self._output_shape = filter_for_nopadding(sh, self.kernel)
if True: # self.want_downsampling: <-- setting want_downsampling to "False" is broken
halfsampler, self._output_shape = halfsampler_for(self._output_shape)
self.transf_mtx = halfsampler.dot(self.transf_mtx)
self.add_dterm('transform', MatVecMult(mtx=self.transf_mtx))
if 'px' in which:
self.transform.vec = self.px
def SecondFundamentalForm(v, f):
from chumpy import hstack, vstack
from chumpy.linalg import Pinv
nbrs = MatVecMult(FirstEdgesMtx(v, f, want_big=True), v.ravel()).reshape((-1,3))
b0 = NormalizedNx3(VertNormalsScaled(f=f, v=v)).reshape((-1,3))
b1 = NormalizedNx3(CrossProduct(b0, nbrs-v)).reshape((-1,3))
b2 = NormalizedNx3(CrossProduct(b0, b1)).reshape((-1,3))
cnct = get_vert_connectivity(v.r, f)
ffs = []
for i in range(v.size/3):
nbrs = v[np.nonzero(np.asarray(cnct[i].todense()).ravel())[0]] - row(v[i])
us = nbrs.dot(b2[i])
vs = nbrs.dot(b1[i])
hs = nbrs.dot(b0[i])
coeffs = Pinv(hstack((col((us*.5)**2), col(us*vs), col((vs*.5)**2)))).dot(hs)
ffs.append(row(coeffs))
# if i == 3586:
# import pdb; pdb.set_trace()
ffs = vstack(ffs)
return ffs
def blurredDiff(self):
edges = self.renderer.boundarybool_image
imshape = self.renderer.shape
rgbEdges = np.tile(edges.reshape([imshape[0], imshape[1], 1]),
[1, 1, 3]).astype(np.bool)
from opendr.filters import filter_for
from opendr.filters import GaussianKernel2D
blur_mtx = filter_for(imshape[0],
imshape[1],
imshape[2] if len(imshape) > 2 else 1,
kernel=GaussianKernel2D(3, 1))
blurred_diff = MatVecMult(blur_mtx, self.renderer -
self.rendererGT).reshape(imshape)
return blurred_diff[rgbEdges]
def on_changed(self, which):
if 'w' in which or 'A' in which:
A = np.abs(self.A.tocoo())
self.wmat = A.tocsc().dot(sp.diags([self.w], [0]))
A9_data = np.tile(A.data / 2., 9)
A9_row = np.hstack([A.row * 9 + i for i in range(9)])
A9_col = np.hstack([A.col * 9 + i for i in range(9)])
self.A9 = sp.csc_matrix((A9_data, (A9_row, A9_col)),
shape=(A.shape[0] * 9, A.shape[1] * 9))
#wmatcoo = self.wmat.tocoo()
#wmat3_data = np.tile(wmatcoo.data, 3)
#wmat3_row = np.hstack([wmatcoo.row*3 + i for i in range(3)])
#wmat3_col = np.hstack([wmatcoo.col*3 + i for i in range(3)])
#self.wmat3 = sp.csc_matrix((wmat3_data, (wmat3_row, wmat3_col)),
# shape=(wmatcoo.shape[0]*3,
# wmatcoo.shape[1]*3))
if 'reg_e' in which or 'model_e' in which:
T = np.einsum('ij,jk->ikj', self.model_e.T,
self.reg_e).reshape(9, -1)
# compute rotation per vertex rv through SVD
T_per_v = (self.wmat.dot(T.T)).reshape(-1, 3, 3)
u_s_v = np.linalg.svd(T_per_v)
col3 = np.asarray([1., 1., -1.])
# negate third column of u if det is negative
# The real rotation, according to Sorkine, would be v.T.dot(u.T),
# because numpy returns usv instead of usv.T. However, we're applying
# e.dot(rot), so we need rot.T which is u.dot(v)
self.rv = np.asarray([
u.dot(v) if np.linalg.det(u.dot(v)) > 0 # np.prod(s) > 0#
else (u * col3).dot(v) for u, s, v in zip(*u_s_v)
])
# average vertex rotations for each edge
self.rf = MatVecMult(mtx=self.A9.T, vec=self.rv.ravel()).reshape(-1, 3)
w3 = ch.tile(self.w, (3, 1)).T.ravel()
self.err = (LoopProd(rot=self.rf, edge=self.model_e).ravel() -
self.reg_e.ravel()) * w3
def verts_decorated(trans,
pose,
v_template,
J,
weights,
kintree_table,
bs_style,
f,
bs_type=None,
posedirs=None,
betas=None,
shapedirs=None,
want_Jtr=False):
for which in [
trans, pose, v_template, weights, posedirs, betas, shapedirs
]:
if which is not None:
assert ischumpy(which)
v = v_template
print "********"
print "betas : " + str(betas.shape)
print "posedirs : " + str(posedirs.shape)
print "shapedirs : " + str(shapedirs.shape)
if shapedirs is not None:
if betas is None:
betas = chumpy.zeros(shapedirs.shape[-1])
v_shaped = v + shapedirs.dot(betas)
#print shapedirs.dot(betas).shape
#v_shaped = np.copy(v)
#for i in range(0,shapedirs.shape[0]):
# v_shaped[i,:] = v[i,:] + shapedirs[i,:,:].dot(betas)
#print v_shaped[1,:]
#xx = np.random.rand(4,4,4)
#print xx
#print xx[0,:,1]
#print (np.array(xx)).tolist()
#print (np.array(xx)).tolist()[1][1][1]
else:
v_shaped = v
if posedirs is not None:
print pose.shape
print posemap(bs_type)(pose).shape
v_posed = v_shaped + posedirs.dot(posemap(bs_type)(pose))
#v_posed = v_shaped
else:
v_posed = v_shaped
v = v_posed
# IS this needed
if sp.issparse(J):
regressor = J
print regressor.shape
J_tmpx = MatVecMult(regressor, v_shaped[:, 0])
J_tmpy = MatVecMult(regressor, v_shaped[:, 1])
J_tmpz = MatVecMult(regressor, v_shaped[:, 2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
#XX = np.array(regressor) * np.array(v_shaped[:,0])
#print "--"
#print XX.shape
#print XX
#print v_shaped[200,2]
#print XX[1]
else:
assert (ischumpy(J))
assert (bs_style == 'lbs')
result, Jtr = lbs.verts_core(pose,
v,
J,
weights,
kintree_table,
want_Jtr=True,
xp=chumpy)
tr = trans.reshape((1, 3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.kintree_table = kintree_table
result.bs_style = bs_style
result.bs_type = bs_type
if posedirs is not None:
result.posedirs = posedirs
result.v_posed = v_posed
if shapedirs is not None:
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
if want_Jtr:
result.J_transformed = Jtr
#print Jtr
return result
def optimize_on_DensePose(dp_iuv,
op_j2d,
w_j2d,
model,
cam,
prior,
body_orient,
body_trans,
body_init,
n_betas=10,
viz=False,
imageid=1):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
outPath = '/home/xiul/databag/dbfusion/record0/smplify/vid{}'.format(
imageid)
if not os.path.isdir(outPath):
os.mkdir(outPath)
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
tarIUV = ch.array(dp_iuv)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((body_orient, prior.weights.dot(prior.means)))
#init_pose = np.hstack((body_orient, body_init))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.array(body_trans),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
J_tmpx = MatVecMult(J_reg, sv[:, 0])
J_tmpy = MatVecMult(J_reg, sv[:, 1])
J_tmpz = MatVecMult(J_reg, sv[:, 2])
Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
#Jtr = J_reg.dot(sv)
cam.v = Jtr
op_j2d = op_j2d * 1280 / 1920
w = 1280
h = 720
reIUV = render_model(sv,
model.f,
w,
h,
cam,
near=0.5,
far=25,
vc=dp_colors,
img=None)
#gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None):
obj_j2d = lambda w, sigma: (w * w_j2d.reshape((-1, 1)) * GMOf(
(op_j2d - cam), sigma))
#input_objective, imshape, normalization, n_levels, as_list
err = tarIUV - reIUV
obj_dense = 100 * gaussian_pyramid(err, n_levels=6, normalization='SSE')
# data term: distance between observed and estimated joints in 2D
# obj_dense = lambda w, sigma: (
# w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
if viz:
import matplotlib.pyplot as plt
from time import gmtime, strftime
def on_step(cstep):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.clf()
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = reIUV.r.copy()
tmp_tar = tarIUV.copy()
plt.imshow(tmp_img)
for j1, j2 in zip(cam.r, op_j2d):
plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r')
plt.subplot(1, 2, 2)
plt.imshow(tmp_tar)
plt.draw()
plt.savefig(os.path.join(
outPath, '{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),
bbox_inches='tight')
on_step(stepI)
else:
on_step = None
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78, 4.78, 4.78, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1, 5, 5, 5])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
if stage >= 4:
objs['dense'] = obj_dense
if stage <= 4:
objs['j2d'] = obj_j2d(1, 100)
else:
objs['j2d'] = obj_j2d(0.1, 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
ch.minimize(objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={
'maxiter': 10000,
'e_3': .001,
'disp': 1
})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if viz:
plt.ioff()
nposeparms = dd['kintree_table'].shape[1]*3
dd['trans'] = np.zeros(3)
dd['pose'] = np.zeros(nposeparms)
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
else:
print type(dd[s])
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
if dd['pose'].ndim != 2 or p.shape[1] != 3:
p = dd['pose'].reshape((-1,3))
p = p[1:]
c= ch.concatenate([(Rodrigues(pp)-ch.eye(3)).ravel() for pp in p]).ravel()
dd['v_posed'] = v_shaped + dd['posedirs'].dot(c)
args = {
'pose': dd['pose'],
'v': dd['v_posed'],
'J': dd['J'],
def optimize_on_DensePose(ori_img,
tar_img,
dp_iuv,
op_j2d,
w_j2d,
model,
cam,
cam_old,
prior,
init_trans,
init_pose,
init_betas,
n_betas=10,
viz=False,
imageid=1):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
outPath = '/home/xiul/databag/net_images/smplify/vid_{:06d}_puredense'.format(
imageid)
if not os.path.isdir(outPath):
os.mkdir(outPath)
dp_iuv_tar = dp_iuv.copy()
width = dp_iuv_tar.shape[1]
height = dp_iuv_tar.shape[0]
dp_iuv_weight = np.zeros(dp_iuv_tar.shape) + 10
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 0] = 10
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 1] = 10
dp_iuv_weight[dp_iuv_tar[:, :, 2] < 0.95, 2] = 15
dp_iuv_weight = ch.array(dp_iuv_weight)
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
tarIUV = ch.array(dp_iuv)
# initialize the shape to the mean shape in the SMPL training set
betas = ch.array(init_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
pose = ch.array(init_pose)
#init_pose = np.hstack((body_orient, body_init))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(trans=ch.array(init_trans),
pose=pose,
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=None)
J_tmpx = MatVecMult(J_reg, sv[:, 0])
J_tmpy = MatVecMult(J_reg, sv[:, 1])
J_tmpz = MatVecMult(J_reg, sv[:, 2])
Jtr = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
#Jtr = J_reg.dot(sv)
cam.v = Jtr
reIUV = render_model(sv,
model.f,
width,
height,
cam,
near=0.5,
far=25,
vc=dp_colors,
img=None)
reModel = render_model(sv,
model.f,
width,
height,
cam,
near=0.5,
far=25,
vc=None,
img=None)
fullModel = render_model(sv,
model.f,
ori_img.shape[1],
ori_img.shape[0],
cam_old,
near=0.5,
far=25,
vc=None,
img=None)
#gaussian_pyramid(input_objective, imshape=None, normalization='SSE', n_levels=3, as_list=False, label=None):
def obj_j2d(w, sigma):
return (w * w_j2d.reshape((-1, 1)) * GMOf((op_j2d - cam), sigma))
#input_objective, imshape, normalization, n_levels, as_list
err = (tarIUV - reIUV) * dp_iuv_weight
obj_dense = 100 * gaussian_pyramid(err, n_levels=4, normalization='SSE')
#obj_dense = gaussian_pyramid(err, n_levels=4, normalization=SSE)
#obj_dense = 10000*err
# data term: distance between observed and estimated joints in 2D
# obj_dense = lambda w, sigma: (
# w * GMOf((tarIUV - reIUV).reshape(-1,3), sigma))
# mixture of gaussians pose prior
def pprior(w):
return w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
def my_exp(x):
return alpha * ch.exp(x)
def obj_angle(w):
return w * ch.concatenate([
my_exp(sv.pose[55]),
my_exp(-sv.pose[58]),
my_exp(-sv.pose[12]),
my_exp(-sv.pose[15])
])
def viz_func(stage_num):
plt.figure(1, figsize=(10, 10))
plt.clf()
plt.subplot(2, 3, 1)
# show optimized joints in 2D
tmp_img = reIUV.r.copy()
tmp_tar = tarIUV.copy()
tmp_model = reModel.r.copy()
full_model = fullModel.r.copy()
# w = tmp_tar.shape[1]
# h = tmp_tar.shape[0]
plt.imshow(tar_img)
plt.imshow(tmp_img, alpha=0.5)
for j1, j2, w_ts in zip(cam.r, op_j2d, w_j2d):
if (w_ts > 0):
plt.plot([j1[0], j2[0]], [j1[1], j2[1]], 'r')
plt.xlim([0, width])
plt.ylim([height, 0])
plt.subplot(2, 3, 2)
plt.imshow(tar_img)
plt.imshow(tmp_tar, alpha=0.5)
plt.xlim([0, width])
plt.ylim([height, 0])
plt.subplot(2, 3, 3)
plt.imshow(tar_img)
tmp_model_alpha = np.ones((tmp_model.shape[0], tmp_model.shape[1]))
tmp_model_alpha[tmp_model[:, :, 0] < 1e-2] = 0
tmp_model = np.dstack((tmp_model, tmp_model_alpha))
plt.imshow(tmp_model)
plt.xlim([0, width])
plt.ylim([height, 0])
plt.subplot(2, 1, 2)
plt.imshow(full_model)
plt.draw()
plt.savefig(os.path.join(outPath, 'stage-{}.png'.format(stage_num)),
bbox_inches='tight')
full_model_int = full_model.copy()
full_model_int *= 255
full_model_int = full_model_int.astype(np.uint8)
cv2.imwrite(os.path.join(outPath, 'model-{}.png'.format(stage_num)),
full_model_int)
out_params = {
'pose': sv.pose.r,
'shape': sv.betas.r,
'trans': sv.trans.r
}
with open(os.path.join(outPath, 'param-{}.pkl'.format(stage_num)),
'w') as fio:
pickle.dump(out_params, fio, pickle.HIGHEST_PROTOCOL)
viz_func(0)
#if viz:
if viz and False:
def on_step(cstep):
"""Create visualization."""
# TODO this function is in vis_func
#plt.savefig(os.path.join(outPath,'{}.png'.format(strftime("%d_%H_%M_%S", gmtime()))),bbox_inches='tight')
on_step(stepI)
else:
on_step = None
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
# opt_weights = zip([4.78, 3.78, 2.78, 1.78],
# [5, 5, 5, 5],
# [1, 0.25, 0.10, 0])
opt_weights = zip([4.78, 4.78, 4.78, 4.78, 4.78], [50, 50, 50, 50, 50],
[0.1, 0.1, 0.5, 0.25, 0.1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas, w_joints) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
#if stage >= 1:
objs['dense'] = obj_dense
objs['j2d'] = obj_j2d(w_joints, 50)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
ch.minimize(objs,
x0=[sv.betas, sv.pose, sv.trans],
method='dogleg',
callback=on_step,
options={
'maxiter': 10000,
'e_3': .0001,
'disp': 1
})
viz_func(stage + 1)
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if viz and False:
plt.ioff()
def verts_decorated_quat(trans,
pose,
v_template,
J,
weights,
kintree_table,
f,
posedirs=None,
betas=None,
add_shape=True,
shapedirs=None,
want_Jtr=False):
for which in [
trans, pose, v_template, weights, posedirs, betas, shapedirs
]:
if which is not None:
assert ischumpy(which)
v = v_template
if shapedirs is not None:
if betas is None:
betas = chumpy.zeros(shapedirs.shape[-1])
if add_shape:
v_shaped = v + shapedirs.dot(betas) #Add Shape of the model.
else:
v_shaped = v
else:
v_shaped = v
quaternion_angles = axis2quat(pose.reshape((-1, 3))).reshape(-1)
shape_feat = betas[1]
feat = ch.concatenate([quaternion_angles, shape_feat], axis=0)
feat = quaternion_angles
poseblends = posedirs.dot(feat)
v_posed = v_shaped + poseblends
v = v_posed
regressor = J
J_tmpx = MatVecMult(regressor, v_shaped[:, 0])
J_tmpy = MatVecMult(regressor, v_shaped[:, 1])
J_tmpz = MatVecMult(regressor, v_shaped[:, 2])
J = chumpy.vstack((J_tmpx, J_tmpy, J_tmpz)).T
result, meta = verts_core(pose,
v,
J,
weights,
kintree_table,
want_Jtr=True)
Jtr = meta.Jtr if meta is not None else None
tr = trans.reshape((1, 3))
result = result + tr
Jtr = Jtr + tr
result.trans = trans
result.f = f
result.pose = pose
result.v_template = v_template
result.J = J
result.weights = weights
result.kintree_table = kintree_table
result.poseblends = poseblends
result.quats = quaternion_angles
if meta is not None:
for field in ['Jtr', 'A', 'A_global', 'A_weighted']:
if (hasattr(meta, field)):
setattr(result, field, getattr(meta, field))
if posedirs is not None:
result.posedirs = posedirs
result.v_posed = v_posed
if shapedirs is not None:
result.shapedirs = shapedirs
result.betas = betas
result.v_shaped = v_shaped
if want_Jtr:
result.J_transformed = Jtr
result.poseblends = poseblends
return result
def optimize_smal(fposes,
ftrans,
fbetas,
model,
cams,
segs,
imgs,
landmarks,
landmarks_names,
key_vids,
symIdx=None,
frameId=0,
opt_model_dir=None,
save_name=None,
COMPUTE_OPT=True,
img_paths=None,
img_offset=None,
img_scales=None):
mesh_v_opt_save_path = join(opt_model_dir,
'mesh_v_opt_no_mc_' + str(frameId) + '.ply')
mesh_v_opt_mc_save_path = join(opt_model_dir,
'mesh_v_opt_' + str(frameId) + '.ply')
mesh_init_save_path = join(opt_model_dir,
'mesh_init_' + str(frameId) + '.ply')
nViews = len(fposes)
if not COMPUTE_OPT:
if not exists(opt_model_dir):
makedirs(opt_model_dir)
dv = 0
compute_texture(nViews, opt_model_dir, dv, model, frameId,
mesh_init_save_path, fposes, ftrans, fbetas,
'_no_refine', cams, imgs, segs, img_paths, img_offset,
img_scales)
return
# Write the initial mesh
np_betas = np.zeros_like(model.betas)
np_betas[:len(fbetas[0])] = fbetas[0]
tmp = verts_decorated(v_template=model.v_template,
pose=ch.zeros_like(model.pose.r),
trans=ch.zeros_like(model.trans),
J=model.J_regressor,
kintree_table=model.kintree_table,
betas=ch.array(np_betas),
weights=model.weights,
posedirs=model.posedirs,
shapedirs=model.shapedirs,
bs_type='lrotmin',
bs_style='lbs',
f=model.f)
tmp_mesh = Mesh(v=tmp.r, f=tmp.f)
tmp_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply')
tmp_mesh.write_ply(tmp_path)
del tmp
assert (nViews == len(cams))
assert (nViews == len(segs))
assert (nViews == len(imgs))
# Define a displacement vector. We set a small non zero displacement as initialization
dv = ch.array(np.random.rand(model.r.shape[0], 3) / 1000.)
# Cell structure for ARAP
f = model.f
_, A3, A = edgesIdx(nV=dv.shape[0], f=f, save_dir='.', name='smal')
wedge = wedges(A3, dv)
s = np.zeros_like(dv)
arap = ARAP(reg_e=MatVecMult(A3.T,
model.ravel() + dv.ravel()).reshape(-1, 3),
model_e=MatVecMult(A3.T, model.ravel()).reshape(-1, 3),
w=wedge,
A=A)
k_arap = settings['ref_k_arap_per_view'] * nViews
for weight, part in zip(settings['ref_W_arap_values'],
settings['ref_W_arap_parts']):
k_arap, W_per_vertex = get_arap_part_weights(
A, k_arap, [part], [weight]) #, animal_name) # was only Head
W = np.zeros((W_per_vertex.shape[0], 3))
for i in range(3):
W[:, i] = W_per_vertex
k_lap = settings['ref_k_lap'] * nViews * W
k_sym = settings['ref_k_sym'] * nViews
k_keyp = settings['ref_k_keyp_weight'] * nViews
# Load already computed mesh
if not exists(opt_model_dir):
makedirs(opt_model_dir)
shape_model, compute = load_shape_models(nViews, opt_model_dir, dv, model,
frameId, mesh_v_opt_save_path,
fposes, ftrans, fbetas)
mv = None
# Remove inside mouth faces
'''
if settings['ref_remove_inside_mouth']:
# Giraffe
faces_orig = shape_model[0].f.copy()
im_v = im_up_v + im_down_v
idx = [np.where(model.f == ix)[0] for ix in im_v]
idx = np.concatenate(idx).ravel()
for i in range(nViews):
shape_model[i].f = np.delete(shape_model[i].f, idx, 0)
'''
if compute:
objs = {}
FIX_CAM = True
free_variables = []
kp_weights = k_keyp * np.ones((landmarks[0].shape[0], 1))
print('removing shoulders, often bad annotated')
kp_weights[landmarks_names.index('leftShoulder'), :] *= 0
kp_weights[landmarks_names.index('rightShoulder'), :] *= 0
objs_pose = None
j2d = None
#k_silh_term = settings['ref_k_silh_term']
k_m2s = settings['ref_k_m2s']
k_s2m = settings['ref_k_s2m']
objs, params_, j2d = set_pose_objs(shape_model,
cams,
landmarks,
key_vids,
kp_weights=kp_weights,
FIX_CAM=FIX_CAM,
ONLY_KEYP=True,
OPT_SHAPE=False)
if np.any(k_arap) != 0:
objs['arap'] = k_arap * arap
if k_sym != 0:
objs['sym_0'] = k_sym * (ch.abs(dv[:, 0] - dv[symIdx, 0]))
objs['sym_1'] = k_sym * (ch.abs(dv[:, 1] + dv[symIdx, 1] -
0.00014954))
objs['sym_2'] = k_sym * (ch.abs(dv[:, 2] - dv[symIdx, 2]))
if np.any(k_lap) != 0:
lap_op = np.asarray(
laplacian(Mesh(v=dv, f=shape_model[0].f)).todense())
objs['lap'] = k_lap * ch.dot(lap_op, dv)
mv = None
mv2 = MeshViewers(shape=(1, nViews)) #None
vc = np.ones_like(dv)
dv_r = fit_silhouettes_pyramid_opt(objs,
shape_model,
dv,
segs,
cams,
j2d=j2d,
weights=1.,
mv=mv,
imgs=imgs,
s2m_weights=k_s2m,
m2s_weights=k_m2s,
max_iter=100,
free_variables=free_variables,
vc=vc,
symIdx=symIdx,
mv2=mv2,
objs_pose=objs_pose)
# Save result image
for i in range(nViews):
img_res = render_mesh(Mesh(shape_model[i].r, shape_model[i].f),
imgs[i].shape[1],
imgs[i].shape[0],
cams[i],
img=imgs[i],
world_frame=True)
img_result = np.hstack((imgs[i], img_res * 255.))
save_img_path = save_name[i].replace('.pkl', '_v_opt.png')
cv2.imwrite(save_img_path, img_result)
shape_model[0].pose[:] = 0
shape_model[0].trans[:] = 0
V = shape_model[0].r.copy()
vm = V[symIdx, :].copy()
vm[:, 1] = -1 * vm[:, 1]
V2 = (V + vm) / 2.0
mesh_out = Mesh(v=V2, f=shape_model[0].f)
mesh_out.show()
mesh_out.write_ply(mesh_v_opt_save_path)
save_dv_data_path = mesh_v_opt_save_path.replace('.ply', '_dv.pkl')
dv_data = {'betas': shape_model[0].betas.r, 'dv': dv_r}
pkl.dump(dv_data, open(save_dv_data_path, 'wb'))
compute_texture(nViews, opt_model_dir, dv, model, frameId,
mesh_v_opt_save_path, fposes, ftrans, fbetas, '_non_opt',
cams, imgs, segs, img_paths, img_offset, img_scales)
return
