def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False):
options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]}
options['disp'] = 1.0
options['delta_0'] = 0.1
options['e_3'] = 1e-4
s = ch.ones(1)
r = ch.zeros(3)
R = Rodrigues(r)
t = ch.zeros(3)
trafo_mesh = s*(R.dot(mesh.v.T)).T + t
sampler = sample_from_mesh(scan, sample_type='vertices')
s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False)
if visualize:
#Visualization code
mv = MeshViewer()
mv.set_static_meshes([scan])
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
def on_show(_):
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
else:
def on_show(_):
pass
ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options)
return s,Rodrigues(r),t
def __init__(self, t, rod, rad, length):
self.t = t # translation of the axis
self.rod = rod # rotation of the axis in Rodrigues form
self.rad = rad # radious of the capsule
self.length = length # length of the axis
axis0 = ch.vstack([0, ch.abs(self.length), 0])
self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T))
v0 = ch.hstack([v[:26].T*rad, (v[26:].T*rad)+ axis0])
self.v = ((t + Rodrigues(rod).dot(v0)).T)
self.set_sphere_centers()
def __init__(self, t, rod, rad, length):
assert (hasattr(t, 'dterms'))
# the translation should be a chumpy object (differentiable wrt shape)
self.t = t # translation of the axis
self.rod = rod # rotation of the axis in Rodrigues form
# the radius should be a chumpy object (differentiable wrt shape)
assert (hasattr(rad, 'dterms'))
self.rad = rad # radius of the capsule
# the length should be a chumpy object (differentiable wrt shape)
assert (hasattr(length, 'dterms'))
self.length = length # length of the axis
axis0 = ch.vstack([0, ch.abs(self.length), 0])
self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T))
v0 = ch.hstack([v[:26].T * rad, (v[26:].T * rad) + axis0])
self.v = ((t + Rodrigues(rod).dot(v0)).T)
self.set_sphere_centers()
def volume(v, f):
# Construct a 3D matrix which is of size (nfaces x 3 x 3)
# Each row corresponds to a face; the third dimension indicates
# which triangle in that face is being referred to
vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :]))
v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0]
v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0]
v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1]
v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1]
v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2]
v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2]
volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1.0 / 6.0)
return ch.abs(ch.sum(volumes))
def volume(v, f):
# Construct a 3D matrix which is of size (nfaces x 3 x 3)
# Each row corresponds to a face; the third dimension indicates
# which triangle in that face is being referred to
vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :]))
v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0]
v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0]
v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1]
v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1]
v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2]
v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2]
volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1. / 6.)
return ch.abs(ch.sum(volumes))
def __init__(self, t, rod, rad, length):
assert (hasattr(t, 'dterms'))
# the translation should be a chumpy object (differentiable wrt shape)
self.t = t # translation of the axis
self.rod = rod # rotation of the axis in Rodrigues form
# the radius should be a chumpy object (differentiable wrt shape)
assert (hasattr(rad, 'dterms'))
self.rad = rad # radius of the capsule
# the length should be a chumpy object (differentiable wrt shape)
assert (hasattr(length, 'dterms'))
self.length = length # length of the axis
axis0 = ch.vstack([0, ch.abs(self.length), 0])
self.axis = ch.vstack((t.T, (t + Rodrigues(rod).dot(axis0)).T))
v0 = ch.hstack([v[:26].T * rad, (v[26:].T * rad) + axis0])
self.v = ((t + Rodrigues(rod).dot(v0)).T)
self.set_sphere_centers()
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None):
from opendr.geometry import Rodrigues
if length_regs is not None:
n_shape_dofs = length_regs.shape[0] - 1
else:
n_shape_dofs = model.betas.r.size
segm = np.argmax(model.weights_prior, axis=1)
J_off = ch.zeros((len(joint2name), 3))
rots = rots0.copy()
mujoco_t_mid = [0, 3, 6, 9]
if wrt_betas is not None:
# if we want to differentiate wrt betas (shape), we must have the
# regressors...
assert (length_regs is not None and rad_regs is not None)
# ... and betas must be a chumpy object
assert (hasattr(wrt_betas, 'dterms'))
pad = ch.concatenate(
(wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1)))
lengths = pad.dot(length_regs)
rads = pad.dot(rad_regs)
else:
lengths = ch.ones(len(joint2name))
rads = ch.ones(len(joint2name))
betas = wrt_betas if wrt_betas is not None else model.betas
n_betas = len(betas)
# the joint regressors are the original, pre-optimized ones
# (middle of the part frontier)
myJ_regressor = model.J_regressor_prior
myJ0 = ch.vstack(
(ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 0] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]),
ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 1] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]),
ch.ch.MatVecMult(myJ_regressor, model.v_template[:, 2] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T
# with small adjustments for hips, spine and feet
myJ = ch.vstack(
[ch.concatenate([myJ0[0, 0], (
.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]), myJ0[9, 2]]),
ch.vstack([myJ0[i] for i in range(1, 7)]), ch.concatenate(
[myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]),
ch.concatenate(
[myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]),
ch.concatenate(
[myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]),
ch.vstack([myJ0[i] for i in range(10, 24)])])
capsules = []
# create one capsule per mujoco joint
for ijoint, segms in enumerate(mujoco2segm):
if wrt_betas is None:
vidxs = np.asarray([segm == k for k in segms]).any(axis=0)
verts = model.v_template[vidxs].r
dims = (verts.max(axis=0) - verts.min(axis=0))
rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[(
np.argmax(dims) + 2) % 3]) / 4.)
lengths[ijoint] = max(dims) - 2. * rads[ijoint].r
# the core joints are different, since the capsule is not in the joint
# but in the middle
if ijoint in mujoco_t_mid:
len_offset = ch.vstack([ch.zeros(1), ch.abs(lengths[ijoint]) / 2.,
ch.zeros(1)]).reshape(3, 1)
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(
3, 1) - Rodrigues(rots[ijoint]).dot(len_offset),
rots[ijoint], rads[ijoint], lengths[ijoint])
else:
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1),
rots[ijoint], rads[ijoint], lengths[ijoint])
caps.id = ijoint
capsules.append(caps)
return capsules
def get_capsules(model, wrt_betas=None, length_regs=None, rad_regs=None):
from opendr.geometry import Rodrigues
if length_regs is not None:
n_shape_dofs = length_regs.shape[0] - 1
else:
n_shape_dofs = model.betas.r.size
segm = np.argmax(model.weights_prior, axis=1)
J_off = ch.zeros((len(joint2name), 3))
rots = rots0.copy()
mujoco_t_mid = [0, 3, 6, 9]
if wrt_betas is not None:
# if we want to differentiate wrt betas (shape), we must have the
# regressors...
assert (length_regs is not None and rad_regs is not None)
# ... and betas must be a chumpy object
assert (hasattr(wrt_betas, 'dterms'))
pad = ch.concatenate(
(wrt_betas, ch.zeros(n_shape_dofs - len(wrt_betas)), ch.ones(1)))
lengths = pad.dot(length_regs)
rads = pad.dot(rad_regs)
else:
lengths = ch.ones(len(joint2name))
rads = ch.ones(len(joint2name))
betas = wrt_betas if wrt_betas is not None else model.betas
n_betas = len(betas)
# the joint regressors are the original, pre-optimized ones
# (middle of the part frontier)
myJ_regressor = model.J_regressor_prior
myJ0 = ch.vstack((ch.ch.MatVecMult(
myJ_regressor, model.v_template[:, 0] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 0]),
ch.ch.MatVecMult(
myJ_regressor, model.v_template[:, 1] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 1]),
ch.ch.MatVecMult(
myJ_regressor, model.v_template[:, 2] +
model.shapedirs[:, :, :n_betas].dot(betas)[:, 2]))).T
# with small adjustments for hips, spine and feet
myJ = ch.vstack([
ch.concatenate([
myJ0[0, 0], (.6 * myJ0[0, 1] + .2 * myJ0[1, 1] + .2 * myJ0[2, 1]),
myJ0[9, 2]
]),
ch.vstack([myJ0[i] for i in range(1, 7)]),
ch.concatenate(
[myJ0[7, 0], (1.1 * myJ0[7, 1] - .1 * myJ0[4, 1]), myJ0[7, 2]]),
ch.concatenate(
[myJ0[8, 0], (1.1 * myJ0[8, 1] - .1 * myJ0[5, 1]), myJ0[8, 2]]),
ch.concatenate(
[myJ0[9, 0], myJ0[9, 1], (.2 * myJ0[9, 2] + .8 * myJ0[12, 2])]),
ch.vstack([myJ0[i] for i in range(10, 24)])
])
capsules = []
# create one capsule per mujoco joint
for ijoint, segms in enumerate(mujoco2segm):
if wrt_betas is None:
vidxs = np.asarray([segm == k for k in segms]).any(axis=0)
verts = model.v_template[vidxs].r
dims = (verts.max(axis=0) - verts.min(axis=0))
rads[ijoint] = .5 * ((dims[(np.argmax(dims) + 1) % 3] + dims[
(np.argmax(dims) + 2) % 3]) / 4.)
lengths[ijoint] = max(dims) - 2. * rads[ijoint].r
# the core joints are different, since the capsule is not in the joint
# but in the middle
if ijoint in mujoco_t_mid:
len_offset = ch.vstack(
[ch.zeros(1),
ch.abs(lengths[ijoint]) / 2.,
ch.zeros(1)]).reshape(3, 1)
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1) -
Rodrigues(rots[ijoint]).dot(len_offset), rots[ijoint],
rads[ijoint], lengths[ijoint])
else:
caps = Capsule(
(J_off[ijoint] + myJ[mujoco2segm[ijoint][0]]).reshape(3, 1),
rots[ijoint], rads[ijoint], lengths[ijoint])
caps.id = ijoint
capsules.append(caps)
return capsules
def diffHog(image, drconv=None, numOrient=9, cwidth=8, cheight=8):
imagegray = 0.3 * image[:, :, 0] + 0.59 * image[:, :,
1] + 0.11 * image[:, :, 2]
sy, sx = imagegray.shape
# gx = ch.empty(imagegray.shape, dtype=np.double)
gx = imagegray[:, 2:] - imagegray[:, :-2]
gx = ch.hstack([np.zeros([sy, 1]), gx, np.zeros([sy, 1])])
gy = imagegray[2:, :] - imagegray[:-2, :]
# gy = imagegray[:, 2:] - imagegray[:, :-2]
gy = ch.vstack([np.zeros([1, sx]), gy, np.zeros([1, sx])])
gx += 1e-5
# gy = imagegray[:-2,1:-1] - imagegray[2:,1:-1] + 0.00001
# gx = imagegray[1:-1,:-2] - imagegray[1:-1, 2:] + 0.00001
distFilter = np.ones([2 * cheight, 2 * cwidth], dtype=np.uint8)
distFilter[np.int(2 * cheight / 2), np.int(2 * cwidth / 2)] = 0
distFilter = (cv2.distanceTransform(distFilter, cv2.DIST_L2, 3) - np.max(
cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) / (
-np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3)))
magn = ch.sqrt(gy**2 + gx**2) * 180 / np.sqrt(2)
angles = ch.arctan(gy / gx) * 180 / np.pi + 90
# meanOrient = np.linspace(0, 180, numOrient)
orientations_arr = np.arange(numOrient)
meanOrient = orientations_arr / numOrient * 180
fb_resttmp = 1 - ch.abs(
ch.expand_dims(angles[:, :], 2) -
meanOrient[1:].reshape([1, 1, numOrient - 1])) * numOrient / 180
zeros_rest = np.zeros([sy, sx, numOrient - 1, 1])
fb_rest = ch.max(ch.concatenate([fb_resttmp[:, :, :, None], zeros_rest],
axis=3),
axis=3)
chMinOrient0 = ch.min(ch.concatenate([
ch.abs(
ch.expand_dims(angles[:, :], 2) -
meanOrient[0].reshape([1, 1, 1]))[:, :, :, None],
ch.abs(180 - ch.expand_dims(angles[:, :], 2) -
meanOrient[0].reshape([1, 1, 1]))[:, :, :, None]
],
axis=3),
axis=3)
zeros_fb0 = np.zeros([sy, sx, 1])
fb0_tmp = ch.concatenate(
[1 - chMinOrient0[:, :] * numOrient / 180, zeros_fb0], axis=2)
fb_0 = ch.max(fb0_tmp, axis=2)
fb = ch.concatenate([fb_0[:, :, None], fb_rest], axis=2)
# fb[:,:,0] = ch.max(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180,0)
# fb = 1./(1. + ch.exp(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180))
Fb = ch.expand_dims(magn, 2) * fb
if drconv is None:
drconv = dr_wrt_convolution(Fb[:, :, 0], distFilter)
Fs_list = [
convolve2D(x=Fb[:, :, Fbi], filter=distFilter,
convolve2DDr=drconv).reshape([Fb.shape[0], Fb.shape[1], 1])
for Fbi in range(numOrient)
]
# Fs_list = [scipy.signal.convolve2d(Fb[:,:,Fbi], distFilter).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)]
Fs = ch.concatenate(Fs_list, axis=2)
# cellCols = np.arange(start=cwidth/2, stop=Fs.shape[1]-cwidth/2 , step=cwidth)
# cellRows = np.arange(start=cheight/2, stop=Fs.shape[0]-cheight/2 , step=cheight)
Fcells = Fs[0:Fs.shape[0]:cheight, 0:Fs.shape[1]:cwidth, :]
epsilon = 1e-5
v = Fcells / ch.sqrt(ch.sum(Fcells**2) + epsilon)
# v = Fcells
# hog, hogim = skimage.feature.hog(imagegray, orientations=numOrient, pixels_per_cell=(cheight, cwidth), visualise=True)
hog_image = HogImage(image=image,
hog=Fcells,
numOrient=numOrient,
cwidth=cwidth,
cheight=cheight)
# plt.imshow(hog_image)
# plt.figure()
# plt.imshow(hogim)
# ipdb.set_trace()
return v, hog_image, drconv
c_pre={};
if (W_Joints):
k=a['Joints_Target'];
j_to_consider = range(0,24)
J_distances = J_reposed[j_to_consider,:] - (k[j_to_consider,:]+trans);
c_pre['Joints']= J_distances*W_Joints;
if(W_FMP2P):
c_pre['FMP2P']= distances*W_FMP2P;
if(W_Norm_B):
c_pre['Norm_B']= ((result.betas)**2)*W_Norm_B;
if(W_Norm_T):
pose_res=result.pose.reshape(-1,3);
angles=ch.sum(ch.abs(pose_res)**2,axis=-1)**(1./2)
pesi = np.ones(24)*8/18
pesi[[0]] = np.ones(1)*[2]
pesi[[10, 11, 22, 23, 15]] = np.ones(5)*[2./18]
pesi[[6,3, 7,8]] = np.ones(4)*[5./18]
pesi[[12, 15]] = np.ones(2)*[1./36]
costo_T= (angles/(ch.pi*pesi))**12
c_pre['Norm_T']=costo_T*W_Norm_T;
if(W_Landmarks):
Tar_Land = Tar_shift[a['landmarks1'].reshape(5)-1];
SMPL_Land=result[a['landmarks2'].reshape(5)-1];
c_pre['Landmarks']= (SMPL_Land-Tar_Land)*W_Landmarks;
(r,b,t)=ch.minimize(c_pre, x0 = [result.pose, result.betas,trans],
method = 'dogleg', callback = None,
def set_pose_objs(sv,
cam,
landmarks,
key_vids,
animal=None,
shape_data_name=None,
nbetas=0,
kp_weights=None,
fix_rot=False,
SOLVE_FLATER=True,
FIX_CAM=False,
ONLY_KEYP=False,
OPT_SHAPE=True,
DO_FREE_SHAPE=False):
nCameras = len(cam)
nClips = nCameras
params = set_params(SOLVE_FLATER, nCameras)
pose_prior = [set_pose_prior(len(sv[0].pose.r)) for _ in range(nClips)]
pose_prior_tail = [
set_pose_prior_tail(len(sv[0].pose.r)) for _ in range(nClips)
]
if OPT_SHAPE:
shape_prior = [
set_shape_prior(DO_FREE_SHAPE, animal, shape_data_name)
for _ in range(nClips)
]
# indices with no prior
noprior_ind = ~pose_prior[0].use_ind
noprior_ind[:3] = False
limit_prior = [set_limit_prior(len(sv[0].pose.r)) for _ in range(nClips)]
init_rot = [sv[ic].pose[:3].r.copy() for ic in range(nClips)]
init_trans = [sv[ic].trans.r.copy() for ic in range(nClips)]
init_pose = [sv[ic].pose.r.copy() for ic in range(nClips)]
# Setup keypoint projection error with multi verts:
j2d = [None] * nCameras
assignments = [None] * nCameras
num_points = [None] * nCameras
use_ids = [None] * nCameras
visible_vids = [None] * nCameras
all_vids = [None] * nCameras
for i in range(nCameras):
visible = landmarks[i][:, 2].astype(bool)
use_ids[i] = [
id for id in np.arange(landmarks[i].shape[0]) if visible[id]
]
visible_vids[i] = np.hstack([key_vids[i][id] for id in use_ids[i]])
group = np.hstack([
index * np.ones(len(key_vids[i][row_id]))
for index, row_id in enumerate(use_ids[i])
])
assignments[i] = np.vstack(
[group == j for j in np.arange(group[-1] + 1)])
num_points[i] = len(use_ids[i])
all_vids[i] = visible_vids[i]
cam[i].v = sv[i][all_vids[i], :]
j2d[i] = landmarks[i][use_ids[i], :2]
if kp_weights is None:
kp_weights = np.ones((landmarks[i].shape[0], 1))
def kp_proj_error(i, w, sigma):
return w * kp_weights[use_ids[i]] * GMOf(
ch.vstack([
cam[i][choice] if np.sum(choice) == 1 else cam[i][choice].mean(
axis=0) for choice in assignments[i]
]) - j2d[i], sigma) / np.sqrt(num_points[i])
objs = {}
for i in range(nCameras):
objs['kp_proj_' + str(i)] = kp_proj_error(i, params['k_kp_term'],
params['k_robust_sig'])
if not ONLY_KEYP:
objs['trans_init_' +
str(i)] = params['k_trans_term'] * (sv[i].trans -
init_trans[i])
if not ONLY_KEYP:
if fix_rot:
objs['fix_rot_' +
str(i)] = params['k_rot_term'] * (sv[i].pose[:3] -
init_rot[i])
if OPT_SHAPE:
if (i > 0):
objs['betas_var_' +
str(i)] = params['betas_var'] * ch.abs(sv[i - 1].betas -
sv[i].betas)
objs['shape_prior_' +
str(i)] = shape_prior[i](sv[i].betas) / np.sqrt(nbetas)
if not FIX_CAM:
for i in range(nCameras):
objs['feq_' + str(i)] = 1e3 * (cam[i].f[0] - cam[i].f[1])
objs['fpos_' + str(i)] = 1e3 * ch.maximum(0, 500 - cam[i].f[0])
if not SOLVE_FLATER:
for i in range(nCameras):
objs['freg_' + str(i)] = 9 * 1e2 * (cam[i].f[0] - 3000) / 1000.
objs['cam_t_pos_' +
str(i)] = 1e3 * ch.maximum(0, 0.01 - sv[i].trans[2])
del objs['trans_init_' + str(i)]
num_pose_prior = len(pose_prior[0](sv[0].pose))
num_limit_prior = len(limit_prior[0](sv[0].pose))
if not ONLY_KEYP:
if np.sum(noprior_ind) > 0:
objs['rest_poseprior_' + str(i)] = params['k_rest_pose_term'] * (
sv[i].pose[noprior_ind] - init_pose[noprior_ind]) / np.sqrt(
len(sv[i].pose[noprior_ind]))
for i in range(nClips):
objs['pose_limit_' +
str(i)] = params['k_limit_term'] * limit_prior[i](
sv[i].pose) / np.sqrt(num_limit_prior)
objs['pose_prior_' +
str(i)] = pose_prior[i](sv[i].pose) / np.sqrt(num_pose_prior)
objs['pose_prior_tail_' + str(i)] = 2.0 * pose_prior_tail[i](
sv[i].pose) / np.sqrt(num_pose_prior)
return objs, params, j2d
def optimize_smal(fposes,
ftrans,
fbetas,
model,
cams,
segs,
imgs,
landmarks,
landmarks_names,
key_vids,
symIdx=None,
frameId=0,
opt_model_dir=None,
save_name=None,
COMPUTE_OPT=True,
img_paths=None,
img_offset=None,
img_scales=None):
mesh_v_opt_save_path = join(opt_model_dir,
'mesh_v_opt_no_mc_' + str(frameId) + '.ply')
mesh_v_opt_mc_save_path = join(opt_model_dir,
'mesh_v_opt_' + str(frameId) + '.ply')
mesh_init_save_path = join(opt_model_dir,
'mesh_init_' + str(frameId) + '.ply')
nViews = len(fposes)
if not COMPUTE_OPT:
if not exists(opt_model_dir):
makedirs(opt_model_dir)
dv = 0
compute_texture(nViews, opt_model_dir, dv, model, frameId,
mesh_init_save_path, fposes, ftrans, fbetas,
'_no_refine', cams, imgs, segs, img_paths, img_offset,
img_scales)
return
# Write the initial mesh
np_betas = np.zeros_like(model.betas)
np_betas[:len(fbetas[0])] = fbetas[0]
tmp = verts_decorated(v_template=model.v_template,
pose=ch.zeros_like(model.pose.r),
trans=ch.zeros_like(model.trans),
J=model.J_regressor,
kintree_table=model.kintree_table,
betas=ch.array(np_betas),
weights=model.weights,
posedirs=model.posedirs,
shapedirs=model.shapedirs,
bs_type='lrotmin',
bs_style='lbs',
f=model.f)
tmp_mesh = Mesh(v=tmp.r, f=tmp.f)
tmp_path = join(opt_model_dir, 'mesh_init_' + str(frameId) + '.ply')
tmp_mesh.write_ply(tmp_path)
del tmp
assert (nViews == len(cams))
assert (nViews == len(segs))
assert (nViews == len(imgs))
# Define a displacement vector. We set a small non zero displacement as initialization
dv = ch.array(np.random.rand(model.r.shape[0], 3) / 1000.)
# Cell structure for ARAP
f = model.f
_, A3, A = edgesIdx(nV=dv.shape[0], f=f, save_dir='.', name='smal')
wedge = wedges(A3, dv)
s = np.zeros_like(dv)
arap = ARAP(reg_e=MatVecMult(A3.T,
model.ravel() + dv.ravel()).reshape(-1, 3),
model_e=MatVecMult(A3.T, model.ravel()).reshape(-1, 3),
w=wedge,
A=A)
k_arap = settings['ref_k_arap_per_view'] * nViews
for weight, part in zip(settings['ref_W_arap_values'],
settings['ref_W_arap_parts']):
k_arap, W_per_vertex = get_arap_part_weights(
A, k_arap, [part], [weight]) #, animal_name) # was only Head
W = np.zeros((W_per_vertex.shape[0], 3))
for i in range(3):
W[:, i] = W_per_vertex
k_lap = settings['ref_k_lap'] * nViews * W
k_sym = settings['ref_k_sym'] * nViews
k_keyp = settings['ref_k_keyp_weight'] * nViews
# Load already computed mesh
if not exists(opt_model_dir):
makedirs(opt_model_dir)
shape_model, compute = load_shape_models(nViews, opt_model_dir, dv, model,
frameId, mesh_v_opt_save_path,
fposes, ftrans, fbetas)
mv = None
# Remove inside mouth faces
'''
if settings['ref_remove_inside_mouth']:
# Giraffe
faces_orig = shape_model[0].f.copy()
im_v = im_up_v + im_down_v
idx = [np.where(model.f == ix)[0] for ix in im_v]
idx = np.concatenate(idx).ravel()
for i in range(nViews):
shape_model[i].f = np.delete(shape_model[i].f, idx, 0)
'''
if compute:
objs = {}
FIX_CAM = True
free_variables = []
kp_weights = k_keyp * np.ones((landmarks[0].shape[0], 1))
print('removing shoulders, often bad annotated')
kp_weights[landmarks_names.index('leftShoulder'), :] *= 0
kp_weights[landmarks_names.index('rightShoulder'), :] *= 0
objs_pose = None
j2d = None
#k_silh_term = settings['ref_k_silh_term']
k_m2s = settings['ref_k_m2s']
k_s2m = settings['ref_k_s2m']
objs, params_, j2d = set_pose_objs(shape_model,
cams,
landmarks,
key_vids,
kp_weights=kp_weights,
FIX_CAM=FIX_CAM,
ONLY_KEYP=True,
OPT_SHAPE=False)
if np.any(k_arap) != 0:
objs['arap'] = k_arap * arap
if k_sym != 0:
objs['sym_0'] = k_sym * (ch.abs(dv[:, 0] - dv[symIdx, 0]))
objs['sym_1'] = k_sym * (ch.abs(dv[:, 1] + dv[symIdx, 1] -
0.00014954))
objs['sym_2'] = k_sym * (ch.abs(dv[:, 2] - dv[symIdx, 2]))
if np.any(k_lap) != 0:
lap_op = np.asarray(
laplacian(Mesh(v=dv, f=shape_model[0].f)).todense())
objs['lap'] = k_lap * ch.dot(lap_op, dv)
mv = None
mv2 = MeshViewers(shape=(1, nViews)) #None
vc = np.ones_like(dv)
dv_r = fit_silhouettes_pyramid_opt(objs,
shape_model,
dv,
segs,
cams,
j2d=j2d,
weights=1.,
mv=mv,
imgs=imgs,
s2m_weights=k_s2m,
m2s_weights=k_m2s,
max_iter=100,
free_variables=free_variables,
vc=vc,
symIdx=symIdx,
mv2=mv2,
objs_pose=objs_pose)
# Save result image
for i in range(nViews):
img_res = render_mesh(Mesh(shape_model[i].r, shape_model[i].f),
imgs[i].shape[1],
imgs[i].shape[0],
cams[i],
img=imgs[i],
world_frame=True)
img_result = np.hstack((imgs[i], img_res * 255.))
save_img_path = save_name[i].replace('.pkl', '_v_opt.png')
cv2.imwrite(save_img_path, img_result)
shape_model[0].pose[:] = 0
shape_model[0].trans[:] = 0
V = shape_model[0].r.copy()
vm = V[symIdx, :].copy()
vm[:, 1] = -1 * vm[:, 1]
V2 = (V + vm) / 2.0
mesh_out = Mesh(v=V2, f=shape_model[0].f)
mesh_out.show()
mesh_out.write_ply(mesh_v_opt_save_path)
save_dv_data_path = mesh_v_opt_save_path.replace('.ply', '_dv.pkl')
dv_data = {'betas': shape_model[0].betas.r, 'dv': dv_r}
pkl.dump(dv_data, open(save_dv_data_path, 'wb'))
compute_texture(nViews, opt_model_dir, dv, model, frameId,
mesh_v_opt_save_path, fposes, ftrans, fbetas, '_non_opt',
cams, imgs, segs, img_paths, img_offset, img_scales)
return
