def fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display):
if nohands:
faces = faces_no_hands(base_smpl.f)
else:
faces = base_smpl.f
vis_rn_b = BoundaryRenderer(camera=camera, frustum=frustum, f=faces, num_channels=1)
vis_rn_m = ColoredRenderer(camera=camera, frustum=frustum, f=faces, vc=np.zeros_like(base_smpl), bgcolor=1,
num_channels=1)
model_template = Smpl(model_data)
model_template.betas[:] = base_smpl.betas.r
g_laplace = regularize_laplace()
g_model = regularize_model()
g_symmetry = regularize_symmetry()
face_ids = get_face_vertex_ids()
for step, (w_laplace, w_model, w_symmetry, sigma) in enumerate(zip(
np.linspace(6.5, 4.0, icp_count) if naked else np.linspace(4.0, 2.0, icp_count),
np.linspace(0.9, 0.6, icp_count) if naked else np.linspace(0.6, 0.3, icp_count),
np.linspace(3.6, 1.8, icp_count),
np.linspace(0.06, 0.003, icp_count),
)):
log.info('# Step {}'.format(step))
L = laplacian(model_template.r, base_smpl.f)
delta = L.dot(model_template.r)
w_laplace *= g_laplace.reshape(-1, 1)
w_model *= g_model.reshape(-1, 1)
w_symmetry *= g_symmetry.reshape(-1, 1)
E = {
'laplace': (sp_dot(L, base_smpl.v_shaped_personal) - delta) * w_laplace,
'model': (base_smpl.v_shaped_personal - model_template) * w_model,
'symmetry': (base_smpl.v_personal + np.array([1, -1, -1])
* base_smpl.v_personal[model_data['vert_sym_idxs']]) * w_symmetry,
}
log.info('## Matching rays with contours')
for current, f in enumerate(tqdm(frames)):
E['silh_{}'.format(current)] = ray_objective(f, sigma, base_smpl, camera, vis_rn_b, vis_rn_m)
#paper 2
E['face_{}'.format(current)] = ray_face(f, sigma, base_smpl, camera, face_ids)
log.info('## Run optimization')
ch.minimize(
E,
[base_smpl.v_personal, model_template.betas],
method='dogleg',
options={'maxiter': 15, 'e_3': 0.001},
callback=get_cb(frames[0], base_smpl, camera, frustum) if display else None
)
def main(pose_file, masks_file, camera_file, out, obj_out, num, icp_count, model_file, first_frame, last_frame,
nohands, naked, display):
# load data
with open(model_file, 'rb') as fp:
model_data = pkl.load(fp)
with open(camera_file, 'rb') as fp:
camera_data = pkl.load(fp)
pose_data = h5py.File(pose_file, 'r')
poses = pose_data['pose'][first_frame:last_frame]
trans = pose_data['trans'][first_frame:last_frame]
masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame]
num_frames = masks.shape[0]
indices_consensus = np.ceil(np.arange(num) * num_frames * 1. / num).astype(np.int)
# init
base_smpl = Smpl(model_data)
base_smpl.betas[:] = np.array(pose_data['betas'], dtype=np.float32)
camera = ProjectPoints(t=np.zeros(3), rt=np.zeros(3), c=camera_data['camera_c'],
f=camera_data['camera_f'], k=camera_data['camera_k'], v=base_smpl)
camera_t = camera_data['camera_t']
camera_rt = camera_data['camera_rt']
frustum = {'near': 0.1, 'far': 1000., 'width': int(camera_data['width']), 'height': int(camera_data['height'])}
frames = []
for i in indices_consensus:
log.info('Set up frame {}...'.format(i))
mask = np.array(masks[i] * 255, dtype=np.uint8)
pose_i = np.array(poses[i], dtype=np.float32)
trans_i = np.array(trans[i], dtype=np.float32)
frames.append(setup_frame_rays(base_smpl, camera, camera_t, camera_rt, pose_i, trans_i, mask))
log.info('Set up complete.')
log.info('Begin consensus fit...')
fit_consensus(frames, base_smpl, camera, frustum, model_data, nohands, icp_count, naked, display)
with open(out, 'wb') as fp:
pkl.dump({
'v_personal': base_smpl.v_personal.r,
'betas': base_smpl.betas.r,
}, fp, protocol=2)
if obj_out is not None:
base_smpl.pose[:] = 0
vt = np.load('assets/basicModel_vt.npy')
ft = np.load('assets/basicModel_ft.npy')
mesh.write(obj_out, base_smpl.r, base_smpl.f, vt=vt, ft=ft)
log.info('Done.')
def main(keypoint_file, masks_file, camera_file, out, model_file, prior_file,
resize, body_height, nohands, display):
# load data
with open(model_file, 'rb') as fp:
model_data = pkl.load(fp, encoding='latin1')
with open(camera_file, 'rb') as fp:
camera_data = pkl.load(fp, encoding='latin1')
with open(prior_file, 'rb') as fp:
prior_data = pkl.load(fp, encoding='latin1')
if 'basicModel_f' in model_file:
regs = np.load(
'vendor/smplify/models/regressors_locked_normalized_female.npz')
b2m = np.load('assets/b2m_f.npy')
else:
regs = np.load(
'vendor/smplify/models/regressors_locked_normalized_male.npz')
b2m = np.load('assets/b2m_m.npy')
keypoints = h5py.File(keypoint_file, 'r')['keypoints']
masks = h5py.File(masks_file, 'r')['masks']
num_frames = masks.shape[0]
# init
base_smpl = Smpl(model_data)
base_smpl.trans[:] = np.array([0, 0, 3])
base_smpl.pose[0] = np.pi
base_smpl.pose[3:] = prior_data['mean']
camera = ProjectPoints(t=np.zeros(3),
rt=np.zeros(3),
c=camera_data['camera_c'] * resize,
f=camera_data['camera_f'] * resize,
k=camera_data['camera_k'],
v=base_smpl)
frustum = {
'near': 0.1,
'far': 1000.,
'width': int(camera_data['width'] * resize),
'height': int(camera_data['height'] * resize)
}
if display:
debug_cam = ProjectPoints(v=base_smpl,
t=camera.t,
rt=camera.rt,
c=camera.c,
f=camera.f,
k=camera.k)
debug_light = LambertianPointLight(f=base_smpl.f,
v=base_smpl,
num_verts=len(base_smpl),
light_pos=np.zeros(3),
vc=np.ones(3),
light_color=np.ones(3))
debug_rn = ColoredRenderer(camera=debug_cam,
v=base_smpl,
f=base_smpl.f,
vc=debug_light,
frustum=frustum)
else:
debug_rn = None
# generic frame loading function
def create_frame(i, smpl, copy=True):
f = FrameData()
f.smpl = copy_smpl(smpl, model_data) if copy else smpl
f.camera = ProjectPoints(v=f.smpl,
t=camera.t,
rt=camera.rt,
c=camera.c,
f=camera.f,
k=camera.k)
f.keypoints = np.array(keypoints[i]).reshape(-1, 3) * np.array(
[resize, resize, 1])
f.J = joints_coco(f.smpl)
f.J_proj = ProjectPoints(v=f.J,
t=camera.t,
rt=camera.rt,
c=camera.c,
f=camera.f,
k=camera.k)
f.mask = cv2.resize(np.array(masks[i], dtype=np.float32), (0, 0),
fx=resize,
fy=resize,
interpolation=cv2.INTER_NEAREST)
f.collision_obj = collision_obj(f.smpl, regs)
f.pose_prior_obj = pose_prior_obj(f.smpl, prior_data)
f.pose_obj = (f.J_proj -
f.keypoints[:, :2]) * f.keypoints[:, 2].reshape(-1, 1)
return f
base_frame = create_frame(0, base_smpl, copy=False)
# get betas from 5 frames
log.info('Initial fit')
num_init = 5
indices_init = np.ceil(np.arange(num_init) * num_frames * 1. /
num_init).astype(np.int)
init_frames = [base_frame]
for i in indices_init[1:]:
init_frames.append(create_frame(i, base_smpl))
init(init_frames, body_height, b2m, debug_rn)
# get pose frame by frame
with h5py.File(out, 'w') as fp:
last_smpl = None
poses_dset = fp.create_dataset("pose", (num_frames, 72),
'f',
chunks=True,
compression="lzf")
trans_dset = fp.create_dataset("trans", (num_frames, 3),
'f',
chunks=True,
compression="lzf")
betas_dset = fp.create_dataset("betas", (10, ),
'f',
chunks=True,
compression="lzf")
for i in xrange(num_frames):
if i == 0:
current_frame = base_frame
else:
current_frame = create_frame(i, last_smpl)
log.info('Fit frame {}'.format(i))
# re-init if necessary
reinit_frame(current_frame, prior_data['mean'], nohands, debug_rn)
# final fit
fit_pose(current_frame, last_smpl, frustum, nohands, debug_rn)
poses_dset[i] = current_frame.smpl.pose.r
trans_dset[i] = current_frame.smpl.trans.r
if i == 0:
betas_dset[:] = current_frame.smpl.betas.r
last_smpl = current_frame.smpl
log.info('Done.')
def main(consensus_file, camera_file, video_file, pose_file, masks_file, out,
model_file, resolution, num, first_frame, last_frame, display):
# load data
with open(model_file, 'rb') as fp:
model_data = pkl.load(fp)
with open(camera_file, 'rb') as fp:
camera_data = pkl.load(fp)
with open(consensus_file, 'rb') as fp:
consensus_data = pkl.load(fp)
pose_data = h5py.File(pose_file, 'r')
poses = pose_data['pose'][first_frame:last_frame]
trans = pose_data['trans'][first_frame:last_frame]
masks = h5py.File(masks_file, 'r')['masks'][first_frame:last_frame]
num_frames = masks.shape[0]
indices_texture = np.ceil(np.arange(num) * num_frames * 1. / num).astype(
np.int)
vt = np.load('assets/basicModel_vt.npy')
ft = np.load('assets/basicModel_ft.npy')
# init
base_smpl = Smpl(model_data)
base_smpl.betas[:] = consensus_data['betas']
base_smpl.v_personal[:] = consensus_data['v_personal']
bgcolor = np.array([1., 0.2, 1.])
iso = Isomapper(vt, ft, base_smpl.f, resolution, bgcolor=bgcolor)
iso_vis = IsoColoredRenderer(vt, ft, base_smpl.f, resolution)
camera = ProjectPoints(t=camera_data['camera_t'],
rt=camera_data['camera_rt'],
c=camera_data['camera_c'],
f=camera_data['camera_f'],
k=camera_data['camera_k'],
v=base_smpl)
frustum = {
'near': 0.1,
'far': 1000.,
'width': int(camera_data['width']),
'height': int(camera_data['height'])
}
rn_vis = ColoredRenderer(f=base_smpl.f,
frustum=frustum,
camera=camera,
num_channels=1)
cap = cv2.VideoCapture(video_file)
for _ in range(first_frame):
cap.grab()
# get part-textures
i = first_frame
tex_agg = np.zeros((resolution, resolution, 25, 3))
tex_agg[:] = np.nan
normal_agg = np.ones((resolution, resolution, 25)) * 0.2
vn = VertNormals(f=base_smpl.f, v=base_smpl)
static_indices = np.indices((resolution, resolution))
while cap.isOpened() and i < indices_texture[-1]:
if i in indices_texture:
log.info('Getting part texture from frame {}...'.format(i))
_, frame = cap.read()
mask = np.array(masks[i], dtype=np.uint8)
pose_i = np.array(poses[i], dtype=np.float32)
trans_i = np.array(trans[i], dtype=np.float32)
base_smpl.pose[:] = pose_i
base_smpl.trans[:] = trans_i
# which faces have been seen and are projected into the silhouette?
visibility = rn_vis.visibility_image.ravel()
visible = np.nonzero(visibility != 4294967295)[0]
proj = camera.r
in_viewport = np.logical_and(
np.logical_and(
np.round(camera.r[:, 0]) >= 0,
np.round(camera.r[:, 0]) < frustum['width']),
np.logical_and(
np.round(camera.r[:, 1]) >= 0,
np.round(camera.r[:, 1]) < frustum['height']),
)
in_mask = np.zeros(camera.shape[0], dtype=np.bool)
idx = np.round(proj[in_viewport][:, [1, 0]].T).astype(
np.int).tolist()
in_mask[in_viewport] = mask[idx]
faces_in_mask = np.where(np.min(in_mask[base_smpl.f], axis=1))[0]
visible_faces = np.intersect1d(faces_in_mask, visibility[visible])
# get the current unwrap
part_tex = iso.render(frame / 255., camera, visible_faces)
# angle under which the texels have been seen
points = np.hstack((proj, np.ones((proj.shape[0], 1))))
points3d = camera.unproject_points(points)
points3d /= np.linalg.norm(points3d, axis=1).reshape(-1, 1)
alpha = np.sum(points3d * -vn.r, axis=1).reshape(-1, 1)
alpha[alpha < 0] = 0
iso_normals = iso_vis.render(alpha)[:, :, 0]
iso_normals[np.all(part_tex == bgcolor, axis=2)] = 0
# texels to consider
part_mask = np.zeros((resolution, resolution))
min_normal = np.min(normal_agg, axis=2)
part_mask[iso_normals > min_normal] = 1.
# update best seen texels
where = np.argmax(np.atleast_3d(iso_normals) - normal_agg, axis=2)
idx = np.dstack(
(static_indices[0], static_indices[1], where))[part_mask == 1]
tex_agg[list(idx[:, 0]),
list(idx[:, 1]),
list(idx[:, 2])] = part_tex[part_mask == 1]
normal_agg[list(idx[:, 0]),
list(idx[:, 1]),
list(idx[:, 2])] = iso_normals[part_mask == 1]
if display:
im.show(part_tex, id='part_tex', waittime=1)
else:
cap.grab()
i += 1
# merge textures
log.info('Computing median texture...')
tex_median = np.nanmedian(tex_agg, axis=2)
log.info('Inpainting unseen areas...')
where = np.max(normal_agg, axis=2) > 0.2
tex_mask = iso.iso_mask
mask_final = np.float32(where)
kernel_size = np.int(resolution * 0.02)
kernel = np.ones((kernel_size, kernel_size), np.uint8)
inpaint_area = cv2.dilate(tex_mask, kernel) - mask_final
tex_final = cv2.inpaint(np.uint8(tex_median * 255),
np.uint8(inpaint_area * 255), 3, cv2.INPAINT_TELEA)
cv2.imwrite(out, tex_final)
log.info('Done.')
#print(join(ROOT,imgname))
rgb_img = cv2.imread(join(ROOT,
imgname))[:, :, ::-1].copy().astype(np.float32)
center = mpi_inf_valid['center'][i]
scale = mpi_inf_valid['scale'][i]
rgb_img = crop(rgb_img, center, scale, [IMG_RES, IMG_RES])
pose = mpi_inf_pred['pose'][i]
betas = mpi_inf_pred['betas'][i]
camera = mpi_inf_pred['camera'][i]
#gt_keypoints[i*batch_size*4+k*4+j] = mpi_inf_valid['S'][i*batch_size+j][k]
camera_t = np.array([
camera[1], camera[2], 2 * focal_length / (IMG_RES * camera[0] + 1e-9)
])
w, h = (IMG_RES, IMG_RES)
rn = ColoredRenderer()
pred_base_smpl = Smpl(model_data)
pred_base_smpl.pose[:] = pose
pred_base_smpl.betas[:] = betas
pred_rot = np.eye(3)
rn.camera = ProjectPoints(t=camera_t,
rt=cv2.Rodrigues(pred_rot)[0].reshape(3),
c=np.array([112, 112]),
f=np.array([5000, 5000]),
k=np.zeros(5),
v=pred_base_smpl)
dist = np.abs(rn.camera.t.r[2] - np.mean(pred_base_smpl.r, axis=0)[2])
verts = pred_base_smpl.r
im = (
render_model(verts, pred_base_smpl.f, w, h, rn.camera, far=20 + dist) *
255.).astype('uint8')
def __init__(self, keypoint_file, masks_file, camera_file, out, model_file,
prior_file, resize, body_height, nohands, display):
# load data
with open(model_file, 'rb') as fp:
self.model_data = pkl.load(fp)
with open(camera_file, 'rb') as fp:
self.camera_data = pkl.load(fp)
with open(prior_file, 'rb') as fp:
self.prior_data = pkl.load(fp)
if 'basicModel_f' in model_file:
self.regs = np.load(
'vendor/smplify/models/regressors_locked_normalized_female.npz'
)
self.b2m = np.load('assets/b2m_f.npy')
else:
self.regs = np.load(
'vendor/smplify/models/regressors_locked_normalized_male.npz')
self.b2m = np.load('assets/b2m_m.npy')
self.keypoints = h5py.File(keypoint_file, 'r')['keypoints']
self.masks = h5py.File(masks_file, 'r')['masks']
self.num_frames = self.masks.shape[0]
self.resize = resize
self.body_height = body_height
self.nohands = nohands
self.out = out
# init
self.base_smpl = Smpl(self.model_data)
self.base_smpl.trans[:] = np.array([0, 0, 3])
self.base_smpl.pose[0] = np.pi
self.base_smpl.pose[3:] = self.prior_data['mean']
self.camera = ProjectPoints(t=np.zeros(3),
rt=np.zeros(3),
c=self.camera_data['camera_c'] * resize,
f=self.camera_data['camera_f'] * resize,
k=self.camera_data['camera_k'],
v=self.base_smpl)
self.frustum = {
'near': 0.1,
'far': 1000.,
'width': int(self.camera_data['width'] * resize),
'height': int(self.camera_data['height'] * resize)
}
if display:
self.debug_cam = ProjectPoints(v=self.base_smpl,
t=self.camera.t,
rt=self.camera.rt,
c=self.camera.c,
f=self.camera.f,
k=self.camera.k)
self.debug_light = LambertianPointLight(f=self.base_smpl.f,
v=self.base_smpl,
num_verts=len(
self.base_smpl),
light_pos=np.zeros(3),
vc=np.ones(3),
light_color=np.ones(3))
self.debug_rn = ColoredRenderer(camera=self.debug_cam,
v=self.base_smpl,
f=self.base_smpl.f,
vc=self.debug_light,
frustum=self.frustum)
else:
self.debug_rn = None
self.base_frame = self.create_frame(0, self.base_smpl, copy=False)
self.temp_poses_dset = Manager().list(
[0 for i in range(self.num_frames)])
self.temp_trans_dset = Manager().list(
[0 for i in range(self.num_frames)])
self.temp_betas_dset = Manager().list([0 for i in range(10)])
num_init = 5
indices_init = np.ceil(
np.arange(num_init) * self.num_frames * 1. / num_init).astype(
np.int)
init_frames = [self.base_frame]
for i in indices_init[1:]:
init_frames.append(self.create_frame(i, self.base_smpl))
self.init(init_frames, self.body_height, self.b2m, self.debug_rn)