def inference_by_smpls(self,
tgt_smpls,
cam_strategy='smooth',
output_dir='',
visualizer=None):
length = len(tgt_smpls)
outputs = []
for t in tqdm(range(length)):
tgt_smpl = tgt_smpls[t] if tgt_smpls is not None else None
tsf_inputs = self.transfer_params_by_smpl(tgt_smpl, cam_strategy)
preds = self.forward(tsf_inputs, self.tsf_info['T'])
if visualizer is not None:
gt = cv_utils.transform_img(self.tsf_info['image'],
image_size=self._opt.image_size,
transpose=True)
visualizer.vis_named_img('pred_' + cam_strategy, preds)
visualizer.vis_named_img('gt',
gt[None, ...],
denormalize=False)
preds = preds[0].permute(1, 2, 0)
preds = preds.cpu().numpy()
outputs.append(preds)
if output_dir:
cv_utils.save_cv2_img(preds,
os.path.join(output_dir,
'pred_%.8d.jpg' % t),
normalize=True)
return outputs
def inference(self, tgt_paths, tgt_smpls=None, cam_strategy='smooth', output_dir='', visualizer=None, verbose=True):
length = len(tgt_paths)
outputs = []
process_bar = tqdm(range(length)) if verbose else range(length)
for t in process_bar:
tgt_path = tgt_paths[t]
tgt_smpl = tgt_smpls[t] if tgt_smpls is not None else None
tsf_inputs = self.transfer_params(tgt_path, tgt_smpl, cam_strategy, t=t)
preds = self.forward(tsf_inputs, self.tsf_info['T'])
if visualizer is not None:
gt = cv_utils.transform_img(self.tsf_info['image'], image_size=self._opt.image_size, transpose=True)
visualizer.vis_named_img('pred_' + cam_strategy, preds)
visualizer.vis_named_img('gt', gt[None, ...], denormalize=False)
preds = preds[0].permute(1, 2, 0)
preds = preds.cpu().numpy()
outputs.append(preds)
if output_dir:
filename = os.path.split(tgt_path)[-1]
cv_utils.save_cv2_img(preds, os.path.join(output_dir, 'pred_' + filename), normalize=True)
cv_utils.save_cv2_img(self.tsf_info['image'], os.path.join(output_dir, 'gt_' + filename),
image_size=self._opt.image_size)
return outputs
def meta_imitate(opt, imitator, prior_tgt_path, save_imgs=True, visualizer=None):
src_path = opt.src_path
all_tgt_paths = scan_tgt_paths(prior_tgt_path, itv=40)
output_dir = opt.output_dir
out_img_dir, out_pair_dir = mkdirs([os.path.join(output_dir, 'imgs'), os.path.join(output_dir, 'pairs')])
img_pair_list = []
for t in tqdm(range(len(all_tgt_paths))):
tgt_path = all_tgt_paths[t]
preds = imitator.inference([tgt_path], visualizer=visualizer, cam_strategy=opt.cam_strategy, verbose=False)
tgt_name = os.path.split(tgt_path)[-1]
out_path = os.path.join(out_img_dir, 'pred_' + tgt_name)
if save_imgs:
cv_utils.save_cv2_img(preds[0], out_path, normalize=True)
write_pair_info(imitator.src_info, imitator.tsf_info,
os.path.join(out_pair_dir, '{:0>8}.pkl'.format(t)), imitator=imitator,
only_vis=opt.only_vis)
img_pair_list.append((src_path, tgt_path))
if save_imgs:
write_pickle_file(os.path.join(output_dir, 'pairs_meta.pkl'), img_pair_list)
def inference(self,
tgt_paths,
tgt_smpls=None,
cam_strategy='smooth',
output_dir='',
visualizer=None,
verbose=True):
length = len(tgt_paths)
outputs = []
bg_img = self.src_info['bg']
src_encoder_outs, src_resnet_outs = self.src_info['feats']
process_bar = tqdm(range(length)) if verbose else range(length)
for t in process_bar:
tgt_path = tgt_paths[t]
tgt_smpl = tgt_smpls[t] if tgt_smpls is not None else None
tsf_inputs = self.transfer_params(tgt_path,
tgt_smpl,
cam_strategy,
t=t)
tsf_color, tsf_mask = self.generator.inference(
src_encoder_outs, src_resnet_outs, tsf_inputs,
self.tsf_info['T'])
preds = tsf_mask * bg_img + (1 - tsf_mask) * tsf_color
if self._opt.front_warp:
preds = self.warp_front(preds, self.tsf_info['tsf_img'],
self.tsf_info['fim'], tsf_mask)
if visualizer is not None:
gt = cv_utils.transform_img(self.tsf_info['image'],
image_size=self._opt.image_size,
transpose=True)
visualizer.vis_named_img('pred_' + cam_strategy, preds)
visualizer.vis_named_img('gt',
gt[None, ...],
denormalize=False)
preds = preds[0].permute(1, 2, 0)
preds = preds.cpu().numpy()
outputs.append(preds)
if output_dir:
filename = os.path.split(tgt_path)[-1]
cv_utils.save_cv2_img(preds,
os.path.join(output_dir,
'pred_' + filename),
normalize=True)
cv_utils.save_cv2_img(self.tsf_info['image'],
os.path.join(output_dir,
'gt_' + filename),
image_size=self._opt.image_size)
return outputs
def imitate(self, src_infos: Dict[str, Any],
ref_infos: Dict[str, Any]) -> List[str]:
"""
Running the motion imitation of the self.model, based on the source information with respect to the
provided reference information. It returns the full paths of synthesized images.
Args:
src_infos (dict): the source information contains:
--images (list of str): the list of full paths of source images (the length is 1)
--smpls (np.ndarray): (length of images, 85)
--kps (np.ndarray): (length of images, 19, 2)
ref_infos (dict): the reference information contains:
--images (list of str): the list of full paths of reference images.
--smpls (np.ndarray): (length of images, 85)
--kps (np.ndarray): (length of images, 19, 2)
--self_imitation (bool): the flag indicates whether it is self-imitation or not.
Returns:
preds_files (list of str): full paths of synthesized images with respects to the images in ref_infos.
"""
tgt_paths = ref_infos["images"]
tgt_smpls = ref_infos["smpls"]
self_imitation = ref_infos["self_imitation"]
if self_imitation:
cam_strategy = "copy"
out_dir = self.si_out_dir
count = self.num_preds_si
self.num_preds_si += len(tgt_paths)
else:
cam_strategy = "smooth"
out_dir = self.ci_out_dir
count = self.num_preds_ci
self.num_preds_ci += len(tgt_paths)
outputs = self.model.inference(tgt_paths,
tgt_smpls=tgt_smpls,
cam_strategy=cam_strategy,
visualizer=None,
verbose=True)
all_preds_files = []
for i, preds in enumerate(outputs):
filename = "{:0>8}.jpg".format(count)
pred_file = os.path.join(out_dir, 'pred_' + filename)
count += 1
cv_utils.save_cv2_img(preds, pred_file, normalize=True)
all_preds_files.append(pred_file)
return all_preds_files
def save_results(src_path, tgt_path, output_dir, preds):
"""
Save the results.
"""
import utils.cv_utils as cv_utils
src_name = get_img_name(src_path)
tgt_name = get_img_name(tgt_path)
preds = preds[0].permute(1, 2, 0)
preds = preds.cpu().numpy()
filepath = os.path.join(output_dir,
'{}->{}.png'.format(src_name, tgt_name))
cv_utils.save_cv2_img(preds, filepath, normalize=True)
print('\n\t\t\tSaving results to {}'.format(filepath))
def personalize(self, src_path, src_smpl=None, output_path='', visualizer=None):
ori_img = cv_utils.read_cv2_img(src_path)
# resize image and convert the color space from [0, 255] to [-1, 1]
img = cv_utils.transform_img(ori_img, self._opt.image_size, transpose=True) * 2 - 1.0
img = torch.tensor(img, dtype=torch.float32).cuda()[None, ...]
if src_smpl is None:
img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0
img_hmr = torch.tensor(img_hmr, dtype=torch.float32).cuda()[None, ...]
src_smpl = self.hmr(img_hmr)
else:
src_smpl = torch.tensor(src_smpl, dtype=torch.float32).cuda()[None, ...]
# source process, {'theta', 'cam', 'pose', 'shape', 'verts', 'j2d', 'j3d'}
src_info = self.hmr.get_details(src_smpl)
src_f2verts, src_fim, src_wim = self.render.render_fim_wim(src_info['cam'], src_info['verts'])
# src_f2pts = src_f2verts[:, :, :, 0:2]
src_info['fim'] = src_fim
src_info['wim'] = src_wim
src_info['cond'], _ = self.render.encode_fim(src_info['cam'], src_info['verts'], fim=src_fim, transpose=True)
src_info['f2verts'] = src_f2verts
src_info['p2verts'] = src_f2verts[:, :, :, 0:2]
src_info['p2verts'][:, :, :, 1] *= -1
if self._opt.only_vis:
src_info['p2verts'] = self.render.get_vis_f2pts(src_info['p2verts'], src_fim)
src_info['part'], _ = self.render.encode_fim(src_info['cam'], src_info['verts'],
fim=src_fim, transpose=True, map_fn=self.part_fn)
# add image to source info
src_info['img'] = img
src_info['image'] = ori_img
# 2. process the src inputs
if self.detector is not None:
bbox, body_mask = self.detector.inference(img[0])
bg_mask = 1 - body_mask
else:
bg_mask = util.morph(src_info['cond'][:, -1:, :, :], ks=self._opt.bg_ks, mode='erode')
body_mask = 1 - bg_mask
if self._opt.bg_model != 'ORIGINAL':
src_info['bg'] = self.bgnet(img, masks=body_mask, only_x=True)
else:
incomp_img = img * bg_mask
bg_inputs = torch.cat([incomp_img, bg_mask], dim=1)
img_bg = self.bgnet(bg_inputs)
src_info['bg_inputs'] = bg_inputs
# src_info['bg'] = img_bg
src_info['bg'] = incomp_img + img_bg * body_mask
ft_mask = 1 - util.morph(src_info['cond'][:, -1:, :, :], ks=self._opt.ft_ks, mode='erode')
src_inputs = torch.cat([img * ft_mask, src_info['cond']], dim=1)
src_info['feats'] = self.generator.encode_src(src_inputs)
src_info['src_inputs'] = src_inputs
src_info = src_info
# if visualizer is not None:
# self.visualize(visualizer, src=img, bg=src_info['bg'])
if output_path:
cv_utils.save_cv2_img(src_info['image'], output_path, image_size=self._opt.image_size)
return src_info
