def inference_by_smpls(self, tgt_smpls, cam_strategy='smooth', output_dir='', visualizer=None): length = len(tgt_smpls) outputs = [] for t in tqdm(range(length)): tgt_smpl = tgt_smpls[t] if tgt_smpls is not None else None tsf_inputs = self.transfer_params_by_smpl(tgt_smpl, cam_strategy) preds = self.forward(tsf_inputs, self.tsf_info['T']) if visualizer is not None: gt = cv_utils.transform_img(self.tsf_info['image'], image_size=self._opt.image_size, transpose=True) visualizer.vis_named_img('pred_' + cam_strategy, preds) visualizer.vis_named_img('gt', gt[None, ...], denormalize=False) preds = preds[0].permute(1, 2, 0) preds = preds.cpu().numpy() outputs.append(preds) if output_dir: cv_utils.save_cv2_img(preds, os.path.join(output_dir, 'pred_%.8d.jpg' % t), normalize=True) return outputs
def inference(self, tgt_paths, tgt_smpls=None, cam_strategy='smooth', output_dir='', visualizer=None, verbose=True): length = len(tgt_paths) outputs = [] process_bar = tqdm(range(length)) if verbose else range(length) for t in process_bar: tgt_path = tgt_paths[t] tgt_smpl = tgt_smpls[t] if tgt_smpls is not None else None tsf_inputs = self.transfer_params(tgt_path, tgt_smpl, cam_strategy, t=t) preds = self.forward(tsf_inputs, self.tsf_info['T']) if visualizer is not None: gt = cv_utils.transform_img(self.tsf_info['image'], image_size=self._opt.image_size, transpose=True) visualizer.vis_named_img('pred_' + cam_strategy, preds) visualizer.vis_named_img('gt', gt[None, ...], denormalize=False) preds = preds[0].permute(1, 2, 0) preds = preds.cpu().numpy() outputs.append(preds) if output_dir: filename = os.path.split(tgt_path)[-1] cv_utils.save_cv2_img(preds, os.path.join(output_dir, 'pred_' + filename), normalize=True) cv_utils.save_cv2_img(self.tsf_info['image'], os.path.join(output_dir, 'gt_' + filename), image_size=self._opt.image_size) return outputs
def meta_imitate(opt, imitator, prior_tgt_path, save_imgs=True, visualizer=None): src_path = opt.src_path all_tgt_paths = scan_tgt_paths(prior_tgt_path, itv=40) output_dir = opt.output_dir out_img_dir, out_pair_dir = mkdirs([os.path.join(output_dir, 'imgs'), os.path.join(output_dir, 'pairs')]) img_pair_list = [] for t in tqdm(range(len(all_tgt_paths))): tgt_path = all_tgt_paths[t] preds = imitator.inference([tgt_path], visualizer=visualizer, cam_strategy=opt.cam_strategy, verbose=False) tgt_name = os.path.split(tgt_path)[-1] out_path = os.path.join(out_img_dir, 'pred_' + tgt_name) if save_imgs: cv_utils.save_cv2_img(preds[0], out_path, normalize=True) write_pair_info(imitator.src_info, imitator.tsf_info, os.path.join(out_pair_dir, '{:0>8}.pkl'.format(t)), imitator=imitator, only_vis=opt.only_vis) img_pair_list.append((src_path, tgt_path)) if save_imgs: write_pickle_file(os.path.join(output_dir, 'pairs_meta.pkl'), img_pair_list)
def inference(self, tgt_paths, tgt_smpls=None, cam_strategy='smooth', output_dir='', visualizer=None, verbose=True): length = len(tgt_paths) outputs = [] bg_img = self.src_info['bg'] src_encoder_outs, src_resnet_outs = self.src_info['feats'] process_bar = tqdm(range(length)) if verbose else range(length) for t in process_bar: tgt_path = tgt_paths[t] tgt_smpl = tgt_smpls[t] if tgt_smpls is not None else None tsf_inputs = self.transfer_params(tgt_path, tgt_smpl, cam_strategy, t=t) tsf_color, tsf_mask = self.generator.inference( src_encoder_outs, src_resnet_outs, tsf_inputs, self.tsf_info['T']) preds = tsf_mask * bg_img + (1 - tsf_mask) * tsf_color if self._opt.front_warp: preds = self.warp_front(preds, self.tsf_info['tsf_img'], self.tsf_info['fim'], tsf_mask) if visualizer is not None: gt = cv_utils.transform_img(self.tsf_info['image'], image_size=self._opt.image_size, transpose=True) visualizer.vis_named_img('pred_' + cam_strategy, preds) visualizer.vis_named_img('gt', gt[None, ...], denormalize=False) preds = preds[0].permute(1, 2, 0) preds = preds.cpu().numpy() outputs.append(preds) if output_dir: filename = os.path.split(tgt_path)[-1] cv_utils.save_cv2_img(preds, os.path.join(output_dir, 'pred_' + filename), normalize=True) cv_utils.save_cv2_img(self.tsf_info['image'], os.path.join(output_dir, 'gt_' + filename), image_size=self._opt.image_size) return outputs
def imitate(self, src_infos: Dict[str, Any], ref_infos: Dict[str, Any]) -> List[str]: """ Running the motion imitation of the self.model, based on the source information with respect to the provided reference information. It returns the full paths of synthesized images. Args: src_infos (dict): the source information contains: --images (list of str): the list of full paths of source images (the length is 1) --smpls (np.ndarray): (length of images, 85) --kps (np.ndarray): (length of images, 19, 2) ref_infos (dict): the reference information contains: --images (list of str): the list of full paths of reference images. --smpls (np.ndarray): (length of images, 85) --kps (np.ndarray): (length of images, 19, 2) --self_imitation (bool): the flag indicates whether it is self-imitation or not. Returns: preds_files (list of str): full paths of synthesized images with respects to the images in ref_infos. """ tgt_paths = ref_infos["images"] tgt_smpls = ref_infos["smpls"] self_imitation = ref_infos["self_imitation"] if self_imitation: cam_strategy = "copy" out_dir = self.si_out_dir count = self.num_preds_si self.num_preds_si += len(tgt_paths) else: cam_strategy = "smooth" out_dir = self.ci_out_dir count = self.num_preds_ci self.num_preds_ci += len(tgt_paths) outputs = self.model.inference(tgt_paths, tgt_smpls=tgt_smpls, cam_strategy=cam_strategy, visualizer=None, verbose=True) all_preds_files = [] for i, preds in enumerate(outputs): filename = "{:0>8}.jpg".format(count) pred_file = os.path.join(out_dir, 'pred_' + filename) count += 1 cv_utils.save_cv2_img(preds, pred_file, normalize=True) all_preds_files.append(pred_file) return all_preds_files
def save_results(src_path, tgt_path, output_dir, preds): """ Save the results. """ import utils.cv_utils as cv_utils src_name = get_img_name(src_path) tgt_name = get_img_name(tgt_path) preds = preds[0].permute(1, 2, 0) preds = preds.cpu().numpy() filepath = os.path.join(output_dir, '{}->{}.png'.format(src_name, tgt_name)) cv_utils.save_cv2_img(preds, filepath, normalize=True) print('\n\t\t\tSaving results to {}'.format(filepath))
def personalize(self, src_path, src_smpl=None, output_path='', visualizer=None): ori_img = cv_utils.read_cv2_img(src_path) # resize image and convert the color space from [0, 255] to [-1, 1] img = cv_utils.transform_img(ori_img, self._opt.image_size, transpose=True) * 2 - 1.0 img = torch.tensor(img, dtype=torch.float32).cuda()[None, ...] if src_smpl is None: img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0 img_hmr = torch.tensor(img_hmr, dtype=torch.float32).cuda()[None, ...] src_smpl = self.hmr(img_hmr) else: src_smpl = torch.tensor(src_smpl, dtype=torch.float32).cuda()[None, ...] # source process, {'theta', 'cam', 'pose', 'shape', 'verts', 'j2d', 'j3d'} src_info = self.hmr.get_details(src_smpl) src_f2verts, src_fim, src_wim = self.render.render_fim_wim(src_info['cam'], src_info['verts']) # src_f2pts = src_f2verts[:, :, :, 0:2] src_info['fim'] = src_fim src_info['wim'] = src_wim src_info['cond'], _ = self.render.encode_fim(src_info['cam'], src_info['verts'], fim=src_fim, transpose=True) src_info['f2verts'] = src_f2verts src_info['p2verts'] = src_f2verts[:, :, :, 0:2] src_info['p2verts'][:, :, :, 1] *= -1 if self._opt.only_vis: src_info['p2verts'] = self.render.get_vis_f2pts(src_info['p2verts'], src_fim) src_info['part'], _ = self.render.encode_fim(src_info['cam'], src_info['verts'], fim=src_fim, transpose=True, map_fn=self.part_fn) # add image to source info src_info['img'] = img src_info['image'] = ori_img # 2. process the src inputs if self.detector is not None: bbox, body_mask = self.detector.inference(img[0]) bg_mask = 1 - body_mask else: bg_mask = util.morph(src_info['cond'][:, -1:, :, :], ks=self._opt.bg_ks, mode='erode') body_mask = 1 - bg_mask if self._opt.bg_model != 'ORIGINAL': src_info['bg'] = self.bgnet(img, masks=body_mask, only_x=True) else: incomp_img = img * bg_mask bg_inputs = torch.cat([incomp_img, bg_mask], dim=1) img_bg = self.bgnet(bg_inputs) src_info['bg_inputs'] = bg_inputs # src_info['bg'] = img_bg src_info['bg'] = incomp_img + img_bg * body_mask ft_mask = 1 - util.morph(src_info['cond'][:, -1:, :, :], ks=self._opt.ft_ks, mode='erode') src_inputs = torch.cat([img * ft_mask, src_info['cond']], dim=1) src_info['feats'] = self.generator.encode_src(src_inputs) src_info['src_inputs'] = src_inputs src_info = src_info # if visualizer is not None: # self.visualize(visualizer, src=img, bg=src_info['bg']) if output_path: cv_utils.save_cv2_img(src_info['image'], output_path, image_size=self._opt.image_size) return src_info