def morph_file(self, img_path, gt_path, expresion): img = cv_utils.read_cv2_img(img_path) gt = cv_utils.read_cv2_img(gt_path) # morphed_img = self._img_morph(img, expresion) morphed_img = self.morph_face(img, gt, expresion) output_name = f'{os.path.basename(img_path)}_{np.random.randint(0, 100)}_out.png' self.save_img(morphed_img, output_name)
def morph_face_from_file_video(self, img_path, n_each_pose=60, fps=30): img = cv_utils.read_cv2_img(img_path) _w_s, h_s, _c = img.shape crop, roi = self.crop(img, idx=0) morphed_imgs = list() t0 = util.time_now() cnt = 0 poses_dict = self.rnd_poses(n_each_pose, fps, self.mouth_pose_names, self.mouth_pose_values) for name in poses_dict: cnt += len(poses_dict[name]) for pose in poses_dict[name]: morphed = self.morph(crop, pose)['result'] img_cp = img.copy() self.blend(img_cp, morphed, roi) cv2.putText(img_cp, f"{name}={pose[np.argmax(pose != 0)]:.03f}", (10, h_s - 20), cv2.FONT_HERSHEY_DUPLEX, 1, (255, 0, 0)) morphed_imgs.append(img_cp) t_avg = util.time_diff_ms(t0) / cnt print(f"infer: {t_avg:.03f}ms") output_name = f'{os.path.basename(img_path)[:-4]}.mp4' # output_name = f'{os.path.basename(img_path)}' \ # f'_{name}_n{len(poses_dict[name])}_{"left" if left else "right"}.mp4' self.save_imgs_as_video(morphed_imgs, output_name, fps)
def get_features(self, img_path, expression): img = cv_utils.read_cv2_img(img_path) if img is None: print('Failing to read sample: ', img_path) return None features = self.img_forward(img, expression) return features
def _extract_smpls(self, input_file): img = cv_utils.read_cv2_img(input_file) img = cv_utils.transform_img(img, image_size=224) * 2 - 1.0 # hmr receive [-1, 1] img = img.transpose((2, 0, 1)) img = torch.FloatTensor(img).cuda()[None, ...] theta = self.hmr(img)[-1] return theta
def morph_file(self, img_path, expresion, i, j=0): img = cv_utils.read_cv2_img(img_path) morphed_img, bbs = self._img_morph(img, expresion) overlay_fake_img = self._overlay_fake_img(img, morphed_img['fake_imgs_masked'], bbs) output_name = '%s_%s_out.png' % (os.path.basename(img_path), i) fake_output_name = '%s_out%s%s.png' % (os.path.basename(img_path), i, j) self._save_img(overlay_fake_img, fake_output_name) return fake_output_name
def transfer(self, tgt_path, tgt_smpl=None, cam_strategy='smooth', t=0, visualizer=None): with torch.no_grad(): # 1. get source info src_info = self.src_info ori_img = cv_utils.read_cv2_img(tgt_path) if tgt_smpl is None: img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0 img_hmr = torch.FloatTensor(img_hmr).cuda()[None, ...] tgt_smpl = self.hmr(img_hmr)[-1] else: tgt_smpl = to_tensor(tgt_smpl).cuda()[None, ...] if t == 0 and cam_strategy == 'smooth': self.first_cam = tgt_smpl[:, 0:3].clone() # 2. compute tsf smpl tsf_smpl = self.swap_smpl(src_info['cam'], src_info['shape'], tgt_smpl, cam_strategy=cam_strategy) tsf_info = self.hmr.get_details(tsf_smpl) # add pose condition and face index map into source info tsf_info['cond'], tsf_info['fim'] = self.render.encode_fim(tsf_info['cam'], tsf_info['verts'], transpose=True) # add part condition into source info tsf_info['part'] = self.render.encode_front_fim(tsf_info['fim'], transpose=True) # 3. calculate syn front image and transformation flows ref_info = self.ref_info selected_part_id = self.PART_IDS['body'] left_id = [i for i in self.PART_IDS['all'] if i not in selected_part_id] src_part_mask = (torch.sum(tsf_info['part'][:, left_id, ...], dim=1) != 0).byte() ref_part_mask = (torch.sum(tsf_info['part'][:, selected_part_id, ...], dim=1) != 0).byte() T_s = self.calculate_trans(src_info['bc_f2pts'], src_info['fim'], tsf_info['fim'], src_part_mask) T_r = self.calculate_trans(ref_info['bc_f2pts'], ref_info['fim'], tsf_info['fim'], ref_part_mask) tsf_s = self.model.transform(src_info['image'], T_s) tsf_r = self.model.transform(ref_info['image'], T_r) tsf_img = tsf_s * src_part_mask.float() + tsf_r * ref_part_mask.float() tsf_inputs = torch.cat([tsf_img, tsf_info['cond']], dim=1) preds = self.forward2(tsf_inputs, src_info['feats'], T_s, ref_info['feats'], T_r, src_info['bg']) if visualizer is not None: visualizer.vis_named_img('src', src_info['image']) visualizer.vis_named_img('ref', ref_info['image']) visualizer.vis_named_img('src_cond', src_info['cond']) visualizer.vis_named_img('ref_cond', ref_info['cond']) visualizer.vis_named_img('tsf_cond', tsf_info['cond']) visualizer.vis_named_img('tsf_s', tsf_s) visualizer.vis_named_img('tsf_r', tsf_r) visualizer.vis_named_img('tsf_img', tsf_img) visualizer.vis_named_img('preds', preds) visualizer.vis_named_img('src_part_mask', src_part_mask) visualizer.vis_named_img('ref_part_mask', ref_part_mask) return preds
def morph_file(self, img_path, expresion, desired_expresion): img = cv_utils.read_cv2_img(img_path) morphed_img, concate_face = self._img_morph(img, expresion, desired_expresion) #concate_img = self._img_concate(img, expresion,desired_expresion) output_name = '%s_out.png' % os.path.basename(img_path) self._save_img(morphed_img, output_name) self._save_img(concate_face, "concate_" + output_name)
def _get_img_by_id(self, idx): if idx < self._dataset_size: img_path = os.path.join(self._imgs_dir, self._info[idx]['file_path']) img = cv_utils.read_cv2_img(img_path) return img, self._info[idx]['file_path'] else: print('You input idx: ', idx) return None, None
def load_init_preds(self, pred_path): pred_img_name = os.path.split(pred_path)[-1] pred_img_path = os.path.join(self._opt.preds_img_folder, 'pred_' + pred_img_name) img = cv_utils.read_cv2_img(pred_img_path) img = cv_utils.transform_img(img, self._opt.image_size, transpose=True) img = img * 2 - 1 return img
def transfer_params(self, tgt_path, tgt_smpl=None, cam_strategy='smooth', t=0): # get source info src_info = self.src_info ori_img = cv_utils.read_cv2_img(tgt_path) if tgt_smpl is None: img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0 img_hmr = torch.tensor(img_hmr, dtype=torch.float32).cuda()[None, ...] tgt_smpl = self.hmr(img_hmr) else: tgt_smpl = torch.tensor(tgt_smpl, dtype=torch.float32).cuda()[None, ...] if t == 0 and cam_strategy == 'smooth': self.first_cam = tgt_smpl[:, 0:3].clone() # get transfer smpl tsf_smpl = self.swap_smpl(src_info['cam'], src_info['shape'], tgt_smpl, cam_strategy=cam_strategy) # transfer process, {'theta', 'cam', 'pose', 'shape', 'verts', 'j2d', 'j3d'} tsf_info = self.hmr.get_details(tsf_smpl) tsf_f2verts, tsf_fim, tsf_wim = self.render.render_fim_wim( tsf_info['cam'], tsf_info['verts']) # src_f2pts = src_f2verts[:, :, :, 0:2] tsf_info['fim'] = tsf_fim tsf_info['wim'] = tsf_wim tsf_info['cond'], _ = self.render.encode_fim(tsf_info['cam'], tsf_info['verts'], fim=tsf_fim, transpose=True) # tsf_info['sil'] = util.morph((tsf_fim != -1).float(), ks=self._opt.ft_ks, mode='dilate') T = self.render.cal_bc_transform(src_info['p2verts'], tsf_fim, tsf_wim) tsf_img = F.grid_sample(src_info['img'], T) tsf_inputs = torch.cat([tsf_img, tsf_info['cond']], dim=1) # add target image to tsf info tsf_info['tsf_img'] = tsf_img tsf_info['image'] = ori_img tsf_info['T'] = T self.T = T self.tsf_info = tsf_info return tsf_inputs
def load_images(self, im_pairs): imgs = [] for im_path in im_pairs: img = cv_utils.read_cv2_img(im_path) img = cv_utils.transform_img(img, self._opt.image_size, transpose=True) img = img * 2 - 1 imgs.append(img) imgs = np.stack(imgs) return imgs
def morph_face_from_file(self, img_path, pose): img = cv_utils.read_cv2_img(img_path) crop, roi = self.crop(img) imgs = self.morph(crop, pose) morphed = imgs['result'] self.blend(img, morphed, roi) output_name = f'{os.path.basename(img_path)}_{np.random.randint(0, 100)}_out.png' self.save_img(img, output_name)
def morph_face_from_file(self, img_path, pose): img = cv_utils.read_cv2_img(img_path) crop_left, roi_l, crop_right, roi_r = self.crop(img) imgs = self.morph(crop_left, pose) morphed = imgs['result'] self.blend(img, morphed, roi_l) # cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 0)) output_name = f'{os.path.basename(img_path)}_{np.random.randint(0, 100)}_out.png' self.save_img(img, output_name)
def __getitem__(self, index): assert (index < self._dataset_size) im_path = (self.all_file[index])[0] label = (self.all_file[index])[1] im = cv_utils.read_cv2_img(im_path) img = self._transform(Image.fromarray(im)) # pack sample = {'img': img, 'label': label} return sample
def morph_file(self, img_path, expression): img = cv_utils.read_cv2_img(img_path) #img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) # Open this one for cropped_28_01 #newX,newY = img.shape[1]*2, img.shape[0]*2 #img = cv2.resize(img,(newX,newY)) morphed_img = self._img_morph(img, expression) #morphed_img = cv2.cvtColor(morphed_img, cv2.COLOR_RGB2BGR) output_name = os.path.join( self._opt.output_dir, '{0}_epoch_{1}_intensity_{2}_out.png'.format( os.path.basename(img_path)[:-4], str(self._opt.load_epoch), str(expression[self._opt.au_index] * 10))) self._save_img(morphed_img, output_name) print('Morphed image is saved at path {}'.format(output_name))
def transfer_params(self, tgt_path, tgt_smpl=None, cam_strategy='smooth', t=0): ori_img = cv_utils.read_cv2_img(tgt_path) if tgt_smpl is None: img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0 img_hmr = torch.tensor(img_hmr, dtype=torch.float32).cuda()[None, ...] tgt_smpl = self.hmr(img_hmr) else: if isinstance(tgt_smpl, np.ndarray): tgt_smpl = torch.tensor(tgt_smpl, dtype=torch.float32).cuda()[None, ...] tsf_inputs = self.transfer_params_by_smpl(tgt_smpl=tgt_smpl, cam_strategy=cam_strategy, t=t) self.tsf_info['image'] = ori_img return tsf_inputs
def personalize(self, src_path, src_smpl=None): with torch.no_grad(): ori_img = cv_utils.read_cv2_img(src_path) # resize image and convert the color space from [0, 255] to [-1, 1] img = cv_utils.transform_img(ori_img, self._opt.image_size, transpose=True) * 2 - 1.0 img = torch.FloatTensor(img).cuda()[None, ...] if src_smpl is None: img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0 img_hmr = torch.FloatTensor(img_hmr).cuda()[None, ...] src_smpl = self.hmr(img_hmr)[-1] else: src_smpl = to_tensor(src_smpl).cuda()[None, ...] # source process, {'theta', 'cam', 'pose', 'shape', 'verts', 'j2d', 'j3d'} src_info = self.hmr.get_details(src_smpl) # add source bary-center points src_info['bc_f2pts'] = self.get_src_bc_f2pts(src_info['cam'], src_info['verts']) # add image to source info src_info['image'] = img # add texture into source info _, src_info['tex'] = self.render.forward(src_info['cam'], src_info['verts'], img, is_uv_sampler=False, reverse_yz=True, get_fim=False) # add pose condition and face index map into source info src_info['cond'], src_info['fim'] = self.render.encode_fim(src_info['cam'], src_info['verts'], transpose=True) # add part condition into source info src_info['part'] = self.render.encode_front_fim(src_info['fim'], transpose=True) # bg input and inpaiting background src_bg_mask = self.morph(src_info['cond'][:, -1:, :, :], ks=15, mode='erode') bg_inputs = torch.cat([img * src_bg_mask, src_bg_mask], dim=1) src_info['bg'] = self.model.bg_model(bg_inputs) # # source identity src_crop_mask = self.morph(src_info['cond'][:, -1:, :, :], ks=3, mode='erode') src_inputs = torch.cat([img * (1 - src_crop_mask), src_info['cond']], dim=1) src_info['feats'] = self.model.src_model.inference(src_inputs) # # self.src_info = src_info return src_info
def _load_pairs(self, vid_info): length = vid_info['length'] pair_ids = np.random.choice(length, size=2, replace=False) smpls = np.concatenate((vid_info['cams'][pair_ids], vid_info['thetas'][pair_ids], vid_info['betas'][pair_ids]), axis=1) images = [] images_paths = vid_info['images'] for t in pair_ids: image_path = images_paths[t] image = cv_utils.read_cv2_img(image_path) images.append(image) return images, smpls
def __getitem__(self, index): ###################### TEST ######################## dataname = self.dataset_name[index] # blurred images blur_paths = self.blur_paths.get(dataname) blurs = [] num_data = len(blur_paths) for i in range(num_data): blur_path = os.path.join(self.root, dataname, 'blurred', blur_paths[i]) blur = cv_utils.read_cv2_img(blur_path, input_nc=1) blurs.append(blur) blurs = np.concatenate(blurs, axis=0) # event images events = self.eventdata.get(dataname) sample = {'events': events, 'blurred': blurs, 'dataname': dataname} return sample
def _load_pairs(self, vid_info): length = vid_info['length'] start = np.random.randint(0, 15) end = np.random.randint(0, length) pair_ids = np.array([start, end], dtype=np.int32) smpls = np.concatenate((vid_info['cams'][pair_ids], vid_info['thetas'][pair_ids], vid_info['betas'][pair_ids]), axis=1) images = [] images_paths = vid_info['images'] for t in pair_ids: image_path = images_paths[t] image = cv_utils.read_cv2_img(image_path) images.append(image) return images, smpls
def morph_file(self, img_path, expresion): img = cv_utils.read_cv2_img(img_path) morphed_img = self._img_morph(img, expresion) output_name = '%s_out.png' % os.path.basename(img_path) self._save_img(morphed_img, output_name)
def _get_img_by_id(self, id): filepath = os.path.join(self._root, self._imgs_dir, id+'.jpg') return cv_utils.read_cv2_img(filepath), filepath
def personalize(self, src_path, src_smpl=None, output_path='', visualizer=None): ori_img = cv_utils.read_cv2_img(src_path) # resize image and convert the color space from [0, 255] to [-1, 1] img = cv_utils.transform_img(ori_img, self._opt.image_size, transpose=True) * 2 - 1.0 img = torch.tensor(img, dtype=torch.float32).cuda()[None, ...] if src_smpl is None: img_hmr = cv_utils.transform_img(ori_img, 224, transpose=True) * 2 - 1.0 img_hmr = torch.tensor(img_hmr, dtype=torch.float32).cuda()[None, ...] src_smpl = self.hmr(img_hmr) else: src_smpl = torch.tensor(src_smpl, dtype=torch.float32).cuda()[None, ...] # source process, {'theta', 'cam', 'pose', 'shape', 'verts', 'j2d', 'j3d'} src_info = self.hmr.get_details(src_smpl) src_f2verts, src_fim, src_wim = self.render.render_fim_wim(src_info['cam'], src_info['verts']) # src_f2pts = src_f2verts[:, :, :, 0:2] src_info['fim'] = src_fim src_info['wim'] = src_wim src_info['cond'], _ = self.render.encode_fim(src_info['cam'], src_info['verts'], fim=src_fim, transpose=True) src_info['f2verts'] = src_f2verts src_info['p2verts'] = src_f2verts[:, :, :, 0:2] src_info['p2verts'][:, :, :, 1] *= -1 if self._opt.only_vis: src_info['p2verts'] = self.render.get_vis_f2pts(src_info['p2verts'], src_fim) src_info['part'], _ = self.render.encode_fim(src_info['cam'], src_info['verts'], fim=src_fim, transpose=True, map_fn=self.part_fn) # add image to source info src_info['img'] = img src_info['image'] = ori_img # 2. process the src inputs if self.detector is not None: bbox, body_mask = self.detector.inference(img[0]) bg_mask = 1 - body_mask else: bg_mask = util.morph(src_info['cond'][:, -1:, :, :], ks=self._opt.bg_ks, mode='erode') body_mask = 1 - bg_mask if self._opt.bg_model != 'ORIGINAL': src_info['bg'] = self.bgnet(img, masks=body_mask, only_x=True) else: incomp_img = img * bg_mask bg_inputs = torch.cat([incomp_img, bg_mask], dim=1) img_bg = self.bgnet(bg_inputs) src_info['bg_inputs'] = bg_inputs # src_info['bg'] = img_bg src_info['bg'] = incomp_img + img_bg * body_mask ft_mask = 1 - util.morph(src_info['cond'][:, -1:, :, :], ks=self._opt.ft_ks, mode='erode') src_inputs = torch.cat([img * ft_mask, src_info['cond']], dim=1) src_info['feats'] = self.generator.encode_src(src_inputs) src_info['src_inputs'] = src_inputs src_info = src_info # if visualizer is not None: # self.visualize(visualizer, src=img, bg=src_info['bg']) if output_path: cv_utils.save_cv2_img(src_info['image'], output_path, image_size=self._opt.image_size) return src_info
def run(self, img_path, expresion): img = cv_utils.read_cv2_img(img_path) morphed_img = self.modify_face(img, expresion) output_name = '%s_out.png' % os.path.basename(img_path) self._save_img(morphed_img, output_name)
def _get_img_by_id(self, id): filepath = os.path.join(self._imgs_dir, '%s.%s' % (id, self.img_format)) return cv_utils.read_cv2_img(filepath), filepath
def morph_file(self, img_path, desired_cond): img = cv_utils.read_cv2_img(img_path) morphed_img = self._morph_face(img, desired_cond) #!!!!!! output_name = '%s_out.png' % os.path.basename(img_path) self._save_img(morphed_img, output_name)# !!!!!!!
def _get_img_by_id(self, id): filepath = id + '.jpg' return cv_utils.read_cv2_img(filepath), filepath
def _get_img_by_id(self, id): filepath = os.path.join(self._imgs_dir, id+'.jpg') return cv_utils.read_cv2_img(filepath), filepath
def _get_img_by_id(self, id): filepath = self._imgs_dir + id + '.jpg' # print('filepath : ',filepath) # print(filepath) return cv_utils.read_cv2_img(filepath), filepath