def build_model(self):
"""
You must define your model in this function, including define the graph and allocate GPU.
This function will be called in @see `MotionImitationRunnerProcessor.run()`.
Returns:
None
"""
# set imitator
self.model = Imitator(self.opt)
def setup(opts):
shutil.move(opts['checkpoint_dir'], 'outputs/checkpoints')
shutil.move(opts['pretrained_renderer_dir'], 'assets/pretrains')
opt = TestOptions().parse()
opt.bg_ks = 13
opt.ft_ks = 3
opt.has_detector = True
opt.front_warp = True
swapper = Swapper(opt=opt)
imitator = Imitator(opt)
return swapper, imitator
def generate_actor_result(test_opt, src_img_path):
imitator = Imitator(test_opt)
src_img_name = os.path.split(src_img_path)[-1][:-4]
test_opt.src_path = src_img_path
if test_opt.post_tune:
adaptive_personalize(test_opt, imitator, visualizer=None)
else:
imitator.personalize(test_opt.src_path, visualizer=None)
action_list_dict = {'dance': MIXAMO_DANCE_ACTION_IDX_LIST,
'base': MIXAMO_BASE_ACTION_IDX_LIST,
'acrobat': MIXAMO_ACROBAT_ACTION_IDX_LIST}
for action_type in ['dance', 'base', 'acrobat']:
for i in action_list_dict[action_type]:
if test_opt.output_dir:
pred_output_dir = os.path.join(test_opt.output_dir, 'mixamo_preds')
if os.path.exists(pred_output_dir):
os.system("rm -r %s" % pred_output_dir)
mkdir(pred_output_dir)
else:
pred_output_dir = None
print(pred_output_dir)
tgt_smpls = load_mixamo_smpl(i)
imitator.inference_by_smpls(tgt_smpls, cam_strategy='smooth', output_dir=pred_output_dir, visualizer=None)
save_dir = os.path.join(test_opt.output_dir, src_img_name, action_type)
mkdir(save_dir)
output_mp4_path = os.path.join(save_dir, 'mixamo_%.4d_%s.mp4' % (i, src_img_name))
img_path_list = sorted(glob.glob('%s/*.jpg' % pred_output_dir))
make_video(output_mp4_path, img_path_list, save_frames_dir=None, fps=30)
def generate_T_pose_novel_view_result(test_opt, src_img_path):
imitator = Imitator(test_opt)
src_img_name = os.path.split(src_img_path)[-1][:-4]
test_opt.src_path = src_img_path
if test_opt.post_tune:
adaptive_personalize(test_opt, imitator, visualizer=None)
else:
imitator.personalize(test_opt.src_path, visualizer=None)
if test_opt.output_dir:
pred_output_dir = os.path.join(test_opt.output_dir, 'T_novel_view_preds')
if os.path.exists(pred_output_dir):
os.system("rm -r %s" % pred_output_dir)
mkdir(pred_output_dir)
else:
pred_output_dir = None
print(pred_output_dir)
tgt_smpls = create_T_pose_novel_view_smpl()
imitator.inference_by_smpls(tgt_smpls, cam_strategy='smooth', output_dir=pred_output_dir, visualizer=None)
save_dir = os.path.join(test_opt.output_dir, src_img_name)
mkdir(save_dir)
output_mp4_path = os.path.join(save_dir, 'T_novel_view_%s.mp4' % src_img_name)
img_path_list = sorted(glob.glob('%s/*.jpg' % pred_output_dir))
make_video(output_mp4_path, img_path_list, save_frames_dir=None, fps=30)
# clean other left
clean(test_opt.output_dir)
verbose=False)
if __name__ == "__main__":
# meta imitator
test_opt = TestOptions().parse()
if test_opt.ip:
visualizer = VisdomVisualizer(env=test_opt.name,
ip=test_opt.ip,
port=test_opt.port)
else:
visualizer = None
# set imitator
imitator = Imitator(test_opt)
if test_opt.post_tune:
adaptive_personalize(test_opt, imitator, visualizer)
imitator.personalize(test_opt.src_path, visualizer=visualizer)
print('\n\t\t\tPersonalization: completed...')
if test_opt.save_res:
pred_output_dir = mkdir(os.path.join(test_opt.output_dir, 'imitators'))
pred_output_dir = clear_dir(pred_output_dir)
else:
pred_output_dir = None
print('\n\t\t\tImitating `{}`'.format(test_opt.tgt_path))
tgt_paths = scan_tgt_paths(test_opt.tgt_path, itv=1)
class LWGEvaluatorModel(MotionImitationModel):
def __init__(self, opt, output_dir):
super().__init__(output_dir)
self.opt = opt
if self.opt.ip:
visualizer = VisdomVisualizer(env=self.opt.name,
ip=self.opt.ip,
port=self.opt.port)
else:
visualizer = None
self.visualizer = visualizer
self.model = None
def imitate(self, src_infos: Dict[str, Any],
ref_infos: Dict[str, Any]) -> List[str]:
"""
Running the motion imitation of the self.model, based on the source information with respect to the
provided reference information. It returns the full paths of synthesized images.
Args:
src_infos (dict): the source information contains:
--images (list of str): the list of full paths of source images (the length is 1)
--smpls (np.ndarray): (length of images, 85)
--kps (np.ndarray): (length of images, 19, 2)
ref_infos (dict): the reference information contains:
--images (list of str): the list of full paths of reference images.
--smpls (np.ndarray): (length of images, 85)
--kps (np.ndarray): (length of images, 19, 2)
--self_imitation (bool): the flag indicates whether it is self-imitation or not.
Returns:
preds_files (list of str): full paths of synthesized images with respects to the images in ref_infos.
"""
tgt_paths = ref_infos["images"]
tgt_smpls = ref_infos["smpls"]
self_imitation = ref_infos["self_imitation"]
if self_imitation:
cam_strategy = "copy"
out_dir = self.si_out_dir
count = self.num_preds_si
self.num_preds_si += len(tgt_paths)
else:
cam_strategy = "smooth"
out_dir = self.ci_out_dir
count = self.num_preds_ci
self.num_preds_ci += len(tgt_paths)
outputs = self.model.inference(tgt_paths,
tgt_smpls=tgt_smpls,
cam_strategy=cam_strategy,
visualizer=None,
verbose=True)
all_preds_files = []
for i, preds in enumerate(outputs):
filename = "{:0>8}.jpg".format(count)
pred_file = os.path.join(out_dir, 'pred_' + filename)
count += 1
cv_utils.save_cv2_img(preds, pred_file, normalize=True)
all_preds_files.append(pred_file)
return all_preds_files
def build_model(self):
"""
You must define your model in this function, including define the graph and allocate GPU.
This function will be called in @see `MotionImitationRunnerProcessor.run()`.
Returns:
None
"""
# set imitator
self.model = Imitator(self.opt)
def personalization(self, src_infos):
"""
some task/method specific data pre-processing or others.
Args:
src_infos (dict): the source information contains:
--images (list of str): the list of full paths of source images (the length is 1)
--smpls (np.ndarray): (length of images, 85)
--kps (np.ndarray): (length of images, 19, 2)
Returns:
processed_src_infos (dict): the source information contains:
--images (list of str): the list of full paths of source images (the length is 1)
--smpls (np.ndarray): (length of images, 85)
--kps (np.ndarray): (length of images, 19, 2)
...
"""
# 1. load the pretrain model
self.model._load_params(self.model.generator, self.opt.load_path)
# 2. post personalization
if self.opt.post_tune:
self.opt.src_path = src_infos["images"][0]
adaptive_personalize(self.opt, self.model, self.visualizer)
processed_src_infos = src_infos
return processed_src_infos
def terminate(self):
"""
Close the model session, like if the model is based on TensorFlow, it needs to call sess.close() to
dealloc the resources.
Returns:
"""
pass
def main():
opt = TrainOptions().parse()
opt.data_root = 'data/people_snapshot_public'
opt.checkpoints_dir = 'outputs/checkpoints/imitator'
opt.log_dir = 'outputs/logs/imitator'
opt.use_loss_gan = True
opt.use_loss_rec = True
opt.use_loss_mask = True
opt.use_loss_verts = False
opt.lambda_gan = 1
opt.lambda_rec = 10
opt.lambda_mask = 0.1
opt.lambda_verts = 10
opt.isHres = False
opt.batch_size = 4
opt.epochs = 20
opt.step_size = 2
opt.lr_gamma = 0.5
train_dataset = People_Snapshot_Dataset(data_root=opt.data_root,
image_size=opt.image_size,
get_uv=True)
train_loader = DataLoader(train_dataset,
batch_size=opt.batch_size,
shuffle=True)
model = Imitator(opt)
for fn in os.listdir(opt.log_dir):
os.remove(os.path.join(opt.log_dir, fn))
if opt.resume:
model.load_checkpoint(opt.start_epoch)
model.train()
total = len(train_loader.dataset)
for epoch in range(opt.start_epoch, opt.epochs):
train_num = 0
for i, data in enumerate(train_loader):
model.set_input(data)
model.optimize_parameters()
N = data['shape'].size(0)
train_num += N
if (i + 1) % opt.print_freq == 0 or train_num == total:
loss_vis = model.get_loss_vis()
mess = "Epoch %d: [%d / %d]" % (epoch, train_num, total)
for name in loss_vis:
mess += " | " + "%s: %.7f" % (name, loss_vis[name])
print(mess)
if (i + 1) % (opt.print_freq * 10) == 0 or train_num == total:
imgs_vis = model.visualize()
img_src_gt = imgs_vis['img_src_gt'].detach().cpu()
img_src_gt = make_grid(img_src_gt,
nrow=img_src_gt.size(0),
padding=0)
img_ref_gt = imgs_vis['img_ref_gt'].detach().cpu()
img_ref_gt = make_grid(img_ref_gt,
nrow=img_ref_gt.size(0),
padding=0)
img_masked_src = imgs_vis['img_masked_src'].detach().cpu()
img_masked_src = make_grid(img_masked_src,
nrow=img_masked_src.size(0),
padding=0)
img_masked_ref = imgs_vis['img_masked_ref'].detach().cpu()
img_masked_ref = make_grid(img_masked_ref,
nrow=img_masked_ref.size(0),
padding=0)
bg = imgs_vis['background'].detach().cpu()
bg = make_grid(bg, nrow=bg.size(0), padding=0)
img_src_rec = imgs_vis['img_src_rec'].detach().cpu()
img_src_rec = make_grid(img_src_rec,
nrow=img_src_rec.size(0),
padding=0)
img_ref_rec = imgs_vis['img_ref_rec'].detach().cpu()
img_ref_rec = make_grid(img_ref_rec,
nrow=img_ref_rec.size(0),
padding=0)
save_image([
img_src_gt, img_ref_gt, img_masked_src, img_masked_ref,
bg, img_src_rec, img_ref_rec
],
os.path.join(
opt.log_dir,
"epoch_{:0>2d}_iter_{:0>5d}.png".format(
epoch, i)),
nrow=1,
padding=0)
model.save_checkpoint(epoch)
model.update_learning_rate()