def swap(opt):
"""
Args:
opt:
Returns:
all_meta_outputs (list of MetaOutput):
"""
print("Step 3: running swapper.")
if opt.ip:
from iPERCore.tools.utils.visualizers.visdom_visualizer import VisdomVisualizer
visualizer = VisdomVisualizer(env=opt.model_id, ip=opt.ip, port=opt.port)
else:
visualizer = None
# set imitator
swapper = ModelsFactory.get_by_name("swapper", opt)
# merge all sources
meta_src_proc = opt.meta_data["meta_src"]
src_info_for_inference = merge_all_source_processed_info(opt, meta_src_proc)
# update number source
opt.num_source = sum(src_info_for_inference["num_source"])
print(f"update the number of sources {src_info_for_inference['num_source']} = {opt.num_source}")
# source setup
swapper.swap_source_setup(
src_path_list=src_info_for_inference["paths"],
src_smpl_list=src_info_for_inference["smpls"],
masks_list=src_info_for_inference["masks"],
bg_img_list=src_info_for_inference["bg"],
offsets_list=src_info_for_inference["offsets"],
links_ids_list=src_info_for_inference["links"],
swap_parts=src_info_for_inference["swap_parts"],
visualizer=visualizer,
swap_masks=None
)
# call swap
all_meta_outputs = []
# check whether it has reference or not
meta_ref_proc = opt.meta_data["meta_ref"]
for j, meta_ref in enumerate(meta_ref_proc):
"""
meta_input:
path: /p300/tpami/neuralAvatar/references/videos/bantangzhuyi_1.mp4
bg_path:
name: bantangzhuyi_1
audio: /p300/tpami/neuralAvatar/references/videos/bantangzhuyi_1.mp3
fps: 30.02
pose_fc: 400.0
cam_fc: 150.0
primitives_dir: ../tests/debug/primitives/bantangzhuyi_1
processed_dir: ../tests/debug/primitives/bantangzhuyi_1/processed
vid_info_path: ../tests/debug/primitives/bantangzhuyi_1/processed/vid_info.pkl
"""
meta_output = MetaSwapImitateOutput(meta_src_proc, meta_ref)
ref_proc_info = ProcessInfo(meta_ref)
ref_proc_info.deserialize()
ref_info = ref_proc_info.convert_to_ref_info()
results_dict = call_imitator_inference(
opt, swapper, meta_output,
ref_paths=ref_info["images"],
ref_smpls=ref_info["smpls"],
visualizer=visualizer
)
# save to video
fuse_src_ref_multi_outputs(
meta_output.out_mp4, src_info_for_inference["src_paths"],
results_dict["ref_imgs_paths"], results_dict["outputs"],
audio_path=meta_output.audio, fps=meta_output.fps,
image_size=opt.image_size, pool_size=opt.num_workers
)
all_meta_outputs.append(meta_output)
for meta_output in all_meta_outputs:
print(meta_output)
print("Step 3: running swapper done.")
return all_meta_outputs
def imitate(opt):
"""
Args:
opt:
Returns:
all_meta_outputs (list of MetaOutput):
"""
print("Step 3: running imitator.")
if opt.ip:
from iPERCore.tools.utils.visualizers.visdom_visualizer import VisdomVisualizer
visualizer = VisdomVisualizer(env=opt.model_id,
ip=opt.ip,
port=opt.port)
else:
visualizer = None
# set imitator
imitator = ModelsFactory.get_by_name("imitator", opt)
meta_src_proc = opt.meta_data["meta_src"]
meta_ref_proc = opt.meta_data["meta_ref"]
all_meta_outputs = []
for i, meta_src in enumerate(meta_src_proc):
"""
meta_input:
path: /p300/tpami/neuralAvatar/sources/fange_1/fange_1_ns=2
bg_path: /p300/tpami/neuralAvatar/sources/fange_1/IMG_7225.JPG
name: fange_1
primitives_dir: ../tests/debug/primitives/fange_1
processed_dir: ../tests/debug/primitives/fange_1/processed
vid_info_path: ../tests/debug/primitives/fange_1/processed/vid_info.pkl
"""
src_proc_info = ProcessInfo(meta_src)
src_proc_info.deserialize()
src_info = src_proc_info.convert_to_src_info(num_source=opt.num_source)
src_info_for_inference = get_src_info_for_inference(opt, src_info)
# source setup
imitator.source_setup(src_path=src_info_for_inference["paths"],
src_smpl=src_info_for_inference["smpls"],
masks=src_info_for_inference["masks"],
bg_img=src_info_for_inference["bg"],
offsets=src_info_for_inference["offsets"],
links_ids=src_info_for_inference["links"],
visualizer=visualizer)
for j, meta_ref in enumerate(meta_ref_proc):
"""
meta_input:
path: /p300/tpami/neuralAvatar/references/videos/bantangzhuyi_1.mp4
bg_path:
name: bantangzhuyi_1
audio: /p300/tpami/neuralAvatar/references/videos/bantangzhuyi_1.mp3
fps: 30.02
pose_fc: 400.0
cam_fc: 150.0
primitives_dir: ../tests/debug/primitives/bantangzhuyi_1
processed_dir: ../tests/debug/primitives/bantangzhuyi_1/processed
vid_info_path: ../tests/debug/primitives/bantangzhuyi_1/processed/vid_info.pkl
"""
meta_output = MetaImitateOutput(meta_src, meta_ref)
ref_proc_info = ProcessInfo(meta_ref)
ref_proc_info.deserialize()
ref_info = ref_proc_info.convert_to_ref_info()
results_dict = call_imitator_inference(
opt,
imitator,
meta_output,
ref_paths=ref_info["images"],
ref_smpls=ref_info["smpls"],
visualizer=visualizer)
# save to video
fuse_src_ref_multi_outputs(meta_output.out_mp4,
src_info_for_inference["paths"],
results_dict["ref_imgs_paths"],
results_dict["outputs"],
audio_path=meta_output.audio,
fps=meta_output.fps,
image_size=opt.image_size,
pool_size=opt.num_workers)
all_meta_outputs.append(meta_output)
for meta_output in all_meta_outputs:
print(meta_output)
print("Step 3: running imitator done.")
return all_meta_outputs
def imitate(opt):
"""
Args:
opt:
Returns:
all_meta_outputs (list of MetaOutput):
"""
print("Step 3: running imitator.")
if opt.ip:
from iPERCore.tools.utils.visualizers.visdom_visualizer import VisdomVisualizer
visualizer = VisdomVisualizer(env=opt.model_id,
ip=opt.ip,
port=opt.port)
else:
visualizer = None
# set imitator
imitator = ModelsFactory.get_by_name("imitator", opt)
meta_src_proc = opt.meta_data["meta_src"]
meta_ref_proc = opt.meta_data["meta_ref"]
all_meta_outputs = []
for i, meta_src in enumerate(meta_src_proc):
"""
meta_input:
path: /p300/tpami/neuralAvatar/sources/fange_1/fange_1_ns=2
bg_path: /p300/tpami/neuralAvatar/sources/fange_1/IMG_7225.JPG
name: fange_1
primitives_dir: ../tests/debug/primitives/fange_1
processed_dir: ../tests/debug/primitives/fange_1/processed
vid_info_path: ../tests/debug/primitives/fange_1/processed/vid_info.pkl
"""
src_proc_info = ProcessInfo(meta_src)
src_proc_info.deserialize()
src_info = src_proc_info.convert_to_src_info(num_source=opt.num_source)
src_info_for_inference = get_src_info_for_inference(opt, src_info)
# 1. personalization
imitator.source_setup(src_path=src_info_for_inference["paths"],
src_smpl=src_info_for_inference["smpls"],
masks=src_info_for_inference["masks"],
bg_img=src_info_for_inference["bg"],
offsets=src_info_for_inference["offsets"],
links_ids=src_info_for_inference["links"],
visualizer=visualizer)
for j, meta_ref in enumerate(meta_ref_proc):
"""
meta_input:
path: /p300/tpami/neuralAvatar/references/videos/bantangzhuyi_1.mp4
bg_path:
name: bantangzhuyi_1
audio: /p300/tpami/neuralAvatar/references/videos/bantangzhuyi_1.mp3
fps: 30.02
pose_fc: 400.0
cam_fc: 150.0
primitives_dir: ../tests/debug/primitives/bantangzhuyi_1
processed_dir: ../tests/debug/primitives/bantangzhuyi_1/processed
vid_info_path: ../tests/debug/primitives/bantangzhuyi_1/processed/vid_info.pkl
"""
ref_proc_info = ProcessInfo(meta_ref)
ref_proc_info.deserialize()
ref_info = ref_proc_info.convert_to_ref_info()
ref_imgs_paths = ref_info["images"]
ref_smpls = ref_info["smpls"]
ref_smpls = add_hands_params_to_smpl(
ref_smpls, imitator.body_rec.np_hands_mean)
meta_output = MetaOutput(meta_src, meta_ref)
# if there are more than 10 frames, then we will use temporal smooth of smpl.
if len(ref_smpls) > 10:
ref_smpls = temporal_smooth_smpls(ref_smpls,
pose_fc=meta_output.pose_fc,
cam_fc=meta_output.cam_fc)
out_imgs_dir = clear_dir(meta_output.imitation_dir)
outputs = imitator.inference(tgt_paths=ref_imgs_paths,
tgt_smpls=ref_smpls,
cam_strategy=opt.cam_strategy,
output_dir=out_imgs_dir,
visualizer=visualizer,
verbose=True)
fuse_source_reference_output(
meta_output.imitation_mp4,
src_info_for_inference["paths"],
ref_imgs_paths,
outputs,
# sorted(glob.glob(os.path.join(meta_output.imitation_dir, "pred_*"))),
audio_path=meta_output.audio,
fps=meta_output.fps,
image_size=opt.image_size,
pool_size=opt.num_workers)
all_meta_outputs.append(meta_output)
for meta_output in all_meta_outputs:
print(meta_output)
print("Step 3: running imitator done.")
return all_meta_outputs