def test_adapter_match():
raw = read_livecap_model(*config.read_livecap_model_args)
model = LivecapModel.from_raw_model_data(raw)
dataset = LiveCapAdapter(config.original_dataset_path, model,
config.original_camera_path,
get_vibe_to_original_joints())
cam_params = np.load(config.intrinsic_params_path)
fx = cam_params['fx']
fy = cam_params['fy']
u = cam_params['u']
v = cam_params['v']
h = cam_params['h']
w = cam_params['w']
renderer = DebugRenderer(w, h, model, dataset.joint_indices)
# project points with the original with the dataset parameters
camera = Camera(config.camera_to_world_matrix, h, w, fx, fy, u, v)
datapoint = dataset[0]
kp3d = datapoint.kp_3d
kp3d_translation = datapoint.kp_3d_translation
p3d = model.get_p3d()[dataset.joint_indices] + kp3d_translation
renderer.debug_3d(p3d, kp3d, kp3d_translation)
kp3d = kp3d + kp3d_translation
kp2d = camera.project(kp3d)
kp2d_true = datapoint.kp_2d
p2d = camera.project(p3d)
frame = datapoint.frame
draw_pixels_on_image(frame, p2d, 'red')
draw_pixels_on_image(frame, kp2d, 'blue')
draw_pixels_on_image(frame, kp2d_true, 'green')
view_image_blocking(frame)
def video_optimization(dataset: LiveCapAdapter, model: Model, camera: Camera,
renderer: Renderer,
energy_weights: Dict) -> List[LivecapPose]:
"""Optimization for the entire video
:returns a list with an optimization result for each frame.
"""
# initialization
initial_pose = model.get_initial_pose()
initial_pose.root_translation = dataset.get_initial_translation()
results = [initial_pose]
scalar = False
debug = True
print(
'\n\n\n\nestimating pose in video with the following energy weights:\n'
+ str(energy_weights))
energy = Energy(model,
camera,
dataset.joint_indices,
renderer,
**energy_weights,
return_scalar=scalar,
debug=debug)
# iterate through all of the frames
for i in range(len(dataset)):
print(f'\nestimating pose in frame {i}...')
entry = dataset[i]
result = frame_optimization(entry, results[-1], energy, scalar)
results.append(result)
# finalize and return the results
return results
def load_livecap_dataset(model: LivecapModel):
return LiveCapAdapter(
root=config.original_dataset_path,
model=model,
camera_fitting_path=config.original_camera_path,
kp_to_joint_map=get_vibe_to_original_joints(),
)
def init_optimization_from_config():
model = Model.from_file(config.model_path)
dataset = LiveCapAdapter(config.livecap_dataset_path, model,
config.fx_fy_t_path)
camera = Camera(config.camera_to_world_matrix, dataset.image_h,
dataset.image_w, dataset.fx, dataset.fy)
renderer = Renderer(RenderMode.image,
model,
camera=camera,
joint_indices=dataset.joint_indices,
**config.scale)
return model, dataset, camera, renderer
def estimate_pose(model: Model,
dataset_path: Path,
save_path: Path = None) -> Animation:
'''
1. Runs optimization etc...
2. Returns animation
'''
logging.info('estimating pose from dateset: ' + str(dataset_path))
dataset = LiveCapAdapter(dataset_path, model.root_joint)
frame_height, frame_width, _ = dataset[0].frame.shape
bind_pose_as_optimization_array = model.root_joint.bind_pose_to_optimization_array(
)
prev_opt = bind_pose_as_optimization_array
key_frames = []
with ModelRenderer(model,
window_width=frame_width,
window_height=frame_height) as renderer:
timestamp = 0.0
for i, frame_data in enumerate(dataset):
logging.info(f'estimating pose in frame {i}')
timestamp = i * (1 / 30)
if i == 150:
break
if i % 2 == 1:
continue
opt = _estimate_pose_in_frame(model.root_joint, frame_data,
renderer, prev_opt)
kf = KeyFrame(model.root_joint.optimization_array_to_pose(opt),
timestamp)
key_frames.append(kf)
prev_opt = opt
animation = Animation(key_frames)
if save_path is None:
logging.info('no save path given. not saving animation!')
else:
save_animation(animation, save_path)
return animation
"""This is a visual test for the starting parameters of the optimization."""
import numpy as np
import config
from lib.utils.model import Model
from lib.data_utils.livecap_dataset_adapter import LiveCapAdapter
from lib.utils.camera import Camera
from lib.utils.renderer import Renderer, RenderMode, view_image_blocking, Color, draw_pixels_on_image
# basic rendering test:
# loading the modules
model = Model.from_file(config.model_path)
dataset = LiveCapAdapter(config.livecap_dataset_path, model,
config.fx_fy_t_path)
camera = Camera(config.camera_to_world_matrix, dataset.image_h,
dataset.image_w, dataset.fx, dataset.fy)
renderer = Renderer(RenderMode.image,
model,
camera=camera,
joint_indices=dataset.joint_indices,
**config.scale)
blocking_renderer = Renderer(RenderMode.blocking,
model,
camera=camera,
joint_indices=dataset.joint_indices,
**config.scale)
# initialization
initial_pose = model.get_initial_pose()
initial_pose.root_translation = dataset.get_initial_translation()
model.apply_livecap_pose(initial_pose)
"""visually test the adapter."""
import config
from lib.image_processing.image_distance_transform import image_distance_transform
from lib.utils.renderer import view_image_blocking, draw_pixels_on_image, Renderer, RenderMode
from lib.data_utils.livecap_dataset_adapter import LiveCapAdapter
from lib.utils.model import Model
import numpy as np
model = Model.from_file(config.model_path)
dataset = LiveCapAdapter(config.livecap_dataset_path, model, config.fx_fy_t_path)
sample_indices = [10, 300, 367, 989]
renderer = Renderer(RenderMode.blocking, model, dataset.image_h, dataset.image_w, show_axes=True,
joint_indices=dataset.joint_indices, **config.scale)
def test_idt():
for i in sample_indices:
entry = dataset[i]
silhouette = entry.silhouette
view_image_blocking(silhouette, 'silhouette')
idt = image_distance_transform(silhouette)
print('idt stats: ', idt.shape, idt.dtype, idt.max())
view_image_blocking(idt, 'idt')
def test_keypoints():
print(f'the kp indices {dataset.kp_indices.shape}, {dataset.kp_indices}')
print(f'the joints indices {dataset.joint_indices.shape}, {dataset.joint_indices}')
for i in sample_indices:
entry = dataset[i]
import time
import config
from lib.utils.renderer import Renderer, view_image_blocking
from lib.utils.model import Model
from lib.data_utils.livecap_dataset_adapter import LiveCapAdapter
from lib.utils.camera import Camera
print('loading model...')
MODEL = Model.from_file(config.model_path)
print('loading dataset...')
DATASET = LiveCapAdapter(config.livecap_dataset_path, MODEL,
config.fx_fy_t_path)
print('loading camera...')
CAMERA = Camera(config.camera_to_world_matrix,
config.image_height,
config.image_width,
fx=DATASET.fx,
fy=DATASET.fy)
print('initializing model pose...')
POSE = MODEL.get_initial_pose()
POSE.root_translation = DATASET.get_initial_translation()
MODEL.apply_livecap_pose(POSE)
NUM_ITERATIONS = 2000
PRINT_EVERY = 100
BLOCKING_CFG = dict(
mode='blocking',
model=MODEL,