def __init__(self, camera_transforms: np.array, height: int, widht: int,
focal: float, transform) -> None:
"""
Parameters
----------
camera_transforms : [np.array]
List of camera transformations for inference.
height : int
Height of image.
widht : int
Width of image.
focal : float
Focal length of camera.
transform :
List of callable transforms for preprocessing.
"""
super().__init__()
self.transform = transform
self.rays = [
] # list of arrays with ray translation, ray direction and rgb
print('Start initializing all rays of all images')
for transform in camera_transforms:
rays_translation, rays_direction = get_rays(
height, widht, focal, transform)
trans_dir_stack = np.stack([rays_translation, rays_direction], -2)
trans_dir_list = trans_dir_stack.reshape((-1, 2, 3))
self.rays.append(trans_dir_list)
self.rays = np.concatenate(self.rays)
print('Finish initializing rays')
def __init__(self, base_directory: str, transforms_file: str,
transform) -> None:
"""
Parameters
----------
base_directory : str
Path to dataset.
transforms_file : str
File path to file containing transformation mappings.
transform :
List of callable transforms for preprocessing.
"""
super().__init__()
self.transform = transform
self.rays = [] # list of arrays with ray translation, ray direction and rgb
print('Start initializing all rays of all images')
with open(transforms_file, 'r') as transforms_file:
transforms_dict = json.load(transforms_file)
camera_angle_x = transforms_dict['camera_angle_x']
for frame in transforms_dict['frames']:
fname = os.path.join(base_directory, frame['file_path'] + '.png')
image = cv2.imread(fname)
camera_transform = np.array(frame['transform_matrix'])
self.h, self.w = image.shape[:2]
self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
rays_translation, rays_direction = get_rays(self.h, self.w, self.focal,
camera_transform)
trans_dir_rgb_stack = np.stack([rays_translation, rays_direction, image], -2)
trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
self.rays.append(trans_dir_rgb_list)
if self.rays:
self.rays = np.concatenate(self.rays)
print('Finish initializing rays')
def __init__(self, image_directory: str, transforms_file: str,
transform) -> None:
"""
Parameters
----------
image_directory : str
Path to images.
transforms_file : str
File path to file containing transformation mappings.
transform :
List of callable transforms for preprocessing.
"""
super().__init__()
self.transform = transform
self.rays = [
] # list of arrays with ray translation, ray direction and rgb
self.human_poses = [] # list of corresponding human poses
self.dependencies_index = []
self.dependencies_hw = []
print('Start initializing all rays of all images')
with open(transforms_file, 'r') as transforms_file:
transforms_dict = json.load(transforms_file)
camera_angle_x = transforms_dict['camera_angle_x']
image_transform_map = transforms_dict.get('image_transform_map')
image_pose_map = transforms_dict.get('image_pose_map')
image_paths = sorted(glob.glob(os.path.join(image_directory, '*.png')))
if not len(image_paths) == len(image_transform_map):
raise ValueError(
'Number of images in image_directory is not the same as number of transforms'
)
for image_path in image_paths:
camera_transform = image_transform_map[os.path.basename(
image_path)]
human_pose = image_pose_map[os.path.basename(image_path)]
self.human_poses.append(human_pose)
image = cv2.imread(image_path)
self.h, self.w = image.shape[:2]
# should we append a list of the different h, w of all images? right now referencing only the last h, w
self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
rays_translation, rays_direction = get_rays(
self.h, self.w, self.focal, camera_transform)
for i in range(self.w):
for j in range(self.h):
index_list = get_dependent_rays_indices(
rays_translation[i][j], rays_direction[i][j],
canonical, goal, camera_transform, self.h, self.w,
self.focal)
self.dependencies_index.append(index_list)
self.dependencies_hw.append((i, j))
trans_dir_rgb_stack = np.stack(
[rays_translation, rays_direction, image], -2)
trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
self.rays.append(trans_dir_rgb_list)
self.rays = np.concatenate(self.rays)
print('Finish initializing rays')
def generate_video_frames(H, W, focal, model, N_samples):
frames = []
for th in tqdm(np.linspace(0., 360., 120, endpoint=False)):
c2w = pose_spherical(th, -30., 4.)
rays_o, rays_d = utils.get_rays(H, W, focal, c2w[:3,:4])
rgb, depth, acc = utils.render_rays(model, rays_o, rays_d, near=2., far=6., N_samples=N_samples)
frames.append((255*np.clip(rgb,0,1)).astype(np.uint8))
return frames
def __init__(self, image_directory: str, transforms_file: str,
transform) -> None:
"""
Parameters
----------
image_directory : str
Path to images.
transforms_file : str
File path to file containing transformation mappings.
transform :
List of callable transforms for preprocessing.
"""
super().__init__()
self.transform = transform
self.rays = [
] # list of arrays with ray translation, ray direction and rgb
self.human_poses = [] # list of corresponding human poses
print('Start initializing all rays of all images')
with open(transforms_file, 'r') as transforms_file:
transforms_dict = json.load(transforms_file)
camera_angle_x = transforms_dict['camera_angle_x']
self.image_transform_map = transforms_dict.get('image_transform_map')
image_pose_map = transforms_dict.get('image_pose_map')
self.expression = [transforms_dict['expression']]
self.betas = [transforms_dict['betas']]
image_paths = sorted(glob.glob(os.path.join(image_directory, '*.png')))
if not len(image_paths) == len(self.image_transform_map):
raise ValueError(
'Number of images in image_directory is not the same as number of transforms'
)
for image_path in image_paths:
camera_transform = np.array(
self.image_transform_map[os.path.basename(image_path)])
human_pose = np.array(image_pose_map[os.path.basename(image_path)])
image = cv2.imread(image_path)
self.h, self.w = image.shape[:2]
# should we append a list of the different h, w of all images? right now referencing only the last h, w
self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
rays_translation, rays_direction = get_rays(
self.h, self.w, self.focal, camera_transform)
trans_dir_rgb_stack = np.stack(
[rays_translation, rays_direction, image], -2)
trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
self.human_poses.append(
np.repeat(human_pose[np.newaxis, :],
trans_dir_rgb_list.shape[0],
axis=0))
self.rays.append(trans_dir_rgb_list)
self.rays = np.concatenate(self.rays)
self.human_poses = np.concatenate(self.human_poses)
self.canonical_smpl = get_smpl_vertices(self.betas, self.expression)
print('Finish initializing rays')
def __init__(self, image_directory: str, transforms_file: str,
transform) -> None:
"""
Parameters
----------
image_directory : str
Path to images.
transforms_file : str
File path to file containing transformation mappings.
transform :
List of callable transforms for preprocessing.
"""
super().__init__()
self.transform = transform
self.rays = [
] # list of arrays with ray translation, ray direction and rgb
print('Start initializing all rays of all images')
with open(transforms_file, 'r') as transforms_file:
transforms_dict = json.load(transforms_file)
camera_angle_x = transforms_dict['camera_angle_x']
self.image_transform_map = transforms_dict.get('image_transform_map')
image_paths = sorted(glob.glob(os.path.join(image_directory, '*.png')))
if not len(image_paths) == len(self.image_transform_map):
raise ValueError(
'Number of images in image_directory is not the same as number of transforms'
)
for image_path in image_paths:
camera_transform = np.array(
self.image_transform_map[os.path.basename(image_path)])
image = cv2.imread(image_path)
self.h, self.w = image.shape[:2]
self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
rays_translation, rays_direction = get_rays(
self.h, self.w, self.focal, camera_transform)
trans_dir_rgb_stack = np.stack(
[rays_translation, rays_direction, image], -2)
trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
self.rays.append(trans_dir_rgb_list)
if self.rays:
self.rays = np.concatenate(self.rays)
print('Finish initializing rays')
def get_warp(canonical: trimesh.base.Trimesh,
goal: trimesh.base.Trimesh,
camera_transform: np.array,
h: int,
w: int,
camera_angle_x: float,
debug: bool = False) -> np.array:
"""
Calculate warp vectors pointing from goal SMPL to canonical SMPL for the
closest ray intersection (wrt. camera origin) and return a warp
for each ray. If the ray doesn't intersect with the goal smpl than
a warp equal to zero will be returned for that ray (pixel)
Parameters
----------
canonical : trimesh.base.Trimesh
Canonical SMPL.
goal : trimesh.base.Trimesh
Goal SMPL.
camera_transform : np.array (4, 4)
Camera transformation matrix.
h : int
Height of camera.
w : int
Width of camera.
camera_angle_x : float
FOV of camera.
debug: bool
If True, a 3D and 2D plot of the image will be created and shown.
Returns
-------
warp_img : np.array (h, w, 3)
Warp vectors (3D) pointing from goal smpl to canonical smpl
intersections.
"""
f = .5 * w / np.tan(.5 * camera_angle_x)
rays_translation, rays_direction = get_rays(h, w, f, camera_transform)
camera_origin = rays_translation[0][0]
# calculate intersections with rays and goal smpl
intersector = RayMeshIntersector(goal)
goal_intersections = intersector.intersects_location(
rays_translation.reshape(-1, 3), rays_direction.reshape(-1, 3))
goal_intersections_points = goal_intersections[0] # (N_intersects, 3)
goal_intersections_face_indices = goal_intersections[2] # (N_intersects, )
goal_intersections_ray_indices = goal_intersections[1] # (N_intersects, )
# Find multiple intersections and use only closest
unique_goal_intersect_points = []
unique_goal_intersect_face_indices = []
unique_goal_intersect_ray_indices = []
depth = np.zeros((w * h))
intersect_indices = np.arange(len(goal_intersections_points))
for ray in np.unique(goal_intersections_ray_indices):
ray_mask = goal_intersections_ray_indices == ray
indices_ray = intersect_indices[ray_mask]
ray_intersects = goal_intersections_points[ray_mask]
distances_camera = np.linalg.norm(ray_intersects - camera_origin,
axis=1)
closest_intersect_index = indices_ray[np.argmin(distances_camera)]
unique_goal_intersect_points.append(
goal_intersections_points[closest_intersect_index])
unique_goal_intersect_face_indices.append(
goal_intersections_face_indices[closest_intersect_index])
unique_goal_intersect_ray_indices.append(
goal_intersections_ray_indices[closest_intersect_index])
depth[ray] = np.min(distances_camera)
assert (len(unique_goal_intersect_points) ==
len(unique_goal_intersect_face_indices) ==
len(unique_goal_intersect_ray_indices))
assert ((np.unique(goal_intersections_ray_indices) ==
unique_goal_intersect_ray_indices).all())
goal_intersections_points = np.array(unique_goal_intersect_points)
goal_intersections_face_indices = np.array(
unique_goal_intersect_face_indices)
goal_intersections_ray_indices = np.array(
unique_goal_intersect_ray_indices)
# Calculate for each intersection on goal SMPL the corresponding
# intersection on the canonical SMPL
canonical_intersections = []
for i, face_idx in enumerate(goal_intersections_face_indices):
vertex_indices = goal.faces[face_idx]
goal_vertices = goal.vertices[vertex_indices]
canonical_vertices = canonical.vertices[vertex_indices]
lin_coeffs_vertices = np.linalg.solve(goal_vertices.T,
goal_intersections_points[i])
canonical_intersection = canonical_vertices.T.dot(lin_coeffs_vertices)
canonical_intersections.append(canonical_intersection)
canonical_intersections = np.array(canonical_intersections)
# Calculate actual warp for intersections
warp = canonical_intersections - goal_intersections_points
# Set each pixel corresponding to ray index to the warp
warp_img_flat = np.zeros((h * w, 3))
warp_img_flat[goal_intersections_ray_indices] = warp
warp_img = warp_img_flat.reshape((h, w, 3))
warp_min = -1 #np.min(warp_img, axis=(0,1))
warp_max = 1 #np.max(warp_img, axis=(0,1))
warp_normalized = (warp_img - warp_min) / (warp_max - warp_min)
if debug:
plt.imshow(warp_normalized)
plt.show()
scene = pyrender.Scene()
lines_warp = np.hstack(
(goal_intersections_points,
goal_intersections_points + warp)).reshape(-1, 3)
primitive = [pyrender.Primitive(lines_warp, mode=1)]
primitive_mesh = pyrender.Mesh(primitive)
scene.add(primitive_mesh)
pyrender.Viewer(scene, use_raymond_lighting=True)
return warp_img, depth.reshape((h, w))