Ejemplo n.º 1
0
 def __init__(self, camera_transforms: np.array, height: int, widht: int,
              focal: float, transform) -> None:
     """
     Parameters
     ----------
     camera_transforms : [np.array]
         List of camera transformations for inference.
     height : int
         Height of image.
     widht : int
         Width of image.
     focal : float
         Focal length of camera.
     transform :
         List of callable transforms for preprocessing.
     """
     super().__init__()
     self.transform = transform
     self.rays = [
     ]  # list of arrays with ray translation, ray direction and rgb
     print('Start initializing all rays of all images')
     for transform in camera_transforms:
         rays_translation, rays_direction = get_rays(
             height, widht, focal, transform)
         trans_dir_stack = np.stack([rays_translation, rays_direction], -2)
         trans_dir_list = trans_dir_stack.reshape((-1, 2, 3))
         self.rays.append(trans_dir_list)
     self.rays = np.concatenate(self.rays)
     print('Finish initializing rays')
Ejemplo n.º 2
0
 def __init__(self, base_directory: str, transforms_file: str,
              transform) -> None:
     """
     Parameters
     ----------
     base_directory : str
         Path to dataset.
     transforms_file : str
         File path to file containing transformation mappings.
     transform :
         List of callable transforms for preprocessing.
     """
     super().__init__()
     self.transform = transform
     self.rays = []  # list of arrays with ray translation, ray direction and rgb
     print('Start initializing all rays of all images')
     with open(transforms_file, 'r') as transforms_file:
         transforms_dict = json.load(transforms_file)
     camera_angle_x = transforms_dict['camera_angle_x']
     for frame in transforms_dict['frames']:
         fname = os.path.join(base_directory, frame['file_path'] + '.png')
         image = cv2.imread(fname)
         camera_transform = np.array(frame['transform_matrix'])
         self.h, self.w = image.shape[:2]
         self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
         rays_translation, rays_direction = get_rays(self.h, self.w, self.focal,
                                                     camera_transform)
         trans_dir_rgb_stack = np.stack([rays_translation, rays_direction, image], -2)
         trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
         self.rays.append(trans_dir_rgb_list)
         
     if self.rays:
         self.rays = np.concatenate(self.rays)
     print('Finish initializing rays')
Ejemplo n.º 3
0
    def __init__(self, image_directory: str, transforms_file: str,
                 transform) -> None:
        """
        Parameters
        ----------
        image_directory : str
            Path to images.
        transforms_file : str
            File path to file containing transformation mappings.
        transform :
            List of callable transforms for preprocessing.
        """
        super().__init__()
        self.transform = transform
        self.rays = [
        ]  # list of arrays with ray translation, ray direction and rgb
        self.human_poses = []  # list of corresponding human poses
        self.dependencies_index = []
        self.dependencies_hw = []
        print('Start initializing all rays of all images')
        with open(transforms_file, 'r') as transforms_file:
            transforms_dict = json.load(transforms_file)
        camera_angle_x = transforms_dict['camera_angle_x']
        image_transform_map = transforms_dict.get('image_transform_map')
        image_pose_map = transforms_dict.get('image_pose_map')

        image_paths = sorted(glob.glob(os.path.join(image_directory, '*.png')))
        if not len(image_paths) == len(image_transform_map):
            raise ValueError(
                'Number of images in image_directory is not the same as number of transforms'
            )
        for image_path in image_paths:
            camera_transform = image_transform_map[os.path.basename(
                image_path)]
            human_pose = image_pose_map[os.path.basename(image_path)]
            self.human_poses.append(human_pose)

            image = cv2.imread(image_path)
            self.h, self.w = image.shape[:2]

            # should we append a list of the different h, w of all images? right now referencing only the last h, w
            self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
            rays_translation, rays_direction = get_rays(
                self.h, self.w, self.focal, camera_transform)
            for i in range(self.w):
                for j in range(self.h):
                    index_list = get_dependent_rays_indices(
                        rays_translation[i][j], rays_direction[i][j],
                        canonical, goal, camera_transform, self.h, self.w,
                        self.focal)
                    self.dependencies_index.append(index_list)
                    self.dependencies_hw.append((i, j))

            trans_dir_rgb_stack = np.stack(
                [rays_translation, rays_direction, image], -2)
            trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
            self.rays.append(trans_dir_rgb_list)
        self.rays = np.concatenate(self.rays)
        print('Finish initializing rays')
Ejemplo n.º 4
0
def generate_video_frames(H, W, focal, model, N_samples):
    frames = []
    for th in tqdm(np.linspace(0., 360., 120, endpoint=False)):
        c2w = pose_spherical(th, -30., 4.)
        rays_o, rays_d = utils.get_rays(H, W, focal, c2w[:3,:4])
        rgb, depth, acc = utils.render_rays(model, rays_o, rays_d, near=2., far=6., N_samples=N_samples)
        frames.append((255*np.clip(rgb,0,1)).astype(np.uint8))
    return frames 
Ejemplo n.º 5
0
    def __init__(self, image_directory: str, transforms_file: str,
                 transform) -> None:
        """
        Parameters
        ----------
        image_directory : str
            Path to images.
        transforms_file : str
            File path to file containing transformation mappings.
        transform :
            List of callable transforms for preprocessing.
        """
        super().__init__()
        self.transform = transform
        self.rays = [
        ]  # list of arrays with ray translation, ray direction and rgb
        self.human_poses = []  # list of corresponding human poses
        print('Start initializing all rays of all images')
        with open(transforms_file, 'r') as transforms_file:
            transforms_dict = json.load(transforms_file)
        camera_angle_x = transforms_dict['camera_angle_x']
        self.image_transform_map = transforms_dict.get('image_transform_map')
        image_pose_map = transforms_dict.get('image_pose_map')
        self.expression = [transforms_dict['expression']]
        self.betas = [transforms_dict['betas']]
        image_paths = sorted(glob.glob(os.path.join(image_directory, '*.png')))
        if not len(image_paths) == len(self.image_transform_map):
            raise ValueError(
                'Number of images in image_directory is not the same as number of transforms'
            )
        for image_path in image_paths:
            camera_transform = np.array(
                self.image_transform_map[os.path.basename(image_path)])
            human_pose = np.array(image_pose_map[os.path.basename(image_path)])

            image = cv2.imread(image_path)
            self.h, self.w = image.shape[:2]

            # should we append a list of the different h, w of all images? right now referencing only the last h, w
            self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
            rays_translation, rays_direction = get_rays(
                self.h, self.w, self.focal, camera_transform)

            trans_dir_rgb_stack = np.stack(
                [rays_translation, rays_direction, image], -2)
            trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
            self.human_poses.append(
                np.repeat(human_pose[np.newaxis, :],
                          trans_dir_rgb_list.shape[0],
                          axis=0))
            self.rays.append(trans_dir_rgb_list)
        self.rays = np.concatenate(self.rays)
        self.human_poses = np.concatenate(self.human_poses)
        self.canonical_smpl = get_smpl_vertices(self.betas, self.expression)
        print('Finish initializing rays')
Ejemplo n.º 6
0
 def __init__(self, image_directory: str, transforms_file: str,
              transform) -> None:
     """
     Parameters
     ----------
     image_directory : str
         Path to images.
     transforms_file : str
         File path to file containing transformation mappings.
     transform :
         List of callable transforms for preprocessing.
     """
     super().__init__()
     self.transform = transform
     self.rays = [
     ]  # list of arrays with ray translation, ray direction and rgb
     print('Start initializing all rays of all images')
     with open(transforms_file, 'r') as transforms_file:
         transforms_dict = json.load(transforms_file)
     camera_angle_x = transforms_dict['camera_angle_x']
     self.image_transform_map = transforms_dict.get('image_transform_map')
     image_paths = sorted(glob.glob(os.path.join(image_directory, '*.png')))
     if not len(image_paths) == len(self.image_transform_map):
         raise ValueError(
             'Number of images in image_directory is not the same as number of transforms'
         )
     for image_path in image_paths:
         camera_transform = np.array(
             self.image_transform_map[os.path.basename(image_path)])
         image = cv2.imread(image_path)
         self.h, self.w = image.shape[:2]
         self.focal = .5 * self.w / np.tan(.5 * camera_angle_x)
         rays_translation, rays_direction = get_rays(
             self.h, self.w, self.focal, camera_transform)
         trans_dir_rgb_stack = np.stack(
             [rays_translation, rays_direction, image], -2)
         trans_dir_rgb_list = trans_dir_rgb_stack.reshape((-1, 3, 3))
         self.rays.append(trans_dir_rgb_list)
     if self.rays:
         self.rays = np.concatenate(self.rays)
     print('Finish initializing rays')
Ejemplo n.º 7
0
def get_warp(canonical: trimesh.base.Trimesh,
             goal: trimesh.base.Trimesh,
             camera_transform: np.array,
             h: int,
             w: int,
             camera_angle_x: float,
             debug: bool = False) -> np.array:
    """
    Calculate warp vectors pointing from goal SMPL to canonical SMPL for the
    closest ray intersection (wrt. camera origin) and return a warp
    for each ray. If the ray doesn't intersect with the goal smpl than
    a warp equal to zero will be returned for that ray (pixel)

    Parameters
    ----------
    canonical : trimesh.base.Trimesh
        Canonical SMPL.
    goal : trimesh.base.Trimesh
        Goal SMPL.
    camera_transform : np.array (4, 4)
        Camera transformation matrix.
    h : int
        Height of camera.
    w : int
        Width of camera.
    camera_angle_x : float
        FOV of camera.
    debug: bool
        If True, a 3D and 2D plot of the image will be created and shown.

    Returns
    -------
    warp_img : np.array (h, w, 3)
        Warp vectors (3D) pointing from goal smpl to canonical smpl
        intersections.

    """
    f = .5 * w / np.tan(.5 * camera_angle_x)
    rays_translation, rays_direction = get_rays(h, w, f, camera_transform)
    camera_origin = rays_translation[0][0]

    # calculate intersections with rays and goal smpl
    intersector = RayMeshIntersector(goal)
    goal_intersections = intersector.intersects_location(
        rays_translation.reshape(-1, 3), rays_direction.reshape(-1, 3))
    goal_intersections_points = goal_intersections[0]  # (N_intersects, 3)
    goal_intersections_face_indices = goal_intersections[2]  # (N_intersects, )
    goal_intersections_ray_indices = goal_intersections[1]  # (N_intersects, )

    # Find multiple intersections and use only closest
    unique_goal_intersect_points = []
    unique_goal_intersect_face_indices = []
    unique_goal_intersect_ray_indices = []
    depth = np.zeros((w * h))
    intersect_indices = np.arange(len(goal_intersections_points))
    for ray in np.unique(goal_intersections_ray_indices):
        ray_mask = goal_intersections_ray_indices == ray
        indices_ray = intersect_indices[ray_mask]
        ray_intersects = goal_intersections_points[ray_mask]
        distances_camera = np.linalg.norm(ray_intersects - camera_origin,
                                          axis=1)
        closest_intersect_index = indices_ray[np.argmin(distances_camera)]
        unique_goal_intersect_points.append(
            goal_intersections_points[closest_intersect_index])
        unique_goal_intersect_face_indices.append(
            goal_intersections_face_indices[closest_intersect_index])
        unique_goal_intersect_ray_indices.append(
            goal_intersections_ray_indices[closest_intersect_index])
        depth[ray] = np.min(distances_camera)
    assert (len(unique_goal_intersect_points) ==
            len(unique_goal_intersect_face_indices) ==
            len(unique_goal_intersect_ray_indices))
    assert ((np.unique(goal_intersections_ray_indices) ==
             unique_goal_intersect_ray_indices).all())
    goal_intersections_points = np.array(unique_goal_intersect_points)
    goal_intersections_face_indices = np.array(
        unique_goal_intersect_face_indices)
    goal_intersections_ray_indices = np.array(
        unique_goal_intersect_ray_indices)

    # Calculate for each intersection on goal SMPL the corresponding
    # intersection on the canonical SMPL
    canonical_intersections = []
    for i, face_idx in enumerate(goal_intersections_face_indices):
        vertex_indices = goal.faces[face_idx]
        goal_vertices = goal.vertices[vertex_indices]
        canonical_vertices = canonical.vertices[vertex_indices]
        lin_coeffs_vertices = np.linalg.solve(goal_vertices.T,
                                              goal_intersections_points[i])
        canonical_intersection = canonical_vertices.T.dot(lin_coeffs_vertices)
        canonical_intersections.append(canonical_intersection)
    canonical_intersections = np.array(canonical_intersections)

    # Calculate actual warp for intersections
    warp = canonical_intersections - goal_intersections_points
    # Set each pixel corresponding to ray index to the warp
    warp_img_flat = np.zeros((h * w, 3))
    warp_img_flat[goal_intersections_ray_indices] = warp
    warp_img = warp_img_flat.reshape((h, w, 3))

    warp_min = -1  #np.min(warp_img, axis=(0,1))
    warp_max = 1  #np.max(warp_img, axis=(0,1))
    warp_normalized = (warp_img - warp_min) / (warp_max - warp_min)
    if debug:
        plt.imshow(warp_normalized)
        plt.show()
        scene = pyrender.Scene()
        lines_warp = np.hstack(
            (goal_intersections_points,
             goal_intersections_points + warp)).reshape(-1, 3)
        primitive = [pyrender.Primitive(lines_warp, mode=1)]
        primitive_mesh = pyrender.Mesh(primitive)
        scene.add(primitive_mesh)
        pyrender.Viewer(scene, use_raymond_lighting=True)
    return warp_img, depth.reshape((h, w))