def set_renderer():
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Initialize an OpenGL perspective camera.
R, T = look_at_view_transform(2.0, 0, 180)
cameras = OpenGLOrthographicCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
bin_size = None,
max_faces_per_bin = None
)
lights = PointLights(device=device, location=((2.0, 2.0, 2.0),))
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=HardPhongShader(
device=device,
cameras=cameras,
lights=lights
)
)
return renderer
def __init__(self, meshes: Meshes, image_size=256, device='cuda'):
"""
Initialization of MaskRenderer. Renderer is initialized with predefined rasterizer and shader.
A soft silhouette shader is used to compute the projection mask.
:param meshes: A batch of meshes. pytorch3d.structures.Meshes. Dimension meshes \in R^N.
View https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py
for additional information.
In our case it is usually only one batch which is the template for a certain category.
:param device: The device, on which the computation is done.
:param image_size: Image size for the rasterization. Default is 256.
"""
super(MaskRenderer, self).__init__()
self.device = device
self._meshes = meshes
cameras = OpenGLOrthographicCameras(device=device)
# parameter settings as of Pytorch3D Tutorial
# (https://pytorch3d.org/tutorials/camera_position_optimization_with_differentiable_rendering)
self._rasterizer = MeshRasterizer(
cameras=cameras,
raster_settings=RasterizationSettings(image_size=image_size))
self._shader = SoftSilhouetteShader(
blend_params=(BlendParams(sigma=1e-4, gamma=1e-4)))
def set_renderer(image_size=512, use_sfm=False):
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Initialize an OpenGL perspective camera.
R, T = look_at_view_transform(2.0, 0, 180)
if use_sfm:
cameras = SfMPerspectiveCameras(focal_length=580.0,
device=device,
R=R,
T=T)
else:
cameras = OpenGLOrthographicCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None,
max_faces_per_bin=None)
lights = PointLights(device=device, location=((2.0, 2.0, 2.0), ))
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
shader = HardPhongShader(device=device, cameras=cameras, lights=lights)
if use_sfm:
renderer = MeshRendererWithDepth(rasterizer=rasterizer, shader=shader)
else:
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
return renderer
def _get_projected_positions_of_sphere_points(self, sphere_points, rotation, translation):
"""
For the given points on unit sphere calculates the 3D coordinates on the mesh template
and projects them back to image plane
:param sphere_points: A (B X 3 X H X W) tensor containing the predicted points on the sphere
:param rotation: A (B X CP X 3 X 3) camera rotation tensor
:param translation: A (B X CP X 3) camera translation tensor
:return: A tuple(xy, z, uv, uv_3d)
- xy - (B X CP X 2 X H X W) x,y values of the 3D points after projecting onto image plane
- z - (B X CP X 1 X H X W) z value of the projection
- uv - (B X 2 X H X W) UV values of the sphere coordinates
- uv_3d - (B X H X W X 3) tensor with the 3D coordinates on the mesh
template for the given sphere coordinates
"""
uv = convert_3d_to_uv_coordinates(sphere_points.permute(0, 2, 3, 1))
batch_size = uv.size(0)
height = uv.size(1)
width = uv.size(2)
num_poses = rotation.size(1)
uv_flatten = uv.view(-1, 2)
uv_3d = self.uv_to_3d(uv_flatten).view(batch_size, 1, -1, 3)
uv_3d = uv_3d.repeat(1, num_poses, 1, 1).view(batch_size*num_poses, -1, 3)
cameras = OpenGLOrthographicCameras(device=sphere_points.device, R=rotation.view(-1, 3, 3), T=translation.view(-1, 3))
xyz_cam = cameras.get_world_to_view_transform().transform_points(uv_3d)
z = xyz_cam[:, :, 2:].view(batch_size, num_poses, height, width, 1)
xy = cameras.transform_points(uv_3d)[:, :, :2].view(batch_size, num_poses, height, width, 2)
xy = xy.permute(0, 1, 4, 2, 3).flip(2)
z = z.permute(0, 1, 4, 2, 3)
uv = uv.permute(0, 3, 1, 2)
uv_3d = uv_3d.view(batch_size, num_poses, height, width, 3)[:, 0, :, :, :].squeeze()
return xy, z, uv, uv_3d
def __init__(self, meshes, image_size=256, device='cuda'):
super(ColorRenderer, self).__init__()
self.meshes = meshes
cameras = OpenGLOrthographicCameras(device=device)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
self.renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=TexturedSoftPhongShader(device=device, lights=lights))
def __init__(self, meshes: Meshes, image_size=256):
"""
Initialization of the Renderer Class. Instances of the mask and depth renderer are create on corresponding
device.
:param device: The device, on which the computation is done.
:param image_size: Image size for the rasterization. Default is 256.
"""
super().__init__()
self.meshes = meshes
device = meshes.device
# TODO: check how to implement weak perspective (scaled orthographic).
cameras = OpenGLOrthographicCameras(device=device)
self._rasterizer = MeshRasterizer(
cameras=cameras,
raster_settings=RasterizationSettings(image_size=image_size,
faces_per_pixel=100))
self._shader = SoftSilhouetteShader(
blend_params=(BlendParams(sigma=1e-4, gamma=1e-4)))
def __init__(self, meshes: Meshes, device: str, image_size: int = 256):
"""
Initialization of DepthRenderer. Initialization of the default mesh rasterizer and silhouette shader which is
used because of simplicity.
:param device: The device, on which the computation is done, e.g. cpu or cuda.
:param image_size: Image size for the rasterization. Default is 256.
:param meshes: A batch of meshes. pytorch3d.structures.Meshes. Dimension meshes in R^N.
View https://github.com/facebookresearch/pytorch3d/blob/master/pytorch3d/structures/meshes.py
for additional information.
"""
super(DepthRenderer, self).__init__()
self._meshes = meshes
# TODO: check how to implement weak perspective (scaled orthographic).
cameras = OpenGLOrthographicCameras(device=device)
raster_settings = RasterizationSettings(image_size=image_size)
self._rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
self._shader = SoftSilhouetteShader(
blend_params=(BlendParams(sigma=1e-4, gamma=1e-4)))
def __init__(self, cfgs):
super(Renderer, self).__init__()
self.device = cfgs.get('device', 'cuda:0')
self.image_size = cfgs.get('image_size', 64)
self.min_depth = cfgs.get('min_depth', 0.9)
self.max_depth = cfgs.get('max_depth', 1.1)
self.rot_center_depth = cfgs.get('rot_center_depth',
(self.min_depth + self.max_depth) / 2)
# todo: FoV (Field of View) was set to be an fixed value of 10 degree (according to the paper).
self.fov = cfgs.get('fov', 10)
self.tex_cube_size = cfgs.get('tex_cube_size', 2)
self.renderer_min_depth = cfgs.get('renderer_min_depth', 0.1)
self.renderer_max_depth = cfgs.get('renderer_max_depth', 10.)
#### camera intrinsics
# (u) (x)
# d * K^-1 (v) = (y)
# (1) (z)
## renderer for visualization
R = [[[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]]
R = torch.FloatTensor(R).to(self.device)
t = torch.zeros(1, 3, dtype=torch.float32).to(self.device)
## todo: K is the camera intrinsic matrix.
fx = (self.image_size - 1) / 2 / (math.tan(
self.fov / 2 * math.pi / 180))
fy = (self.image_size - 1) / 2 / (math.tan(
self.fov / 2 * math.pi / 180))
cx = (self.image_size - 1) / 2
cy = (self.image_size - 1) / 2
K = [[fx, 0., cx], [0., fy, cy], [0., 0., 1.]]
K = torch.FloatTensor(K).to(self.device)
self.inv_K = torch.inverse(K).unsqueeze(0)
self.K = K.unsqueeze(0)
## todo: define Renderer.
## use renderer from pytorch3d.
# fixme: znear and zfar is equivalent to the neural renderer default settings.
cameras = OpenGLOrthographicCameras(device=self.device,
R=R,
T=t,
znear=0.01,
zfar=100)
# cameras = OpenGLPerspectiveCameras(device=self.device, R=R, T=t,
# znear=self.renderer_min_depth,
# zfar=self.renderer_max_depth,
# fov=self.fov)
raster_settings = PointsRasterizationSettings(
image_size=self.image_size,
radius=0.003,
points_per_pixel=10,
bin_size=None,
max_points_per_bin=None)
self.renderer = PointsRenderer(
rasterizer=PointsRasterizer(cameras=cameras,
raster_settings=raster_settings),
compositor=AlphaCompositor(composite_params=None))
'car_mean_shape.mat'),
device=device)
uv_to_3d = UVto3D(mean_shape, device)
key_point_colors = np.random.uniform(0, 1, (len(dataset.kp_names), 3))
for i, data in enumerate(data_loader):
img = data['img'].to(device, dtype=torch.float)
scale = data['scale'].to(device, dtype=torch.float)
trans = data['trans'].to(device, dtype=torch.float)
quat = data['quat'].to(device, dtype=torch.float)
rotation, translation = get_scaled_orthographic_projection(
scale, trans, quat, True)
camera = OpenGLOrthographicCameras(device=device,
R=rotation,
T=translation)
kps = (((data['kp'].to(device, dtype=torch.float) + 1) / 2) * 255).to(
torch.int32)
kp = draw_key_points(img, kps, key_point_colors)
fig = plt.figure()
plt.subplot(2, 2, 1)
plt.imshow(kp[0].permute(1, 2, 0).cpu())
# Plot the key points directly using the 3D keypoints and projecting them onto image plane
kp_3d = torch.from_numpy(dataset.kp_3d).to(
device, dtype=torch.float32).unsqueeze(0)
xyz = camera.transform_points(kp_3d)
xy = (((xyz[:, :, :2] + 1) / 2) * 255).to(torch.int32)
kp_xy = torch.cat((xy, kps[:, :, 2:]), dim=2)