def set_renderer(image_size=512, use_sfm=False):
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Initialize an OpenGL perspective camera.
R, T = look_at_view_transform(2.0, 0, 180)
if use_sfm:
cameras = SfMPerspectiveCameras(focal_length=580.0,
device=device,
R=R,
T=T)
else:
cameras = OpenGLOrthographicCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None,
max_faces_per_bin=None)
lights = PointLights(device=device, location=((2.0, 2.0, 2.0), ))
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
shader = HardPhongShader(device=device, cameras=cameras, lights=lights)
if use_sfm:
renderer = MeshRendererWithDepth(rasterizer=rasterizer, shader=shader)
else:
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
return renderer
def set_renderer():
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Initialize an OpenGL perspective camera.
R, T = look_at_view_transform(2.0, 0, 180)
cameras = OpenGLOrthographicCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
bin_size = None,
max_faces_per_bin = None
)
lights = PointLights(device=device, location=((2.0, 2.0, 2.0),))
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=HardPhongShader(
device=device,
cameras=cameras,
lights=lights
)
)
return renderer
def create_renderer(self):
self.num_angles = self.config.num_angles
azim = torch.linspace(-1 * self.config.angle_range,
self.config.angle_range, self.num_angles)
R, T = look_at_view_transform(dist=1.0, elev=0, azim=azim)
T[:, 1] = -85
T[:, 2] = 200
cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=self.config.img_size,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=self.device, location=[[0.0, 85, 100.0]])
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=self.device,
cameras=cameras,
lights=lights))
return renderer
def visualize_pred(img,
category,
pred,
image_name,
mesh_path,
down_sample_rate=8,
device='cuda:0'):
render_image_size = max(IMAGE_SIZES[category])
crop_size = IMAGE_SIZES[category]
cameras = OpenGLPerspectiveCameras(device=device, fov=12.0)
raster_settings = RasterizationSettings(image_size=render_image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
raster_settings1 = RasterizationSettings(image_size=render_image_size //
down_sample_rate,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings1)
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
lights=lights,
cameras=cameras))
theta_pred = pred['theta']
elevation_pred = pred['elevation']
azimuth_pred = pred['azimuth']
distance_pred = pred['distance']
cad_idx = pred['cad_idx']
dx = pred['dx'] * down_sample_rate
dy = pred['dy'] * down_sample_rate
x3d, xface = load_off(mesh_path + '/%02d.off' % cad_idx)
verts = torch.from_numpy(x3d).to(device)
verts = pre_process_mesh_pascal(verts)
faces = torch.from_numpy(xface).to(device)
verts_rgb = torch.ones_like(verts)[None]
textures = Textures(verts_rgb.to(device))
meshes = Meshes(verts=[verts], faces=[faces], textures=textures)
img_ = get_img(theta_pred, elevation_pred, azimuth_pred, distance_pred,
meshes, phong_renderer, crop_size, render_image_size,
device)
C = camera_position_from_spherical_angles(distance_pred,
elevation_pred,
azimuth_pred,
degrees=False,
device=device)
# get_image = np.concatenate((img, alpha_merge_imgs(img, img_)), axis=1)
img_ = shift_img(img_, dx, dy)
get_image = alpha_merge_imgs(img, img_)
img = Image.fromarray(get_image).save(image_name)
def createRenderer(image_size,faces_per_pixel,lights_location):
# Function: createRenderer
# Inputs: image_size,faces_per_pixel,lights_location
# Process: creates an image renderer
# Output: returns renderer
cameras = OpenGLPerspectiveCameras()
#Settings for Raster
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=0.0,
faces_per_pixel=faces_per_pixel,
)
# We can add a point light in front of the object.
lights = PointLights(location=(lights_location,))
created_renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=HardPhongShader(cameras=cameras, lights=lights)
)
return created_renderer
def createRenderer(device, camera, light, imageSize):
'''
It creates a pytorch3D renderer with the given camera pose, light source
and output image size.
Parameters
----------
device :
Device on which the renderer is created.
camera :
Camera pose.
light :
Position of the light source.
imageSize :
The size of the rendered image.
Returns
-------
renderer :
Pytorch3D renderer.
'''
if camera is None:
camera = (2.0, -20.0, 180.0)
if light is None:
light = (0.0, 2.0, 0.0)
# Initialize an OpenGL perspective camera.
# With world coordinates +Y up, +X left and +Z into the screen.
R, T = look_at_view_transform(camera[0], camera[1], camera[2])
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that
# the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for
# explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of
# the difference between naive and coarse-to-fine rasterization.
raster_settings = RasterizationSettings(
image_size=imageSize,
blur_radius=0.0,
faces_per_pixel=1,
)
# Place a point light at -y direction.
lights = PointLights(device=device,
location=[[light[0], light[1], light[2]]])
# Create a phong renderer by composing a rasterizer and a shader.
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
cameras=cameras,
lights=lights))
return renderer
def render_phong(self, meshes):
lights = PointLights(device=self.device, location=((0.0, 0.0, 2.0), ))
raster_settings = RasterizationSettings(
image_size=self.resulution,
blur_radius=0.0,
faces_per_pixel=1,
)
phong_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=self.cameras,
raster_settings=raster_settings),
shader=HardPhongShader(device=self.device, lights=lights))
return phong_renderer(meshes_world=meshes)
def set_phong_renderer(self, light_location):
# Place a point light in front of the object
self.light_location = light_location
lights = PointLights(device=self.device, location=[light_location])
# Create a phong renderer by composing a rasterizer and a shader
self.phong_renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=self.cameras,
raster_settings=self.raster_settings
),
shader=HardPhongShader(device=self.device, lights=lights)
)
def render(mesh, model_id, shapenet_dataset, device, camera=None):
# Rendering settings.
# camera_distance = 1
# camera_elevation = 0.5 + 100 * random.random()
# camera_azimuth = 30 + 90 * random.random()
# R, T = look_at_view_transform(camera_distance, camera_elevation, camera_azimuth)
# camera = FoVPerspectiveCameras(R=R, T=T, device=device)
# raster_settings = RasterizationSettings(image_size=512)
# lights = PointLights(location=torch.tensor([0.0, 1.0, -2.0], device=device)[None],device=device)
# #rendering_settings = cameras, raster_settings, lights
# image = shapenet_dataset.render(
# model_ids=[model_id],
# device=device,
# cameras=camera,
# raster_settings=raster_settings,
# lights=lights,
# )[..., :3]
if not camera:
camera_elevation = 0 + 180 * torch.rand(
(1)) #torch.linspace(0, 180, batch_size)
camera_azimuth = -180 + 2 * 180 * torch.rand(
(1)) #torch.linspace(-180, 180, batch_size)
#R, T = look_at_view_transform(camera_distance, camera_elevation, camera_azimuth)
R, T = look_at_view_transform(1.9, camera_elevation, camera_azimuth)
camera = FoVPerspectiveCameras(R=R, T=T, device=device)
camera.eval() #necessary ?
raster_settings = RasterizationSettings(image_size=224) # TODO ?????
lights = PointLights(location=torch.tensor([0.0, 1.0, -2.0],
device=device)[None],
device=device)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=camera, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
cameras=camera))
renderer.eval()
#rendering_settings = cameras, raster_settings, lights
#image = shapenet_dataset.render(
# model_ids=[model_id],
# device=device,
# cameras=camera,
# raster_settings=raster_settings,
# lights=lights,
#)[..., :3]
image = renderer(mesh)[..., :3]
#plt.imshow(image.squeeze().detach().cpu().numpy())
#plt.show()
image = image.permute(0, 3, 1, 2)
return image, camera #TODO batch of images
def visualize_gif(img, distance_ckpt, elevation_ckpt, azimuth_ckpt,
theta_ckpt):
cameras = OpenGLPerspectiveCameras(device=device, fov=12.0)
raster_settings = RasterizationSettings(image_size=render_image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
raster_settings1 = RasterizationSettings(image_size=render_image_size //
down_smaple_rate,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings1)
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
lights=lights,
cameras=cameras))
distance_ckpt = torch.from_numpy(np.array(distance_ckpt,
dtype=np.float32)).to(device)
elevation_ckpt = torch.from_numpy(
np.array(elevation_ckpt, dtype=np.float32)).to(device)
azimuth_ckpt = torch.from_numpy(np.array(azimuth_ckpt,
dtype=np.float32)).to(device)
theta_ckpt = torch.from_numpy(np.array(theta_ckpt,
dtype=np.float32)).to(device)
img_ = get_img(theta_pred, elevation_pred, azimuth_pred, distance_pred,
phong_renderer)
C = camera_position_from_spherical_angles(distance_pred,
elevation_pred,
azimuth_pred,
degrees=False,
device=device)
get_images = []
for i in range(10):
get_image = np.concatenate((img, alpha_merge_imgs(img, img_[i])),
axis=1)
get_images.append(Image.fromarray(get_image))
get_images[0].save('optim.gif',
save_all=True,
append_images=get_images[1::])
def __init__(self, image_size, device):
super(Renderer, self).__init__()
self.image_size = image_size
R, T = look_at_view_transform(2.7, 0, 0, device=device)
self.cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
self.mesh_color = torch.FloatTensor(config.MESH_COLOR).to(device)[None, None, :] / 255.0
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=self.image_size,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
faces_per_pixel=100,
)
self.silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=self.cameras,
raster_settings=raster_settings
),
shader=SoftSilhouetteShader(blend_params=blend_params)
)
raster_settings_color = RasterizationSettings(
image_size=self.image_size,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=device, location=[[0.0, 0.0, 3.0]])
self.color_renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=self.cameras,
raster_settings=raster_settings_color
),
shader=HardPhongShader(
device=device,
cameras=self.cameras,
lights=lights,
)
)
def create_renderer(self):
self.num_angles_train = self.config.num_angles_train
self.num_angles_test = self.config.num_angles_test
azim_train = torch.linspace(-1 * self.config.angle_range_train, self.config.angle_range_train, self.num_angles_train)
azim_test = torch.linspace(-1 * self.config.angle_range_test, self.config.angle_range_test, self.num_angles_test)
# Cameras for SMPL meshes:
camera_dist = 2.2
R, T = look_at_view_transform(camera_dist, 6, azim_train)
train_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T)
self.train_cameras = train_cameras
R, T = look_at_view_transform(camera_dist, 6, azim_test)
test_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T)
self.test_cameras = test_cameras
raster_settings = RasterizationSettings(
image_size=self.config.img_size,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=self.device, location=[[0.0, 85, 100.0]])
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=train_cameras,
raster_settings=raster_settings
),
shader=HardPhongShader(
device=self.device,
cameras=train_cameras,
lights=lights
)
)
return renderer
# Create a silhouette mesh renderer by composing a rasterizer and a shader.
silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
# We will also create a phong renderer. This is simpler and only needs to render one face per pixel.
raster_settings = RasterizationSettings(image_size=256,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
# We can add a point light in front of the object.
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
lights=lights))
# Select the viewpoint using spherical angles
viewpoint = [1.5, 240.0, 10.0] # distance, elevation, azimuth, stuck..
#viewpoint = [1.5, 140.0, 10.0] # distance, elevation, azimuth, ok...
#viewpoint = [1.5, 160.0, 10.0] # distance, elevation, azimuth, ok...
# Get the position of the camera based on the spherical angles
R, T = look_at_view_transform(viewpoint[0],
viewpoint[1],
viewpoint[2],
device=device)
# Render the teapot providing the values of R and T.
silhouete = silhouette_renderer(meshes_world=teapot_mesh, R=R, T=T)
image = phong_renderer(meshes_world=teapot_mesh, R=R, T=T)
def batch_render(
verts,
faces,
faces_per_pixel=10,
K=None,
rot=None,
trans=None,
colors=None,
color=(0.53, 0.53, 0.8), # light_purple
ambient_col=0.5,
specular_col=0.2,
diffuse_col=0.3,
face_colors=None,
# color = (0.74117647, 0.85882353, 0.65098039), # light_blue
image_sizes=None,
out_res=512,
bin_size=0,
shading="soft",
mode="rgb",
blend_gamma=1e-4,
min_depth=None,
):
device = torch.device("cuda:0")
K = K.to(device)
width, height = image_sizes[0]
out_size = int(max(image_sizes[0]))
raster_settings = RasterizationSettings(
image_size=out_size,
blur_radius=0.0,
faces_per_pixel=faces_per_pixel,
bin_size=bin_size,
)
fx = K[:, 0, 0]
fy = K[:, 1, 1]
focals = torch.stack([fx, fy], 1)
px = K[:, 0, 2]
py = K[:, 1, 2]
principal_point = torch.stack([width - px, height - py], 1)
if rot is None:
rot = torch.eye(3).unsqueeze(0).to(device)
if trans is None:
trans = torch.zeros(3).unsqueeze(0).to(device)
cameras = PerspectiveCameras(
device=device,
focal_length=focals,
principal_point=principal_point,
image_size=[(out_size, out_size) for _ in range(len(verts))],
R=rot,
T=trans,
)
if mode == "rgb":
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
lights = DirectionalLights(
device=device,
direction=((0.6, -0.6, -0.6), ),
ambient_color=((ambient_col, ambient_col, ambient_col), ),
diffuse_color=((diffuse_col, diffuse_col, diffuse_col), ),
specular_color=((specular_col, specular_col, specular_col), ),
)
if shading == "soft":
shader = SoftPhongShader(device=device,
cameras=cameras,
lights=lights)
elif shading == "hard":
shader = HardPhongShader(device=device,
cameras=cameras,
lights=lights)
else:
raise ValueError(
f"Shading {shading} for mode rgb not in [sort|hard]")
elif mode == "silh":
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
shader = SoftSilhouetteShader(blend_params=blend_params)
elif shading == "faceidx":
shader = FaceIdxShader()
elif (mode == "facecolor") and (shading == "hard"):
shader = FaceColorShader(face_colors=face_colors)
elif (mode == "facecolor") and (shading == "soft"):
shader = SoftFaceColorShader(face_colors=face_colors,
blend_gamma=blend_gamma)
else:
raise ValueError(
f"Unhandled mode {mode} and shading {shading} combination")
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=shader,
)
if min_depth is not None:
verts = torch.cat([verts[:, :, :2], verts[:, :, 2:].clamp(min_depth)],
2)
if mode == "rgb":
if colors is None:
colors = get_colors(verts, color)
tex = textures.TexturesVertex(verts_features=colors)
meshes = Meshes(verts=verts, faces=faces, textures=tex)
elif mode in ["silh", "facecolor"]:
meshes = Meshes(verts=verts, faces=faces)
else:
raise ValueError(f"Render mode {mode} not in [rgb|silh]")
square_images = renderer(meshes, cameras=cameras)
square_images = torch.flip(square_images, (1, 2))
height_off = abs(int(width - height))
if width > height:
images = square_images[:, height_off:, :]
else:
images = square_images[:, :, height_off:]
return images
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None,
)
lights = PointLights(device=device,
ambient_color=((0.5, 0.5, 0.5), ),
location=((0.0, 0.0, 1e5), ))
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
cameras=cameras,
lights=lights))
#-----render-----
class AverageMeter(object):
"""Computes and stores the average and current value"""
def __init__(self, name, fmt=':f'):
self.name = name
self.fmt = fmt
self.reset()
def reset(self):
self.val = 0
self.avg = 0
self.sum = 0
def initRender(self, method, image_size):
cameras = OpenGLPerspectiveCameras(device=self.device, fov=15)
if (method == "soft-silhouette"):
blend_params = BlendParams(sigma=1e-7, gamma=1e-7)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-7 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
elif (method == "hard-silhouette"):
blend_params = BlendParams(sigma=1e-7, gamma=1e-7)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-7 - 1.) * blend_params.sigma,
faces_per_pixel=1)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
elif (method == "soft-depth"):
# Soft Rasterizer - from https://github.com/facebookresearch/pytorch3d/issues/95
#blend_params = BlendParams(sigma=1e-7, gamma=1e-7)
blend_params = BlendParams(sigma=1e-3, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=image_size,
#blur_radius= np.log(1. / 1e-7 - 1.) * blend_params.sigma,
blur_radius=np.log(1. / 1e-3 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftDepthShader(blend_params=blend_params))
elif (method == "hard-depth"):
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0,
faces_per_pixel=20)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardDepthShader())
elif (method == "blurry-depth"):
# Soft Rasterizer - from https://github.com/facebookresearch/pytorch3d/issues/95
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftDepthShader(blend_params=blend_params))
elif (method == "soft-phong"):
blend_params = BlendParams(sigma=1e-3, gamma=1e-3)
raster_settings = RasterizationSettings(
image_size=image_size,
blur_radius=np.log(1. / 1e-3 - 1.) * blend_params.sigma,
faces_per_pixel=self.faces_per_pixel)
# lights = DirectionalLights(device=self.device,
# ambient_color=[[0.25, 0.25, 0.25]],
# diffuse_color=[[0.6, 0.6, 0.6]],
# specular_color=[[0.15, 0.15, 0.15]],
# direction=[[0.0, 1.0, 0.0]])
lights = DirectionalLights(device=self.device,
direction=[[0.0, 1.0, 0.0]])
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(device=self.device,
blend_params=blend_params,
lights=lights))
elif (method == "hard-phong"):
blend_params = BlendParams(sigma=1e-8, gamma=1e-8)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1)
lights = DirectionalLights(device=self.device,
ambient_color=[[0.25, 0.25, 0.25]],
diffuse_color=[[0.6, 0.6, 0.6]],
specular_color=[[0.15, 0.15, 0.15]],
direction=[[-1.0, -1.0, 1.0]])
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=self.device,
lights=lights))
else:
print("Unknown render method!")
return None
return renderer
def main():
# Set the cuda device
device = torch.device("cuda:0")
torch.cuda.set_device(device)
mesh = load_objs_as_meshes(obj_paths, device=device)
texture_image = mesh.textures.maps_padded()
cameras = OpenGLRealPerspectiveCameras(
focal_length=((K[0, 0], K[1, 1]), ), # Nx2
principal_point=((K[0, 2], K[1, 2]), ), # Nx2
x0=0,
y0=0,
w=H,
h=H, # HEIGHT,
znear=ZNEAR,
zfar=ZFAR,
device=device,
)
# To blend the 100 faces we set a few parameters which control the opacity and the sharpness of
# edges. Refer to blending.py for more details.
blend_params = BlendParams(sigma=1e-4,
gamma=1e-4,
background_color=(0.0, 0.0, 0.0))
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 640x640. To form the blended image we use 100 faces for each pixel. Refer to rasterize_meshes.py
# for an explanation of this parameter.
silhouette_raster_settings = RasterizationSettings(
image_size=IMG_SIZE, # longer side or scaled longer side
blur_radius=np.log(1.0 / 1e-4 - 1.0) * blend_params.sigma,
faces_per_pixel=
100, # the nearest faces_per_pixel points along the z-axis.
bin_size=0,
)
# Create a silhouette mesh renderer by composing a rasterizer and a shader.
silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=silhouette_raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params),
)
# We will also create a phong renderer. This is simpler and only needs to render one face per pixel.
phong_raster_settings = RasterizationSettings(image_size=IMG_SIZE,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
# We can add a point light in front of the object.
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=phong_raster_settings),
shader=HardPhongShader(device=device,
cameras=cameras,
lights=lights,
blend_params=blend_params),
)
batch_R = torch.tensor(np.stack([R]), device=device,
dtype=torch.float32).permute(0, 2, 1) # Bx3x3
batch_T = torch.tensor(np.stack([t]), device=device,
dtype=torch.float32) # Bx3
silhouete = silhouette_renderer(meshes_world=mesh, R=batch_R, T=batch_T)
image_ref = phong_renderer(meshes_world=mesh, R=batch_R, T=batch_T)
# crop results
silhouete = silhouete[:, :H, :W, :].cpu().numpy()
image_ref = image_ref[:, :H, :W, :3].cpu().numpy()
pred_images = image_ref
opengl = mmcv.imread(osp.join("Render/OpenGL.png"), "color") / 255.0
for i in range(pred_images.shape[0]):
pred_mask = silhouete[i, :, :, 3].astype("float32")
print("num rendered images", pred_images.shape[0])
image = pred_images[i]
diff_opengl = np.abs(opengl[:, :, ::-1].astype("float32") -
image.astype("float32"))
print("image", image.shape, image.min(), image.max())
print("dr mask area: ", pred_mask.sum())
print_stat(pred_mask, "pred_mask")
show_ims = [image, diff_opengl, opengl[:, :, ::-1]]
show_titles = ["image", "diff_opengl", "opengl"]
grid_show(show_ims, show_titles, row=1, col=3)
def run_on_image(self, image, id_str, gt_verts, gt_faces):
deprocess = imagenet_deprocess(rescale_image=False)
with torch.no_grad():
voxel_scores, meshes_pred = self.predictor(image)
sid, mid, iid = id_str.split('-')
iid = int(iid)
#Transform vertex space
metadata_path = os.path.join('./datasets/shapenet/ShapeNetV1processed',
sid, mid, "metadata.pt")
metadata = torch.load(metadata_path)
K = metadata["intrinsic"]
RTs = metadata["extrinsics"]
rot_y_90 = torch.tensor([[0, 0, 1, 0], [0, 1, 0, 0], [-1, 0, 0, 0],
[0, 0, 0, 1]]).to(RTs)
mesh = meshes_pred[-1][0]
#For some strange reason all classes (expect vehicle class) require a 90 degree rotation about the y-axis
#for the GT mesh
invRT = torch.inverse(RTs[iid].mm(rot_y_90))
invRT_no_rot = torch.inverse(RTs[iid])
mesh._verts_list[0] = project_verts(mesh._verts_list[0], invRT.cpu())
#Get look at view extrinsics
render_metadata_path = os.path.join(
'datasets/shapenet/ShapeNetRenderingExtrinsics', sid, mid,
'rendering_metadata.pt')
render_metadata = torch.load(render_metadata_path)
render_RTs = render_metadata['extrinsics']
plt.figure(figsize=(10, 10))
R = render_RTs[iid][:3, :3].unsqueeze(0)
T = render_RTs[iid][:3, 3].unsqueeze(0)
cameras = OpenGLPerspectiveCameras(R=R, T=T)
#Phong Renderer
lights = PointLights(location=[[0.0, 0.0, -3.0]])
raster_settings = RasterizationSettings(image_size=256,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(lights=lights))
#Silhouette Renderer
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(
image_size=256,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma,
faces_per_pixel=50,
)
silhouette_renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
verts, faces = mesh.get_mesh_verts_faces(0)
verts_rgb = torch.ones_like(verts)[None]
textures = Textures(verts_rgb=verts_rgb)
mesh.textures = textures
verts_rgb = torch.ones_like(gt_verts)[None]
textures = Textures(verts_rgb=verts_rgb)
#Invert without the rotation for the vehicle class
if sid == '02958343':
gt_verts = project_verts(gt_verts, invRT_no_rot.cpu())
else:
gt_verts = project_verts(gt_verts, invRT.cpu())
gt_mesh = Meshes(verts=[gt_verts], faces=[gt_faces], textures=textures)
img = image_to_numpy(deprocess(image[0]))
mesh_image = phong_renderer(meshes_world=mesh, R=R, T=T)
gt_silh_image = (silhouette_renderer(meshes_world=gt_mesh, R=R, T=T) >
0).float()
silhouette_image = (silhouette_renderer(meshes_world=mesh, R=R, T=T) >
0).float()
plt.subplot(2, 2, 1)
plt.imshow(img)
plt.title('input image')
plt.subplot(2, 2, 2)
plt.imshow(mesh_image[0, ..., :3].cpu().numpy())
plt.title('rendered mesh')
plt.subplot(2, 2, 3)
plt.imshow(gt_silh_image[0, ..., 3].cpu().numpy())
plt.title('silhouette of gt mesh')
plt.subplot(2, 2, 4)
plt.imshow(silhouette_image[0, ..., 3].cpu().numpy())
plt.title('silhouette of rendered mesh')
plt.show()
#plt.savefig('./output_demo/figures/'+id_str+'.png')
vis_utils.visualize_prediction(id_str, img, mesh, self.output_dir)
# Renderer setting
cameras = FoVOrthographicCameras(device=device)
lights = PointLights(
ambient_color=((0.99, 0.99, 0.99), ),
diffuse_color=((0, 0, 0), ),
specular_color=((0, 0, 0), ),
device=device,
)
raster_settings = RasterizationSettings(image_size=image_size)
blend_params = BlendParams(background_color=(0.0, 0.0, 0.0), sigma=2e-4)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=HardPhongShader(device=device,
cameras=cameras,
lights=lights,
blend_params=blend_params),
)
soft_raster_settings = RasterizationSettings(
image_size=image_size,
faces_per_pixel=3,
blur_radius=np.log(1. / 1e-4 - 1.) * blend_params.sigma)
renderer_silhouette = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=soft_raster_settings),
shader=SoftSilhouetteShader(blend_params=blend_params))
# Optimization parameters
N = len(meshes[0].verts_packed())
offset_values = torch.zeros((len(meshes), 2),
device=device,