def __init__(self, image_size):
super(Renderer, self).__init__()
self.image_size = image_size
self.dog_obj = load_objs_as_meshes(['data/dog_B/dog_B/dog_B_tpose.obj'])
raster_settings = RasterizationSettings(
image_size=self.image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None
)
R, T = look_at_view_transform(2.7, 0, 0)
cameras = OpenGLPerspectiveCameras(device=R.device, R=R, T=T)
lights = PointLights(device=R.device, location=[[0.0, 1.0, 0.0]])
self.renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=SoftPhongShader(
device=R.device,
cameras=cameras,
lights=lights
)
)
def set_renderer():
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Initialize an OpenGL perspective camera.
R, T = look_at_view_transform(2.0, 0, 180)
cameras = OpenGLOrthographicCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
bin_size = None,
max_faces_per_bin = None
)
lights = PointLights(device=device, location=((2.0, 2.0, 2.0),))
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings
),
shader=HardPhongShader(
device=device,
cameras=cameras,
lights=lights
)
)
return renderer
def define_render(num):
shapenet_cam_params_file = '../data/metadata/rendering_metadata.json'
with open(shapenet_cam_params_file) as f:
shapenet_cam_params = json.load(f)
param_num = num
R, T = look_at_view_transform(
dist=shapenet_cam_params["distance"][param_num] * 5,
elev=shapenet_cam_params["elevation"][param_num],
azim=shapenet_cam_params["azimuth"][param_num])
cameras = FoVPerspectiveCameras(
device=device,
R=R,
T=T,
fov=shapenet_cam_params["field_of_view"][param_num])
raster_settings = RasterizationSettings(
image_size=512,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=device, location=[[0.0, 0.0, -3.0]])
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights))
return renderer
def sphere_render_bsdf(bsdf,
integrator=None,
device="cuda",
size=256,
chunk_size=128,
scale=100):
from pytorch3d.pathtracer.shapes import Sphere
from pytorch3d.pathtracer.bsdf import Diffuse
from pytorch3d.pathtracer import pathtrace
from pytorch3d.renderer import (
look_at_view_transform,
OpenGLPerspectiveCameras,
PointLights,
)
import pytorch3d.pathtracer.integrators as integrators
sphere = Sphere([0, 0, 0], 1, device=device)
R, T = look_at_view_transform(dist=2., elev=0, azim=0)
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
lights = PointLights(device=device, location=[[0., 1., 4.]], scale=scale)
if integrator is None:
integrator = integrators.Direct()
return pathtrace(
sphere,
cameras=cameras,
lights=lights,
chunk_size=chunk_size,
size=size,
bsdf=bsdf,
integrator=integrator,
device=device,
silent=True,
)[0]
def rotate_mesh_around_axis(
mesh, rot: list, renderer, dist: float = 3.5, save: str = "", device: str = ""
):
if not device:
device = torch.cuda.current_device()
rot_x = RotateAxisAngle(rot[0], "X", device=device)
rot_y = RotateAxisAngle(rot[1], "Y", device=device)
rot_z = RotateAxisAngle(rot[2], "Z", device=device)
rot = Transform3d(device=device).stack(rot_x, rot_y, rot_z)
verts, faces = mesh.get_mesh_verts_faces(0)
verts = rot_x.transform_points(verts)
verts = rot_y.transform_points(verts)
verts = rot_z.transform_points(verts)
mesh = mesh.update_padded(verts.unsqueeze(0))
dist = dist
elev = 0
azim = 0
R, T = look_at_view_transform(dist=dist, elev=elev, azim=azim, device=device)
image_ref = renderer(meshes_world=mesh, R=R, T=T, device=device)
image_ref = image_ref.cpu().numpy()[..., :3]
plt.imshow(image_ref.squeeze())
plt.show()
if save:
verts, faces = mesh.get_mesh_verts_faces(0)
save_obj(save, verts, faces)
return mesh
def get_segmentation(obj_path, image_path, renderer):
input_image = cv2.imread(image_path)
input_image = input_image[:, :input_image.shape[1] // 3]
input_image = cv2.resize(input_image, (1024, 1024))
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Load obj file
verts_rgb_colors = get_verts_rgb_colors(obj_path)
verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device)
textures = TexturesVertex(verts_features=verts_rgb_colors)
# wo_textures = TexturesVertex(verts_features=torch.ones_like(verts_rgb_colors)*0.75)
# Load obj
mesh = load_objs_as_meshes([obj_path], device=device)
# Set mesh
vers = mesh._verts_list
faces = mesh._faces_list
mesh_w_tex = Meshes(vers, faces, textures)
mesh_wo_tex = Meshes(vers, faces, wo_textures)
R, T = look_at_view_transform(1.8, 0, 0, device=device)
images_w_tex = renderer(mesh_w_tex, R=R, T=T)
images_w_tex = np.clip(images_w_tex[0, ..., :3].cpu().numpy(), 0.0,
1.0)[:, :, ::-1] * 255
images_wo_tex = renderer(mesh_wo_tex, R=R, T=T)
images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0,
1.0)[:, :, ::-1] * 255
return input_image, images_w_tex, images_wo_tex
def project_mesh(mesh, angle):
start = time.time()
m = Metadata()
R, T = look_at_view_transform(1.75,
-45,
angle,
up=((0, 1, 0), ),
at=((0, -0.25, 0), ))
cameras = OpenGLPerspectiveCameras(device=m.device, R=R, T=T)
raster_settings = m.raster_settings
lights = m.lights
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardFlatShader(cameras=cameras,
device=m.device,
lights=lights))
verts = mesh.verts_list()[0]
# faces = meshes.faces_list()[0]
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
# verts_rgb = torch.ones((len(mesh.verts_list()[0]), 1))[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb.to(m.device))
mesh.textures = textures
mesh.textures._num_faces_per_mesh = mesh._num_faces_per_mesh.tolist()
mesh.textures._num_verts_per_mesh = mesh._num_verts_per_mesh.tolist()
image = renderer(mesh)
return image
def render_obj(verts, faces, distance, elevation, azimuth):
device = torch.device("cuda:0")
verts_rgb = torch.ones_like(verts)[None]
textures = Textures(verts_rgb=verts_rgb.to(device))
cur_mesh = Meshes(verts=[verts.to(device)],
faces=[faces.to(device)],
textures=textures)
cameras = OpenGLPerspectiveCameras(device=device)
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(image_size=256,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=PhongShader(device=device,
lights=lights))
R, T = look_at_view_transform(distance, elevation, azimuth, device=device)
return phong_renderer(meshes_world=cur_mesh, R=R, T=T).cpu().numpy()
def set_renderer(image_size=512, use_sfm=False):
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Initialize an OpenGL perspective camera.
R, T = look_at_view_transform(2.0, 0, 180)
if use_sfm:
cameras = SfMPerspectiveCameras(focal_length=580.0,
device=device,
R=R,
T=T)
else:
cameras = OpenGLOrthographicCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(image_size=image_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None,
max_faces_per_bin=None)
lights = PointLights(device=device, location=((2.0, 2.0, 2.0), ))
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
shader = HardPhongShader(device=device, cameras=cameras, lights=lights)
if use_sfm:
renderer = MeshRendererWithDepth(rasterizer=rasterizer, shader=shader)
else:
renderer = MeshRenderer(rasterizer=rasterizer, shader=shader)
return renderer
def render_mesh(verts, faces):
device = verts[0].get_device()
N = len(verts)
num_verts_per_mesh = []
for i in range(N):
num_verts_per_mesh.append(verts[i].shape[0])
verts_rgb = torch.ones((N, np.max(num_verts_per_mesh), 3),
requires_grad=False,
device=device)
for i in range(N):
verts_rgb[i, num_verts_per_mesh[i]:, :] = -1
textures = Textures(verts_rgb=verts_rgb)
meshes = Meshes(verts=verts, faces=faces, textures=textures)
elev = torch.rand(N) * 30 - 15
azim = torch.rand(N) * 360 - 180
R, T = look_at_view_transform(dist=2, elev=elev, azim=azim)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
sigma = 1e-4
raster_settings = RasterizationSettings(
image_size=128,
blur_radius=np.log(1. / 1e-4 - 1.) * sigma,
faces_per_pixel=40,
perspective_correct=False)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=SoftSilhouetteShader())
return renderer(meshes)
def test_load_apartment(self):
"""
This is the example habitat example scene from inside
http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes.zip
The scene is "already lit", i.e. the textures reflect the lighting
already, so we want to render them with full ambient light.
"""
self.skipTest("Data not available")
glb = DATA_DIR / "apartment_1.glb"
self.assertTrue(glb.is_file())
device = torch.device("cuda:0")
mesh = _load(glb, device=device)
if DEBUG:
texturesuv_image_PIL(mesh.textures).save(DATA_DIR /
"out_apartment.png")
for i in range(19):
# random locations in the apartment
eye = ((np.random.uniform(-6, 0.5), np.random.uniform(-8, 2), 0), )
at = ((np.random.uniform(-6, 0.5), np.random.uniform(-8, 2), 0), )
up = ((0, 0, -1), )
RT = look_at_view_transform(eye=eye, at=at, up=up)
_render(mesh, f"apartment_eau{i}", RT=RT, use_ambient=True)
for i in range(12):
# panning around the inner room from one location
pan = axis_angle_to_matrix(
torch.FloatTensor([0, radians(30 * i), 0]))
_render(mesh, f"apartment{i}", 1.0, -90, pan, use_ambient=True)
def generate_video_from_obj(obj_path, video_path, renderer):
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Load obj file
verts_rgb_colors = get_verts_rgb_colors(obj_path)
verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device)
textures = Textures(verts_rgb=verts_rgb_colors)
wo_textures = Textures(verts_rgb=torch.ones_like(verts_rgb_colors) * 0.75)
# Load obj
mesh = load_objs_as_meshes([obj_path], device=device)
# Set mesh
vers = mesh._verts_list
faces = mesh._faces_list
mesh_w_tex = Meshes(vers, faces, textures)
mesh_wo_tex = Meshes(vers, faces, wo_textures)
# create VideoWriter
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
out = cv2.VideoWriter(video_path, fourcc, 20.0, (1024, 512))
for i in tqdm(range(90)):
R, T = look_at_view_transform(1.8, 0, i * 4, device=device)
images_w_tex = renderer(mesh_w_tex, R=R, T=T)
images_w_tex = np.clip(images_w_tex[0, ..., :3].cpu().numpy(), 0.0,
1.0)[:, :, ::-1] * 255
images_wo_tex = renderer(mesh_wo_tex, R=R, T=T)
images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0,
1.0)[:, :, ::-1] * 255
image = np.concatenate([images_w_tex, images_wo_tex], axis=1)
out.write(image.astype('uint8'))
out.release()
def _get_renderer(self, device):
R, T = look_at_view_transform(10, 0, 0) # camera's position
cameras = FoVPerspectiveCameras(
device=device,
R=R,
T=T,
znear=0.01,
zfar=50,
fov=2 * np.arctan(self.img_size // 2 / self.focal) * 180. / np.pi)
lights = PointLights(device=device,
location=[[0.0, 0.0, 1e5]],
ambient_color=[[1, 1, 1]],
specular_color=[[0., 0., 0.]],
diffuse_color=[[0., 0., 0.]])
raster_settings = RasterizationSettings(
image_size=self.img_size,
blur_radius=0.0,
faces_per_pixel=1,
)
blend_params = blending.BlendParams(background_color=[0, 0, 0])
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=SoftPhongShader(device=device,
cameras=cameras,
lights=lights,
blend_params=blend_params))
return renderer
def setup(self, device):
R, T = look_at_view_transform(self.viewpoint_distance,
self.viewpoint_elevation,
self.viewpoint_azimuth,
device=device)
cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=self.opt.fast_image_size,
blur_radius=self.opt.raster_blur_radius,
faces_per_pixel=self.opt.raster_faces_per_pixel,
)
rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
lights = PointLights(device=device,
location=[self.opt.lights_location])
lights = DirectionalLights(device=device,
direction=[self.opt.lights_direction])
shader = SoftPhongShader(
device=device,
cameras=cameras,
lights=lights,
blend_params=BlendParams(
self.opt.blend_params_sigma,
self.opt.blend_params_gamma,
self.opt.blend_params_background_color,
),
)
self.renderer = MeshRenderer(
rasterizer=rasterizer,
shader=shader,
)
def __init__(self,
dir: str,
rasterization_settings: dict,
znear: float = 1.0,
zfar: float = 1000.0,
scale_min: float = 0.5,
scale_max: float = 2.0,
device: str = 'cuda'):
super(ToyNeuralGraphicsDataset, self).__init__()
device = torch.device(device)
self.device = device
self.scale_min = scale_min
self.scale_max = scale_max
self.scale_range = scale_max - scale_min
objs = [
os.path.join(dir, f) for f in os.listdir(dir) if f.endswith('.obj')
]
self.meshes = load_objs_as_meshes(objs, device=device)
R, T = look_at_view_transform(0, 0, 0)
self.cameras = FoVPerspectiveCameras(R=R,
T=T,
znear=znear,
zfar=zfar,
device=device)
self.renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=self.cameras,
raster_settings=RasterizationSettings(**rasterization_settings),
),
shader=HardFlatShader(
device=device,
cameras=self.cameras,
))
def render_with_batch_size(self, batch_size, dist, light_location,
output_path):
self.meshes = self.meshes.extend(batch_size)
self.batch_size = batch_size
elev = torch.linspace(0, 180, batch_size)
azim = torch.linspace(-180, 180, batch_size)
self.R, self.T = look_at_view_transform(dist=dist,
elev=elev,
azim=azim)
self.cameras = OpenGLPerspectiveCameras(device=self.device,
R=self.R,
T=self.T)
#set light locatioin
self.light_location = light_location
lights = PointLights(device=self.device,
location=[self.light_location])
# call pytorch3d mesh renderer with shong shader
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=self.cameras,
raster_settings=self.raster_settings),
shader=TexturedSoftPhongShader(device=self.device,
cameras=self.cameras,
lights=lights))
images = renderer(self.meshes, cameras=self.cameras, lights=lights)
for i in range(self.batch_size):
img = images[i, ..., :3].cpu().numpy() * 255
img = img.astype('uint8')
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
cv2.imwrite(output_path + 'render-image-' + str(i) + '.png', img)
def init_renderer(self):
# nsh_face_mesh = meshio.Mesh('data/mesh/nsh_bfm_face.obj')
# self.nsh_face_tri = torch.from_numpy(nsh_face_mesh.triangles).type(
# torch.int64).to(self.device)
R, T = look_at_view_transform(10, 0, 0)
cameras = OpenGLPerspectiveCameras(znear=0.001,
zfar=30.0,
aspect_ratio=1.0,
fov=12.5936,
degrees=True,
R=R,
T=T,
device=self.device)
raster_settings = RasterizationSettings(image_size=self.im_size,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0,
cull_backfaces=True)
self.rasterizer = MeshRasterizer(cameras=cameras,
raster_settings=raster_settings)
lights = DirectionalLights(device=self.device)
shader = TexturedSoftPhongShader(device=self.device,
cameras=cameras,
lights=lights)
self.renderer = MeshRenderer(rasterizer=self.rasterizer, shader=shader)
def create_renderer(self):
self.num_angles = self.config.num_angles
azim = torch.linspace(-1 * self.config.angle_range,
self.config.angle_range, self.num_angles)
R, T = look_at_view_transform(dist=1.0, elev=0, azim=azim)
T[:, 1] = -85
T[:, 2] = 200
cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T)
raster_settings = RasterizationSettings(
image_size=self.config.img_size,
blur_radius=0.0,
faces_per_pixel=1,
)
lights = PointLights(device=self.device, location=[[0.0, 85, 100.0]])
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=self.device,
cameras=cameras,
lights=lights))
return renderer
def render_rotating_volume(volume_model,
device,
n_frames=50,
video_size=400,
n_pts_per_ray=192):
renderer = get_renderer(video_size, n_pts_per_ray)
# Render frames.
with torch.inference_mode():
print("Generating rotating volume ...")
elev = 30
azimuths = torch.linspace(0., 360., n_frames, device=device)
frames = []
for azim in tqdm(azimuths):
R, T = look_at_view_transform(dist=args.camera_radius,
elev=elev,
azim=azim)
batch_cameras = FoVPerspectiveCameras(device=device, R=R, T=T)
rgbo = volume_model(batch_cameras, renderer)
rgb = rgbo[Ellipsis, :3]
opacity = rgbo[Ellipsis, 3:4]
frame = opacity * rgb + 1 - opacity
frame = frame.clamp(0.0, 1.0)
frames.append(frame)
frames = torch.cat(frames).clamp(0., 1.)
frames = frames.movedim(-1, 1) # THWC to TCHW.
return frames.cpu().numpy()
def sphere_examples(bsdf, device="cuda", size=256, chunk_size=128, scale=100):
from pytorch3d.pathtracer.shapes import Sphere
from pytorch3d.pathtracer.bsdf import Diffuse
from pytorch3d.pathtracer import pathtrace
from pytorch3d.renderer import (
look_at_view_transform,
OpenGLPerspectiveCameras,
PointLights,
)
import pytorch3d.pathtracer.integrators as integrators
sphere = Sphere([0, 0, 0], 1, device=device)
R, T = look_at_view_transform(dist=2., elev=0, azim=0)
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
lights = PointLights(device=device, location=[[0., 1., 4.]], scale=scale)
out = []
for basis in bsdf.bsdfs:
expected = pathtrace(
sphere,
cameras=cameras,
lights=lights,
chunk_size=chunk_size,
size=size,
bsdf=basis,
integrator=integrators.Direct(),
device=device,
silent=True,
)[0]
out.append(expected)
return out
def compute_sem_dis_loss(self, mesh, num_render, semantic_discriminator_net,
device):
# need to make sure this matches render settings for discriminator training set
# TODO: alternatively, randomize the angles each time?
# 0., 45., 90., 135., 180., 225., 270., 315.
azims = torch.linspace(0, 360, num_render + 1)[:-1]
elevs = torch.Tensor([25 for i in range(num_render)])
dists = torch.ones(num_render) * 1.7
R, T = look_at_view_transform(dists, elevs, azims)
meshes = mesh.extend(num_render)
renders = utils.render_mesh(meshes,
R,
T,
device,
img_size=224,
silhouette=True)
# converting from [num_render, 224, 224, 4] silhouette render (only channel 4 has info)
# to [num_render, 224, 224, 3] rgb image (black/white)
renders_binary_rgb = torch.unsqueeze(renders[..., 3], 3).repeat(1, 1, 1, 3)
loss = torch.sigmoid(
semantic_discriminator_net(renders_binary_rgb.permute(0, 3, 1, 2)))
loss = torch.mean(loss)
return loss, renders_binary_rgb
def translate_mesh_on_axis(
mesh, t: list, renderer, dist: float = 3.5, save: str = "", device: str = ""
):
translation = Transform3d(device=device).translate(t[0], t[1], t[2])
verts, faces = mesh.get_mesh_verts_faces(0)
verts = translation.transform_points(verts)
mesh = mesh.update_padded(verts.unsqueeze(0))
dist = dist
elev = 0
azim = 0
R, T = look_at_view_transform(dist=dist, elev=elev, azim=azim, device=device)
image_ref = renderer(meshes_world=mesh, R=R, T=T, device=device)
image_ref = image_ref.cpu().numpy()
plt.imshow(image_ref.squeeze())
plt.show()
if save:
verts, faces = mesh.get_mesh_verts_faces(0)
save_obj(save, verts, faces)
return mesh
def test_sampler():
heatmap = torch.zeros(1, 1, 5, 5)
heatmap[0, 0, 2, 2] = 1.0
from filter import sample_particles_from_heatmap_2d
hws, alphas = sample_particles_from_heatmap_2d(heatmap, {'cup': 1},
deterministic=True,
h_min=-4.0,
h_max=4.0,
w_min=-4.0,
w_max=4.0)
vert = torch.tensor(
[[-1, -1.1, 0], [1, -1.1, 0], [1, 1.1, 0], [-1, 1.1, 0]],
dtype=torch.float)
face = torch.LongTensor([[0, 1, 2], [0, 2, 3]])
white = torch.ones_like(vert)
from tracker import World
world = World()
world.add_mesh('cup', vert, face, white)
scene = world.create_scene(hws, alphas, 1)
print(hws['cup'])
print(scene.verts_padded().shape)
plt.scatter(scene.verts_padded()[0, :, 0], scene.verts_padded()[0, :, 1])
plt.show()
cameras = FoVOrthographicCameras(device=device,
max_x=2.5,
max_y=2.5,
min_x=-2.5,
min_y=-2.5,
scale_xyz=((1, 1, 1), ))
raster_settings = RasterizationSettings(
image_size=5,
blur_radius=0,
faces_per_pixel=6,
)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings,
),
shader=IdentityShader())
distance, elevation, azimuth = 30, 0.0, 0
R, T = look_at_view_transform(distance, elevation, azimuth, device=device)
image = renderer(meshes_world=scene.to(device), R=R, T=T)
fig = plt.figure(figsize=(10, 10))
for i in range(image.size(-2)):
panel = fig.add_subplot(3, 3, i + 1)
panel.imshow(image[..., i, 0:3].squeeze().cpu())
plt.grid(False)
plt.show()
def change_cameras(self, mode, camera_dist=2.2):
azim_train = torch.linspace(-1 * self.config.angle_range_train, self.config.angle_range_train, self.num_angles_train)
azim_test = torch.linspace(-1 * self.config.angle_range_test, self.config.angle_range_test, self.num_angles_test)
R, T = look_at_view_transform(camera_dist, 6, azim_train)
train_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T)
self.train_cameras = train_cameras
R, T = look_at_view_transform(camera_dist, 6, azim_test)
test_cameras = FoVPerspectiveCameras(device=self.device, R=R, T=T)
self.test_cameras = test_cameras
if mode == 'train':
self.renderer.rasterizer.cameras=self.train_cameras
self.renderer.shader.cameras=self.train_cameras
elif mode == 'test':
self.renderer.rasterizer.cameras=self.test_cameras
self.renderer.shader.cameras=self.test_cameras
def test_bounding_box():
verts = torch.tensor([[-2, -1, 0], [2, -1, 0], [2, 1, 0], [-2, 1, 0]],
dtype=torch.float) * 20.0
faces = torch.LongTensor([[0, 1, 2], [0, 2, 3]])
white = torch.ones_like(verts)
red = white * torch.tensor([1.0, 0.0, 0.0])
green = white * torch.tensor([0.0, 1.0, 0.0])
blue = white * torch.tensor([0.0, 0.0, 1.0])
meshes = Meshes(verts=[verts],
faces=[faces],
textures=TexturesVertex([blue]))
distance = 30
elevation = 0.0
azimuth = 0
R, T = look_at_view_transform(distance, elevation, azimuth)
cameras = FoVOrthographicCameras(max_x=64.0,
max_y=64.0,
min_x=-64.0,
min_y=-64.0,
scale_xyz=((1, 1, 1), ),
R=R,
T=T)
bb = BoundingBoxes(meshes, cameras, screen_size=(128, 128))
raster_settings = RasterizationSettings(
image_size=128,
blur_radius=0,
faces_per_pixel=6,
)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings,
),
shader=IdentityShader())
fig, ax = plt.subplots(ncols=1,
nrows=1,
figsize=(10, 10),
constrained_layout=False)
ax.imshow(renderer(meshes)[0, :, :, 0, :])
boxes_rect = patches.Rectangle(bb.bottom_left(0),
width=bb.width(0),
height=bb.height(0),
linewidth=4,
edgecolor='r',
facecolor='none')
ax.add_patch(boxes_rect)
plt.show()
def createRenderer(device, camera, light, imageSize):
'''
It creates a pytorch3D renderer with the given camera pose, light source
and output image size.
Parameters
----------
device :
Device on which the renderer is created.
camera :
Camera pose.
light :
Position of the light source.
imageSize :
The size of the rendered image.
Returns
-------
renderer :
Pytorch3D renderer.
'''
if camera is None:
camera = (2.0, -20.0, 180.0)
if light is None:
light = (0.0, 2.0, 0.0)
# Initialize an OpenGL perspective camera.
# With world coordinates +Y up, +X left and +Z into the screen.
R, T = look_at_view_transform(camera[0], camera[1], camera[2])
cameras = OpenGLPerspectiveCameras(device=device, R=R, T=T)
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that
# the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for
# explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of
# the difference between naive and coarse-to-fine rasterization.
raster_settings = RasterizationSettings(
image_size=imageSize,
blur_radius=0.0,
faces_per_pixel=1,
)
# Place a point light at -y direction.
lights = PointLights(device=device,
location=[[light[0], light[1], light[2]]])
# Create a phong renderer by composing a rasterizer and a shader.
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardPhongShader(device=device,
cameras=cameras,
lights=lights))
return renderer
def renderImage(self):
# Function: renderImage
# Inputs: self
# Process: returns image of the object
R, T = look_at_view_transform(self.camera_position[0], self.camera_position[1], self.camera_position[2])
image = self.renderer(meshes_world=self.mesh.clone(),R=R,T=T)
return image
def _render(self):
"""Render the unit cube at various camera poses.
We deliberately sample the camera poses such that all vertices
of the cube are always in view.
"""
opts = self.opts
# TODO(ycho): is `no_grad()` here necessary?
with th.no_grad():
# Sample a Unit-ray viewing direction.
ray = th.randn(size=(opts.batch_size, 3), device=self.device)
ray /= th.norm(ray, dim=1, keepdim=True)
# Compute distance along the ray according to constraints.
distance = (self.min_distance +
(opts.max_distance - self.min_distance) *
th.rand(size=(opts.batch_size, 1), device=self.device))
# NOTE(ycho): `sqrt(3)/2` here comes from max radius of unit cube.
# The generic thing to do would be to compute the radius of our
# cloud. We're not explicitly taking the cube into account.
max_tangential_offset = th.clamp(distance * self.tan_half_fov -
(0.5 * np.sqrt(3)),
min=0.0)
# NOTE(ycho): We're sampling an offset orthogonal to the view ray.
# The constraint here is to be visible within the view plane.
offset = th.randn(size=(opts.batch_size, 3), device=self.device)
offset = offset - th.einsum('...a,...a->...', offset,
ray)[:, None] * ray
offset *= (max_tangential_offset /
th.norm(offset, dim=1, keepdim=True))
# Finally, we have the full definition of the ray.
at = offset
pos = offset + ray * distance
# Compose the `lookat` camera transform according to this position
R, T = look_at_view_transform(eye=pos, at=at, device=self.device)
if self.opts.use_mesh:
# NOTE(ycho): Ignoring alpha dimension
img = self.renderer(self.meshes, R=R, T=T)[..., :3]
else:
img = self.renderer(point_clouds=self.clouds, R=R, T=T)
# pytorch3d uses `NHWC` convension, convert to NCHW.
img = img.permute(0, 3, 1, 2)
# pytorch3d uses a `float` tensor for representing images,
# whereas we'd like for the image to come out as `uint8` ONLY for
# compatibility with the output from the `Objectron` dataset.
img = img.mul_(255.0).to(dtype=th.uint8)
return (img, R, T)
def test_scene():
world = engine.World()
world.add_mesh('red_box', verts, faces, red)
world.add_mesh('green_box', verts, faces, green)
world.add_mesh('blue_box', verts, faces, blue)
scene_spec = [
{'red_box_0': 'red_box', 'green_box_0': 'green_box'},
{'blue_box_0': 'blue_box', 'blue_box_1': 'blue_box'}
]
world.create_scenes(scene_spec)
poses = [
[Translate(0, -30, 0), Translate(-10, -10, 0)],
[Translate(40, 0, 0), Translate(-10, -10, 0)]
]
world.update_scenes(poses)
batch = world.batch()
labels = world.labels()
distance = 30
elevation = 0.0
azimuth = 0
R, T = look_at_view_transform(distance, elevation, azimuth)
cameras = FoVOrthographicCameras(max_x=64.0, max_y=64.0,
min_x=-64.0, min_y=-64.0,
scale_xyz=((1, 1, 1),),
R=R, T=T)
raster_settings = RasterizationSettings(
image_size=128,
blur_radius=0,
faces_per_pixel=6,
)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=raster_settings,
),
shader=IdentityShader()
)
boxes = world.bounding_boxes(cameras, (128, 128))
image = renderer(batch)
fig, ax = plt.subplots(nrows=1, ncols=1)
ax.imshow(image[0, :, :, 0, :])
for box in boxes[0]:
ax.add_patch(box.get_patch())
plt.show()
def forward(self):
R, T = look_at_view_transform(dist=self.dist, elev=self.elev, azim=self.azim, at=self.p, up=self.u,
device=self.device)
predict_image = self.renderer(meshes_world=self.mesh.clone(), R=R, T=T)
ssim_loss = pytorch_ssim.SSIM(window_size=11)
predict_image = predict_image[..., :3]
loss = 1 - ssim_loss(self.gray_image_ref, predict_image)
return loss
