def generate_video_from_obj(obj_path, video_path, renderer):
# Setup
device = torch.device("cuda:0")
torch.cuda.set_device(device)
# Load obj file
verts_rgb_colors = get_verts_rgb_colors(obj_path)
verts_rgb_colors = torch.from_numpy(verts_rgb_colors).to(device)
textures = Textures(verts_rgb=verts_rgb_colors)
wo_textures = Textures(verts_rgb=torch.ones_like(verts_rgb_colors) * 0.75)
# Load obj
mesh = load_objs_as_meshes([obj_path], device=device)
# Set mesh
vers = mesh._verts_list
faces = mesh._faces_list
mesh_w_tex = Meshes(vers, faces, textures)
mesh_wo_tex = Meshes(vers, faces, wo_textures)
# create VideoWriter
fourcc = cv2.VideoWriter_fourcc(*'MP4V')
out = cv2.VideoWriter(video_path, fourcc, 20.0, (1024, 512))
for i in tqdm(range(90)):
R, T = look_at_view_transform(1.8, 0, i * 4, device=device)
images_w_tex = renderer(mesh_w_tex, R=R, T=T)
images_w_tex = np.clip(images_w_tex[0, ..., :3].cpu().numpy(), 0.0,
1.0)[:, :, ::-1] * 255
images_wo_tex = renderer(mesh_wo_tex, R=R, T=T)
images_wo_tex = np.clip(images_wo_tex[0, ..., :3].cpu().numpy(), 0.0,
1.0)[:, :, ::-1] * 255
image = np.concatenate([images_w_tex, images_wo_tex], axis=1)
out.write(image.astype('uint8'))
out.release()
def test_init_rgb_uv_fail(self):
V = 20
# Maps has wrong shape
with self.assertRaisesRegex(ValueError, 'maps'):
Textures(
maps=torch.ones((5, 16, 16, 3, 4)),
faces_uvs=torch.randint(size=(5, 10, 3), low=0, high=V),
verts_uvs=torch.ones((5, V, 2)),
)
# faces_uvs has wrong shape
with self.assertRaisesRegex(ValueError, 'faces_uvs'):
Textures(
maps=torch.ones((5, 16, 16, 3)),
faces_uvs=torch.randint(size=(5, 10, 3, 3), low=0, high=V),
verts_uvs=torch.ones((5, V, 2)),
)
# verts_uvs has wrong shape
with self.assertRaisesRegex(ValueError, 'verts_uvs'):
Textures(
maps=torch.ones((5, 16, 16, 3)),
faces_uvs=torch.randint(size=(5, 10, 3), low=0, high=V),
verts_uvs=torch.ones((5, V, 2, 3)),
)
# verts_rgb has wrong shape
with self.assertRaisesRegex(ValueError, 'verts_rgb'):
Textures(verts_rgb=torch.ones((5, 16, 16, 3)))
# maps provided without verts/faces uvs
with self.assertRaisesRegex(ValueError,
'faces_uvs and verts_uvs are required'):
Textures(maps=torch.ones((5, 16, 16, 3)))
def test_clone(self):
V = 20
tex = Textures(
maps=torch.ones((5, 16, 16, 3)),
faces_uvs=torch.randint(size=(5, 10, 3), low=0, high=V),
verts_uvs=torch.ones((5, V, 2)),
)
tex_cloned = tex.clone()
self.assertSeparate(tex._faces_uvs_padded, tex_cloned._faces_uvs_padded)
self.assertSeparate(tex._verts_uvs_padded, tex_cloned._verts_uvs_padded)
self.assertSeparate(tex._maps_padded, tex_cloned._maps_padded)
def test_to(self):
V = 20
tex = Textures(
maps=torch.ones((5, 16, 16, 3)),
faces_uvs=torch.randint(size=(5, 10, 3), low=0, high=V),
verts_uvs=torch.ones((5, V, 2)),
)
device = torch.device("cuda:0")
tex = tex.to(device)
self.assertTrue(tex._faces_uvs_padded.device == device)
self.assertTrue(tex._verts_uvs_padded.device == device)
self.assertTrue(tex._maps_padded.device == device)
def test_interpolate_attributes_grad(self):
verts = torch.randn((4, 3), dtype=torch.float32)
faces = torch.tensor([[2, 1, 0], [3, 1, 0]], dtype=torch.int64)
vert_tex = torch.tensor(
[[0, 1, 0], [0, 1, 1], [1, 1, 0], [1, 1, 1]],
dtype=torch.float32,
requires_grad=True,
)
tex = Textures(verts_rgb=vert_tex[None, :])
mesh = Meshes(verts=[verts], faces=[faces], textures=tex)
pix_to_face = torch.tensor([0, 1], dtype=torch.int64).view(1, 1, 1, 2)
barycentric_coords = torch.tensor([[0.5, 0.3, 0.2], [0.3, 0.6, 0.1]],
dtype=torch.float32).view(
1, 1, 1, 2, -1)
fragments = Fragments(
pix_to_face=pix_to_face,
bary_coords=barycentric_coords,
zbuf=torch.ones_like(pix_to_face),
dists=torch.ones_like(pix_to_face),
)
grad_vert_tex = torch.tensor(
[
[0.3, 0.3, 0.3],
[0.9, 0.9, 0.9],
[0.5, 0.5, 0.5],
[0.3, 0.3, 0.3],
],
dtype=torch.float32,
)
texels = interpolate_vertex_colors(fragments, mesh)
texels.sum().backward()
self.assertTrue(hasattr(vert_tex, "grad"))
self.assertTrue(torch.allclose(vert_tex.grad, grad_vert_tex[None, :]))
def load_objs_as_meshes(files: list, device=None, load_textures: bool = True):
"""
Load meshes from a list of .obj files using the load_obj function, and
return them as a Meshes object. This only works for meshes which have a
single texture image for the whole mesh. See the load_obj function for more
details. material_colors and normals are not stored.
Args:
f: A list of file-like objects (with methods read, readline, tell,
and seek), pathlib paths or strings containing file names.
device: Desired device of returned Meshes. Default:
uses the current device for the default tensor type.
load_textures: Boolean indicating whether material files are loaded
Returns:
New Meshes object.
"""
mesh_list = []
for f_obj in files:
# TODO: update this function to support the two texturing options.
verts, faces, aux = load_obj(f_obj, load_textures=load_textures)
verts = verts.to(device)
tex = None
tex_maps = aux.texture_images
if tex_maps is not None and len(tex_maps) > 0:
verts_uvs = aux.verts_uvs[None, ...].to(device) # (1, V, 2)
faces_uvs = faces.textures_idx[None, ...].to(device) # (1, F, 3)
image = list(tex_maps.values())[0].to(device)[None]
tex = Textures(verts_uvs=verts_uvs, faces_uvs=faces_uvs, maps=image)
mesh = Meshes(verts=[verts], faces=[faces.verts_idx.to(device)], textures=tex)
mesh_list.append(mesh)
if len(mesh_list) == 1:
return mesh_list[0]
return join_meshes_as_batch(mesh_list)
def project_mesh(mesh, angle):
start = time.time()
m = Metadata()
R, T = look_at_view_transform(1.75,
-45,
angle,
up=((0, 1, 0), ),
at=((0, -0.25, 0), ))
cameras = OpenGLPerspectiveCameras(device=m.device, R=R, T=T)
raster_settings = m.raster_settings
lights = m.lights
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=HardFlatShader(cameras=cameras,
device=m.device,
lights=lights))
verts = mesh.verts_list()[0]
# faces = meshes.faces_list()[0]
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
# verts_rgb = torch.ones((len(mesh.verts_list()[0]), 1))[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb.to(m.device))
mesh.textures = textures
mesh.textures._num_faces_per_mesh = mesh._num_faces_per_mesh.tolist()
mesh.textures._num_verts_per_mesh = mesh._num_verts_per_mesh.tolist()
image = renderer(mesh)
return image
def test_interpolate_attributes(self):
"""
This tests both interpolate_vertex_colors as well as
interpolate_face_attributes.
"""
verts = torch.randn((4, 3), dtype=torch.float32)
faces = torch.tensor([[2, 1, 0], [3, 1, 0]], dtype=torch.int64)
vert_tex = torch.tensor([[0, 1, 0], [0, 1, 1], [1, 1, 0], [1, 1, 1]],
dtype=torch.float32)
tex = Textures(verts_rgb=vert_tex[None, :])
mesh = Meshes(verts=[verts], faces=[faces], textures=tex)
pix_to_face = torch.tensor([0, 1], dtype=torch.int64).view(1, 1, 1, 2)
barycentric_coords = torch.tensor([[0.5, 0.3, 0.2], [0.3, 0.6, 0.1]],
dtype=torch.float32).view(
1, 1, 1, 2, -1)
expected_vals = torch.tensor([[0.5, 1.0, 0.3], [0.3, 1.0, 0.9]],
dtype=torch.float32).view(1, 1, 1, 2, -1)
fragments = Fragments(
pix_to_face=pix_to_face,
bary_coords=barycentric_coords,
zbuf=torch.ones_like(pix_to_face),
dists=torch.ones_like(pix_to_face),
)
texels = interpolate_vertex_colors(fragments, mesh)
self.assertTrue(torch.allclose(texels, expected_vals[None, :]))
def load(self, obj_filename):
# Load obj file
extension = obj_filename[-3:]
if extension == 'obj':
verts, faces, aux = load_obj(obj_filename)
verts_idx = faces.verts_idx
elif extension == 'ply':
verts, faces = load_ply(obj_filename)
verts_idx = faces
if os.path.exists(obj_filename[:-3]+'npy'):
colors = np.load(obj_filename[:-3]+'npy')
verts_rgb = torch.FloatTensor(colors[...,[2,1,0]])
verts_rgb = verts_rgb.unsqueeze(0)
verts_rgb = verts_rgb.to(self.device)
else:
# Initialize each vertex to be white in color - bgr
verts_rgb = torch.ones_like(verts)[None]
verts_rgb = verts_rgb.to(self.device)
#textures = Textures(faces_uvs=faces.textures_idx[None,...], verts_uvs=aux.verts_uvs[None,...], verts_rgb=verts_rgb.to(self.device))
# Create a Meshes object for the face.
self.face_mesh = Meshes(
verts = [verts.to(self.device)],
faces = [verts_idx.to(self.device)],
textures= Textures(verts_rgb=verts_rgb)
)
def render_shape(model, R, T, args, vertices):
verts, faces_idx, _ = load_obj(args.obj_path)
faces = faces_idx.verts_idx
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb)
# center = verts.mean(0)
# verts = verts - center
# scale = max(verts.abs().max(0)[0])
# verts = verts / scale
mesh = Meshes(verts=[verts], faces=[faces], textures=textures)
if args.lights:
images = []
for i in range(model.nviews):
if args.net_version == 1:
model.lights.location = model.camera_position[i]
else:
model.lights.location = vertices[i]
imgs = model.renderer(
meshes_world=mesh.to(device=args.device).clone(),
R=R[None, i],
T=T[None, i],
lights=model.lights).cpu().detach().numpy()
images.extend(imgs)
else:
# model.lights.location = model.light_position
meshes = mesh.extend(args.nviews)
images = model.renderer(meshes.to(device=args.device), R=R,
T=T) # , lights=model.lights)
images = images.detach().cpu().numpy()
return images
def render_obj(verts, faces, distance, elevation, azimuth):
device = torch.device("cuda:0")
verts_rgb = torch.ones_like(verts)[None]
textures = Textures(verts_rgb=verts_rgb.to(device))
cur_mesh = Meshes(verts=[verts.to(device)],
faces=[faces.to(device)],
textures=textures)
cameras = OpenGLPerspectiveCameras(device=device)
blend_params = BlendParams(sigma=1e-4, gamma=1e-4)
raster_settings = RasterizationSettings(image_size=256,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=0)
lights = PointLights(device=device, location=((2.0, 2.0, -2.0), ))
phong_renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=cameras, raster_settings=raster_settings),
shader=PhongShader(device=device,
lights=lights))
R, T = look_at_view_transform(distance, elevation, azimuth, device=device)
return phong_renderer(meshes_world=cur_mesh, R=R, T=T).cpu().numpy()
def project_to_image_plane(self, vertices, texture_map):
# self.renderer
if False: # hardcoded example
with torch.no_grad():
transform = transforms.Compose([
transforms.ToTensor(),
])
direc = Path('bareteeth.000001.26_C/minibatch_0_Netural_0')
tex = Image.open(direc / 'mesh.png')
texture_map = transform(tex).unsqueeze(0)
mesh = load_objs_as_meshes([direc / 'mesh.obj'],
device=self.device)
vertices = mesh.verts_padded()
# final_obj = os.path.join('out/', 'final_model.obj')
# import datetime
# now = datetime.datetime.now()
# final_obj = f'{self.save_dir}/web/images/{now.strftime("%Y-%m-%d_%H:%M:%S")}_fake_mesh.obj'
# final_obj = f'{self.save_dir}/web/images/{self.opt.epoch_count:03d}_fake_mesh.obj'
# save_obj(final_obj, vertices[0], torch.from_numpy(self.flamelayer.faces.astype(np.int32)))
self.estimated_texture_map = texture_map.permute(0, 2, 3, 1)
texture = Textures(self.estimated_texture_map,
faces_uvs=self.faces_uvs1,
verts_uvs=self.verts_uvs1)
self.estimated_mesh = make_mesh(vertices.squeeze(),
self.flamelayer.faces, False, texture)
# save_obj(final_obj, estimated_mesh.verts_packed(), torch.from_numpy(self.flamelayer.faces.astype(np.int32)),
# verts_uvs=estimated_mesh.textures.verts_uvs_packed(), texture_map=self.estimated_texture_map,
# faces_uvs=estimated_mesh.textures.faces_uvs_packed())
images = self.renderer(self.estimated_mesh, materials=self.materials)
silhouette_images = self.silhouette_renderer(
self.estimated_mesh, materials=self.materials)[..., 3].unsqueeze(0)
negative_silhouette_images = self.negative_silhouette_renderer(
self.estimated_mesh, materials=self.materials)[..., 3].unsqueeze(0)
if self.opt.verbose:
transforms.ToPILImage()(silhouette_images.squeeze().permute(
0, 1).cpu()).save('out/silhouette.png')
# transforms.ToPILImage()(images.squeeze().permute(2, 0, 1).cpu()).save('out/img.png')
cull_backfaces_mask = (
1 - (silhouette_images - negative_silhouette_images).abs())
img = (images[0][..., :3].detach().cpu().numpy() * 255).astype(
np.uint8)
if self.opt.verbose:
Image.fromarray(img).save('out/test1.png')
images = Normalize(images)
silhouette_images = silhouette_images.clamp(0, 1)
segmented_3d_model_image = self.segmentation_3d_renderer(
self.estimated_mesh)
# Image.fromarray(
# ((255 * segmentation_image[0, ..., :3]).squeeze().detach().cpu().numpy().astype(np.uint8))).save(
# str('out/segmentatino_texture.png')
# )
return images[..., :3].permute(
0, 3, 1, 2
), silhouette_images, cull_backfaces_mask, segmented_3d_model_image[
..., :3].permute(0, 3, 1, 2)
def render(self,
model_ids: Optional[List[str]] = None,
categories: Optional[List[str]] = None,
sample_nums: Optional[List[int]] = None,
idxs: Optional[List[int]] = None,
shader_type=HardPhongShader,
device="cpu",
**kwargs) -> torch.Tensor:
"""
If a list of model_ids are supplied, render all the objects by the given model_ids.
If no model_ids are supplied, but categories and sample_nums are specified, randomly
select a number of objects (number specified in sample_nums) in the given categories
and render these objects. If instead a list of idxs is specified, check if the idxs
are all valid and render models by the given idxs. Otherwise, randomly select a number
(first number in sample_nums, default is set to be 1) of models from the loaded dataset
and render these models.
Args:
model_ids: List[str] of model_ids of models intended to be rendered.
categories: List[str] of categories intended to be rendered. categories
and sample_nums must be specified at the same time. categories can be given
in the form of synset offsets or labels, or a combination of both.
sample_nums: List[int] of number of models to be randomly sampled from
each category. Could also contain one single integer, in which case it
will be broadcasted for every category.
idxs: List[int] of indices of models to be rendered in the dataset.
shader_type: Select shading. Valid options include HardPhongShader (default),
SoftPhongShader, HardGouraudShader, SoftGouraudShader, HardFlatShader,
SoftSilhouetteShader.
device: torch.device on which the tensors should be located.
**kwargs: Accepts any of the kwargs that the renderer supports.
Returns:
Batch of rendered images of shape (N, H, W, 3).
"""
paths = self._handle_render_inputs(model_ids, categories, sample_nums,
idxs)
meshes = load_objs_as_meshes(paths, device=device, load_textures=False)
meshes.textures = Textures(
verts_rgb=torch.ones_like(meshes.verts_padded(), device=device))
cameras = kwargs.get("cameras", OpenGLPerspectiveCameras()).to(device)
renderer = MeshRenderer(
rasterizer=MeshRasterizer(
cameras=cameras,
raster_settings=kwargs.get("raster_settings",
RasterizationSettings()),
),
shader=shader_type(
device=device,
cameras=cameras,
lights=kwargs.get("lights", PointLights()).to(device),
),
)
return renderer(meshes)
def load_mesh_from_obj(file, device):
verts, faces_idx, _ = load_obj(file)
faces = faces_idx.verts_idx
# Initialize each vertex to be white in color.
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb.to(device))
# Create a Meshes object for the teapot. Here we have only one mesh in the batch.
teapot_mesh = Meshes(verts=[verts.to(device)],
faces=[faces.to(device)],
textures=textures)
return teapot_mesh
def make_mesh(flamelayer, detach):
device = torch.device("cuda:0")
verts, _, _ = flamelayer()
if detach:
verts = verts.detach()
# Initialize each vertex to be white in color.
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb.to(device))
faces = torch.tensor(np.int32(flamelayer.faces), dtype=torch.long).cuda()
return Meshes(verts=[verts.to(device)],
faces=[faces.to(device)],
textures=textures)
def rendering(self, light_params, coeffs, vertices, gen_uvmaps,
face_model):
ambient_color = torch.clamp(0.5 + 0.5 * light_params[:, 0:3], 0, 1)
diffuse_color = torch.clamp(0.5 + 0.5 * light_params[:, 3:6], 0, 1)
specular_color = torch.clamp(0.2 + 0.2 * light_params[:, 6:9], 0, 1)
direction = light_params[:, 9:12]
directions = torch.cat([
direction, direction *
torch.tensor([[-1, 1, 1]], dtype=torch.float, device=self.device)
],
dim=0)
lights = DirectionalLights(ambient_color=ambient_color.repeat(2, 1),
diffuse_color=diffuse_color.repeat(2, 1),
specular_color=specular_color.repeat(2, 1),
direction=directions,
device=self.device)
self.renderer.shader.lights = lights
_, _, _, angles, _, trans = utils.split_bfm09_coeff(coeffs)
reflect_angles = torch.cat([
angles, angles *
torch.tensor([[1, -1, -1]], dtype=torch.float, device=self.device)
],
dim=0)
reflect_trans = torch.cat([
trans, trans *
torch.tensor([[-1, 1, 1]], dtype=torch.float, device=self.device)
],
dim=0)
rotated_vert = self.rotate_vert(vertices.repeat(2, 1, 1),
reflect_angles, reflect_trans)
fliped_uv = torch.flip(gen_uvmaps / 2 + 0.5,
(2, 3)).repeat(2, 1, 1, 1).permute(0, 2, 3, 1)
texture = Textures(
maps=fliped_uv,
faces_uvs=self.meshes[face_model].textures.faces_uvs_padded(),
verts_uvs=self.meshes[face_model].textures.verts_uvs_padded())
meshes = Meshes(rotated_vert, self.meshes[face_model].faces_padded(),
texture)
renders = self.renderer(meshes)
renders[..., :3] = renders[..., :3] * 2 - 1
renders[..., -1] = (renders[..., -1] > 0).float()
renders = renders.permute(0, 3, 1, 2).contiguous()
return renders
def get_template_texture(vertices: torch.tensor, faces: torch.tensor,
texture_map: torch.tensor):
device = vertices.device
verts_uv = convert_3d_to_uv_coordinates(vertices)
vertex_rgb = torch.nn.functional.grid_sample(
texture_map.unsqueeze(0), 2 * verts_uv.unsqueeze(0).unsqueeze(0) - 1)
vertex_rgb = vertex_rgb.squeeze(2).permute(0, 2, 1) * 255
texture = Textures([texture_map.cpu().permute(1, 2, 0)],
faces_uvs=faces.unsqueeze(0),
verts_uvs=verts_uv.unsqueeze(0),
verts_rgb=vertex_rgb).to(device)
return texture
def make_mesh(verts: torch.tensor,
faces: np.ndarray,
detach: bool,
textures=None) -> Meshes:
device = torch.device("cuda:0")
if detach:
verts = verts.detach()
# Initialize each vertex to be white in color.
if textures is None:
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb.to(device))
faces = torch.tensor(np.int32(faces), dtype=torch.long).cuda()
# return Meshes(
# verts=[verts.to(device)],
# faces=[faces.to(device)],
# textures=textures
# )
return Meshes(verts=verts.to(device),
faces=faces.to(device).repeat(verts.shape[0], 1, 1),
textures=textures.to(device))
def test_getitem(self):
N = 5
V = 20
source = {
"maps": torch.rand(size=(N, 16, 16, 3)),
"faces_uvs": torch.randint(size=(N, 10, 3), low=0, high=V),
"verts_uvs": torch.rand((N, V, 2)),
}
tex = Textures(
maps=source["maps"],
faces_uvs=source["faces_uvs"],
verts_uvs=source["verts_uvs"],
)
verts = torch.rand(size=(N, V, 3))
faces = torch.randint(size=(N, 10, 3), high=V)
meshes = Meshes(verts=verts, faces=faces, textures=tex)
def tryindex(index):
tex2 = tex[index]
meshes2 = meshes[index]
tex_from_meshes = meshes2.textures
for item in source:
basic = source[item][index]
from_texture = getattr(tex2, item + "_padded")()
from_meshes = getattr(tex_from_meshes, item + "_padded")()
if isinstance(index, int):
basic = basic[None]
self.assertClose(basic, from_texture)
self.assertClose(basic, from_meshes)
self.assertEqual(
from_texture.ndim, getattr(tex, item + "_padded")().ndim
)
if item == "faces_uvs":
faces_uvs_list = tex_from_meshes.faces_uvs_list()
self.assertEqual(basic.shape[0], len(faces_uvs_list))
for i, faces_uvs in enumerate(faces_uvs_list):
self.assertClose(faces_uvs, basic[i])
tryindex(2)
tryindex(slice(0, 2, 1))
index = torch.tensor([1, 0, 1, 0, 0], dtype=torch.bool)
tryindex(index)
index = torch.tensor([0, 0, 0, 0, 0], dtype=torch.bool)
tryindex(index)
index = torch.tensor([1, 2], dtype=torch.int64)
tryindex(index)
tryindex([2, 4])
def load_texture(aux, faces):
if aux.verts_uvs is None:
return None
device = torch.device(config.cuda.device)
vertices_uvs = aux.verts_uvs[None, ...].to(device)
faces_uvs = faces.textures_idx[None, ...].to(device)
texture_maps = aux.texture_images
texture_maps = list(texture_maps.values())[0]
texture_maps = texture_maps[None, ...].to(device)
return Textures(verts_uvs=vertices_uvs,
faces_uvs=faces_uvs,
maps=texture_maps)
def forward(self,
shape_params=None,
expression_params=None,
pose_params=None,
neck_pose=None,
eye_pose=None,
transl=None,
texture_params=None):
vertices, landmarks = super(TexturedFLAME,
self).forward(shape_params,
expression_params,
pose_params, neck_pose,
eye_pose, transl)
texture = torch.reshape(
torch.add(self.texture_mean,
torch.matmul(self.texture_params, self.texture_dir)),
self.texture_shape)
texture = texture.clamp(0.0, 255.0)
texture = texture / 255.0
texture = torch.cat(self.batch_size * [texture.unsqueeze(0)])
textures = Textures(maps=texture,
faces_uvs=self.faces_uvs,
verts_uvs=self.verts_uvs)
meshes = Meshes(
vertices,
torch.cat(vertices.shape[0] * [self.faces_tensor.unsqueeze(0)]),
textures)
images = bgr_to_rgb(
self.renderer(meshes)[..., :3].permute(0, -1, 1, 2))
# images = self.renderer(meshes)[..., :3].permute(0, -1, 1, 2)
landmarks = self.transform_points(landmarks)
landmarks[:, :, 0] *= -1
landmarks[:, :, 1] *= -1
for bi in range(landmarks.shape[0]):
for pi in range(landmarks.shape[1]):
landmarks[bi, pi,
0] = self._ndc_to_pix(landmarks[bi, pi, 0],
self.crop_size)
landmarks[bi, pi,
1] = self._ndc_to_pix(landmarks[bi, pi, 1],
self.crop_size)
landmarks = landmarks[:, :, :2] # x y only
return vertices, landmarks, images
def load_moon_mesh(_obj_filename):
# Load the object
verts, faces, aux = load_obj(_obj_filename)
faces_idx = faces.verts_idx.to(DEVICE)
verts = verts.to(DEVICE)
verts_uvs = aux.verts_uvs[None, ...].to(DEVICE)
faces_uvs = faces.textures_idx[None, ...].to(DEVICE)
tex_maps = aux.texture_images
texture_image = list(tex_maps.values())[0]
texture_image = texture_image[None, ...].to(DEVICE)
tex = Textures(verts_uvs=verts_uvs,
faces_uvs=faces_uvs,
maps=texture_image)
moon_mesh = Meshes(verts=[verts], faces=[faces_idx], textures=tex)
return moon_mesh
def set_moon_mesh(self):
vertices, faces, aux = load_obj(OBJECT_PATH)
faces_idx = faces.verts_idx.to(self.cuda_device)
vertices = vertices.to(self.cuda_device)
verts_uvs = aux.verts_uvs[None, ...].to(self.cuda_device)
faces_uvs = faces.textures_idx[None, ...].to(self.cuda_device)
tex_maps = aux.texture_images
texture_image = list(tex_maps.values())[0]
texture_image = texture_image[None, ...].to(self.cuda_device)
tex = Textures(verts_uvs=verts_uvs,
faces_uvs=faces_uvs,
maps=texture_image)
self.moon_mesh = Meshes(verts=[vertices],
faces=[faces_idx],
textures=tex)
def test_interpolate_texture_map(self):
barycentric_coords = torch.tensor(
[[0.5, 0.3, 0.2], [0.3, 0.6, 0.1]], dtype=torch.float32
).view(1, 1, 1, 2, -1)
dummy_verts = torch.zeros(4, 3)
vert_uvs = torch.tensor(
[[1, 0], [0, 1], [1, 1], [0, 0]], dtype=torch.float32
)
face_uvs = torch.tensor([[0, 1, 2], [1, 2, 3]], dtype=torch.int64)
interpolated_uvs = torch.tensor(
[[0.5 + 0.2, 0.3 + 0.2], [0.6, 0.3 + 0.6]], dtype=torch.float32
)
# Create a dummy texture map
H = 2
W = 2
x = torch.linspace(0, 1, W).view(1, W).expand(H, W)
y = torch.linspace(0, 1, H).view(H, 1).expand(H, W)
tex_map = torch.stack([x, y], dim=2).view(1, H, W, 2)
pix_to_face = torch.tensor([0, 1], dtype=torch.int64).view(1, 1, 1, 2)
fragments = Fragments(
pix_to_face=pix_to_face,
bary_coords=barycentric_coords,
zbuf=pix_to_face,
dists=pix_to_face,
)
tex = Textures(
maps=tex_map,
faces_uvs=face_uvs[None, ...],
verts_uvs=vert_uvs[None, ...],
)
meshes = Meshes(verts=[dummy_verts], faces=[face_uvs], textures=tex)
texels = interpolate_texture_map(fragments, meshes)
# Expected output
pixel_uvs = interpolated_uvs * 2.0 - 1.0
pixel_uvs = pixel_uvs.view(2, 1, 1, 2)
tex_map = torch.flip(tex_map, [1])
tex_map = tex_map.permute(0, 3, 1, 2)
tex_map = torch.cat([tex_map, tex_map], dim=0)
expected_out = F.grid_sample(tex_map, pixel_uvs, align_corners=False)
self.assertTrue(
torch.allclose(texels.squeeze(), expected_out.squeeze())
)
def test_extend(self):
B = 10
mesh = TestMeshes.init_mesh(B, 30, 50)
V = mesh._V
F = mesh._F
tex = Textures(
maps=torch.randn((B, 16, 16, 3)),
faces_uvs=torch.randint(size=(B, F, 3), low=0, high=V),
verts_uvs=torch.randn((B, V, 2)),
)
tex_mesh = Meshes(
verts=mesh.verts_padded(), faces=mesh.faces_padded(), textures=tex
)
N = 20
new_mesh = tex_mesh.extend(N)
self.assertEqual(len(tex_mesh) * N, len(new_mesh))
tex_init = tex_mesh.textures
new_tex = new_mesh.textures
for i in range(len(tex_mesh)):
for n in range(N):
self.assertClose(
tex_init.faces_uvs_list()[i],
new_tex.faces_uvs_list()[i * N + n],
)
self.assertClose(
tex_init.verts_uvs_list()[i],
new_tex.verts_uvs_list()[i * N + n],
)
self.assertAllSeparate(
[
tex_init.faces_uvs_padded(),
new_tex.faces_uvs_padded(),
tex_init.verts_uvs_padded(),
new_tex.verts_uvs_padded(),
tex_init.maps_padded(),
new_tex.maps_padded(),
]
)
with self.assertRaises(ValueError):
tex_mesh.extend(N=-1)
def __getitem__(self, index):
path = self.paths[index][0]
label = self.paths[index][1]
# Load the obj and ignore the textures and materials.
verts, faces_idx, _ = load_obj(path)
faces = faces_idx.verts_idx
# center = verts.mean(0)
# verts = verts - center
# scale = max(verts.abs().max(0)[0])
# verts = verts / scale
# Initialize each vertex to be white in color.
verts_rgb = torch.ones_like(verts)[None] # (1, V, 3)
textures = Textures(verts_rgb=verts_rgb)
# Create a Meshes object for the teapot. Here we have only one mesh in the batch.
mesh = Meshes(verts=[verts], faces=[faces], textures=textures)
return mesh, label
def load_mesh(obj_path):
device = torch.device('cuda')
vertices, faces, aux = load_obj(obj_path)
vertices_uvs = aux.verts_uvs[None, ...].to(device)
faces_uvs = faces.textures_idx[None, ...].to(device)
texture_maps = aux.texture_images
texture_maps = list(texture_maps.values())[0]
texture_maps = texture_maps[None, ...].to(device)
textures = Textures(
verts_uvs=vertices_uvs,
faces_uvs=faces_uvs,
maps=texture_maps,
)
vertices = vertices.to(device)
faces = faces.verts_idx.to(device)
mesh = Meshes(verts=[vertices], faces=[faces], textures=textures)
return mesh
def render_images(vertices, faces, texture, faces_uvs, verts_uvs, crop_size,
device):
textures = Textures(maps=texture, faces_uvs=faces_uvs, verts_uvs=verts_uvs)
meshes = Meshes(vertices, faces, textures)
R, T = look_at_view_transform(1.0, 0.5, 0, device=device)
camera = OpenGLPerspectiveCameras(R=R, T=T, fov=20, device=device)
raster_settings = RasterizationSettings(image_size=crop_size * 2,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=None,
max_faces_per_bin=None)
lights = PointLights(location=[[0.0, 0.0, -3.0]], device=device)
renderer = MeshRenderer(rasterizer=MeshRasterizer(
cameras=camera, raster_settings=raster_settings),
shader=TexturedSoftPhongShader(cameras=camera,
device=device,
lights=lights))
images = renderer(meshes)
return images
def get_mesh_from_model_single(self,
model,
basis,
image_folder=config.test_align,
outdir=config.test_obj):
print("Extract param...")
# f_model = np.load(config.f_model_path)
# mu_shape = f_model['mu']
# w_shape = f_model['w_shape']
# index = f_model['faces']
types = ('*.jpg', '*.png')
image_path_list = []
for files in types:
image_path_list.extend(glob(os.path.join(image_folder, files)))
total_num = len(image_path_list)
print(total_num)
# total = 0
# count = np.zeros(1000)
#landmarks_map = np.load('landmark.npy')
for i, image_path in enumerate(image_path_list):
image = Image.open(image_path)
data = config.transform_eval_fs(image)
data = data.unsqueeze(0).cuda()
_, pred_shape, _, pred_camera_exp = model(data)
param = torch.cat((pred_shape, pred_camera_exp), dim=1)
shape_para = pred_shape[:, 0:199]
exp_para = pred_camera_exp[:, 7:36]
camera_para = pred_camera_exp[:, 0:7]
# print(pred_camera_exp[:, 4:7])
exp_face, rotated, scaled = basis.module(pred_shape, exp_para,
camera_para)
# print(scaled)
scaled_raw = scaled.clone().reshape(3, -1).t()
scaled_raw[:, 0] = scaled[:, 0] - 56
scaled_raw[:, 1] = 56 - scaled[:, 1]
# print(scaled_raw)
pred_face = basis.module.get_shape_obj(shape_para)
#exp_face = basis.module.get_exp_obj(shape_para,exp_para)
face_shape = pred_face.squeeze().reshape(-1,
3) #.data.cpu().numpy()
exp_face = exp_face.squeeze().reshape(-1, 3).data.cpu().numpy()
rotated = rotated.squeeze().reshape(3, -1).t()
# scaled_mesh = scaled.squeeze().t()
scaled = scaled.squeeze()[:2, :].t().unsqueeze(0)
scaled[:, :, 0] = scaled[:, :, 0] / 96
scaled[:, :, 1] = 1 - scaled[:, :, 1] / 112
faces = torch.Tensor(config.index.astype(int))
tex_rgb = torch.ones_like(rotated).unsqueeze(0)
texture_img = config.transform_raw_img(image).unsqueeze(
0).transpose(1, 3).transpose(2, 1)
#print(scaled_raw)
textures = Textures(maps=texture_img,
verts_uvs=scaled,
faces_uvs=faces.long().unsqueeze(0))
mesh = Meshes(verts=[face_shape], faces=[faces],
textures=textures).to(config.device)
normals = mesh.verts_normals_padded()
print(normals)
##########################################################################
# Initialize an OpenGL perspective camera.
# With world coordinates +Y up, +X left and +Z in, the front of the cow is facing the -Z direction.
# So we move the camera by 180 in the azimuth direction so it is facing the front of the cow.
#print(rotated)
R, T = look_at_view_transform(100, 0, 0)
#print(R,T)
# principal_point = torch.Tensor[1.0/56,1.0/56]
cameras = SfMOrthographicCameras(device=config.device,
focal_length=1.0 / 56,
R=R,
T=T)
#cameras = SfMOrthographicCameras(device=config.device,focal_length=1.0/56,R=R, T=T)
# Define the settings for rasterization and shading. Here we set the output image to be of size
# 512x512. As we are rendering images for visualization purposes only we will set faces_per_pixel=1
# and blur_radius=0.0. We also set bin_size and max_faces_per_bin to None which ensure that
# the faster coarse-to-fine rasterization method is used. Refer to rasterize_meshes.py for
# explanations of these parameters. Refer to docs/notes/renderer.md for an explanation of
# the difference between naive and coarse-to-fine rasterization.
raster_settings = RasterizationSettings(
image_size=112,
blur_radius=0.0,
faces_per_pixel=1,
bin_size=
None, # this setting controls whether naive or coarse-to-fine rasterization is used
max_faces_per_bin=
None # this setting is for coarse rasterization
)
# Place a point light in front of the object. As mentioned above, the front of the cow is facing the
# -z direction.
lights = PointLights(device=config.device,
location=[[0.0, 0.0, 3.0]])
# Create a phong renderer by composing a rasterizer and a shader. The textured phong shader will
# interpolate the texture uv coordinates for each vertex, sample from a texture image and
# apply the Phong lighting model
renderer = MeshRenderer(
rasterizer=MeshRasterizer(cameras=cameras,
raster_settings=raster_settings),
shader=TexturedSoftPhongShader(device=config.device,
cameras=cameras,
lights=None))
# ## 3. Render the mesh
# The light is in front of the object so it is bright and the image has specular highlights.
##########################################################################
# pose_exp = pose_exp.squeeze().data.cpu().numpy()
# R = Q2R(pose_exp[0:4])
# res.append((param.data.cpu().numpy(),imname))
# norm.append(torch.norm(param,dim=1).mean().item())
# self.vertices2obj(vertex2.data.cpu().numpy().reshape(len(vertex2),-1,3),target.data.cpu().numpy(), imname=imname)
# face_shape = (mu_shape + param @ w_shape).reshape(-1,3)
#filepath = image_path.replace('\n',"")
#identify = int(filepath.split('/')[-2])
rotated = rotated.data.cpu().numpy()
filename = image_path.split('/')[-1][:-3] + "obj"
filename_exp = image_path.split('/')[-1][:-4] + "_e.obj"
filename_all = image_path.split('/')[-1][:-4] + "_a.obj"
filename_render = image_path.split('/')[-1][:-4] + ".png"
#outdir = os.path.join(out_root,str(identify))
# if not os.path.exists(outdir):
# os.mkdir(outdir)
filename = os.path.join(outdir, filename)
filename_exp = os.path.join(outdir, filename_exp)
filename_all = os.path.join(outdir, filename_all)
filename_render = os.path.join(outdir, filename_render)
images = renderer(mesh)
#print(images)
images = images[0, :112, :96, :]
mask = images[..., 3].unsqueeze(-1)
#mask
out_img = torch.where(mask == 0, texture_img[0].to(config.device),
images[..., :3])
#print(texture_img,out_img.shape)
plt.figure(figsize=(10, 10))
#print("saving figure!")
plt.imsave(filename_render, out_img[..., :3].data.cpu().numpy())
plt.grid("off")
plt.axis("off")
#print(face_shape)
# self.write_obj(filename,face_shape)
# self.write_obj(filename_exp,exp_face)
self.write_obj(filename_all, rotated)
def test_join_meshes(self):
"""
Test that join_meshes and load_objs_as_meshes are consistent with single
meshes.
"""
def check_triple(mesh, mesh3):
"""
Verify that mesh3 is three copies of mesh.
"""
def check_item(x, y):
self.assertEqual(x is None, y is None)
if x is not None:
self.assertClose(torch.cat([x, x, x]), y)
check_item(mesh.verts_padded(), mesh3.verts_padded())
check_item(mesh.faces_padded(), mesh3.faces_padded())
if mesh.textures is not None:
check_item(mesh.textures.maps_padded(),
mesh3.textures.maps_padded())
check_item(
mesh.textures.faces_uvs_padded(),
mesh3.textures.faces_uvs_padded(),
)
check_item(
mesh.textures.verts_uvs_padded(),
mesh3.textures.verts_uvs_padded(),
)
check_item(
mesh.textures.verts_rgb_padded(),
mesh3.textures.verts_rgb_padded(),
)
DATA_DIR = (Path(__file__).resolve().parent.parent /
"docs/tutorials/data")
obj_filename = DATA_DIR / "cow_mesh/cow.obj"
mesh = load_objs_as_meshes([obj_filename])
mesh3 = load_objs_as_meshes([obj_filename, obj_filename, obj_filename])
check_triple(mesh, mesh3)
self.assertTupleEqual(mesh.textures.maps_padded().shape,
(1, 1024, 1024, 3))
mesh_notex = load_objs_as_meshes([obj_filename], load_textures=False)
mesh3_notex = load_objs_as_meshes(
[obj_filename, obj_filename, obj_filename], load_textures=False)
check_triple(mesh_notex, mesh3_notex)
self.assertIsNone(mesh_notex.textures)
verts = torch.randn((4, 3), dtype=torch.float32)
faces = torch.tensor([[2, 1, 0], [3, 1, 0]], dtype=torch.int64)
vert_tex = torch.tensor([[0, 1, 0], [0, 1, 1], [1, 1, 0], [1, 1, 1]],
dtype=torch.float32)
tex = Textures(verts_rgb=vert_tex[None, :])
mesh_rgb = Meshes(verts=[verts], faces=[faces], textures=tex)
mesh_rgb3 = join_meshes([mesh_rgb, mesh_rgb, mesh_rgb])
check_triple(mesh_rgb, mesh_rgb3)
teapot_obj = DATA_DIR / "teapot.obj"
mesh_teapot = load_objs_as_meshes([teapot_obj])
teapot_verts, teapot_faces = mesh_teapot.get_mesh_verts_faces(0)
mix_mesh = load_objs_as_meshes([obj_filename, teapot_obj],
load_textures=False)
self.assertEqual(len(mix_mesh), 2)
self.assertClose(mix_mesh.verts_list()[0], mesh.verts_list()[0])
self.assertClose(mix_mesh.faces_list()[0], mesh.faces_list()[0])
self.assertClose(mix_mesh.verts_list()[1], teapot_verts)
self.assertClose(mix_mesh.faces_list()[1], teapot_faces)
cow3_tea = join_meshes([mesh3, mesh_teapot], include_textures=False)
self.assertEqual(len(cow3_tea), 4)
check_triple(mesh_notex, cow3_tea[:3])
self.assertClose(cow3_tea.verts_list()[3], mesh_teapot.verts_list()[0])
self.assertClose(cow3_tea.faces_list()[3], mesh_teapot.faces_list()[0])
