def render(cls,
mesh: TriangleMesh,
dist,
elev,
azim,
uv=None,
texture=None,
img_size=128):
renderer = DIBRenderer(img_size, img_size, mode='Phong')
cls.check_mesh_parameters(mesh, uv, texture)
dist, elev, azim = cls.check_camera_parameters(dist, elev, azim)
renderer.set_look_at_parameters([azim], [elev], [dist])
vertices = mesh.vertices.clone().to(DEVICE)[None]
faces = mesh.faces.clone().to(DEVICE)
if uv is None and texture is None:
rand_color = cls.get_random_color(vertices.size(1))
uv, texture = rand_color[0], rand_color[1]
render_imgs, render_alphas, face_norms = renderer.forward(
points=[vertices, faces],
uv_bxpx2=uv,
texture_bx3xthxtw=texture,
lightdirect_bx3=light,
material_bx3x3=material,
shininess_bx1=shininess)
return render_imgs, render_alphas, face_norms
def main():
args = parse_arguments()
###########################
# Load mesh
###########################
mesh = TriangleMesh.from_obj(args.mesh)
vertices = mesh.vertices
faces = mesh.faces.int()
# Expand such that batch size = 1
vertices = vertices[None, :, :].cuda()
faces = faces[None, :, :].cuda()
###########################
# Normalize mesh position
###########################
vertices_max = vertices.max()
vertices_min = vertices.min()
vertices_middle = (vertices_max + vertices_min) / 2.
vertices = (vertices - vertices_middle) * MESH_SIZE
###########################
# Generate vertex color
###########################
vert_min = torch.min(vertices, dim=1, keepdims=True)[0]
vert_max = torch.max(vertices, dim=1, keepdims=True)[0]
colors = (vertices - vert_min) / (vert_max - vert_min)
###########################
# Render
###########################
renderer = Renderer(HEIGHT, WIDTH, mode='VertexColor')
loop = tqdm.tqdm(list(range(0, 360, 4)))
loop.set_description('Drawing')
os.makedirs(args.output_path, exist_ok=True)
writer = imageio.get_writer(os.path.join(args.output_path, 'example.gif'), mode='I')
for azimuth in loop:
renderer.set_look_at_parameters([90 - azimuth],
[CAMERA_ELEVATION],
[CAMERA_DISTANCE])
predictions, _, _ = renderer(points=[vertices, faces[0].long()], colors=[colors])
image = predictions.detach().cpu().numpy()[0]
writer.append_data((image * 255).astype(np.uint8))
writer.close()
def __init__(self, N=6, f_dim=256, point_num=1024):
super(Renderer, self).__init__()
'''
初始化参数:
'''
self.N = N
self.f_dim = f_dim
self.point_num = point_num
# DIB渲染器
# self.renderer = DIBRenderer(height=640, width=400, mode='Lambertian',camera_fov_y=66.96 * np.pi / 180.0)
self.renderer = DIBRenderer(height=256,
width=256,
mode='Lambertian',
camera_fov_y=66.96 * np.pi / 180.0)
self.renderer.set_look_at_parameters([0], [0], [0],
fovx=57.77316 * np.pi / 180.0,
fovy=44.95887 * np.pi / 180.0,
near=0.01,
far=10.0)
# 设置相机参数
# self.renderer.set_camera_parameters()
# 预定义相机外参
# 真实参数
# 相机全部往中心移动
# x = 0.597105
# y = 1.062068
# z = 1.111316
# x = -0.3
# y = 1.2
# z = 0.1
x = 0
y = 0
z = 0
self.cam_mat = np.array(
[[[0.57432211, 0.77105488, 0.27500633],
[-0.56542476, 0.13069975, 0.81437854],
[0.59198729, -0.62321099, 0.51103728]],
[[0.45602357, 0.72700674, 0.51332611],
[0.14605884, -0.63010819, 0.76264702],
[0.87790052, -0.2728092, -0.39352995]],
[[0.60918383, 0.52822546, 0.59150057],
[0.73834933, -0.64995523, -0.17999572],
[0.28937056, 0.54638454, -0.78595714]],
[[7.71746128e-01, 4.78767298e-01, 4.18556793e-01],
[4.76878378e-01, -2.72559500e-04, -8.78969248e-01],
[-4.20707651e-01, 8.77941797e-01, -2.28524119e-01]],
[[0.78888283, 0.55521065, 0.2634483],
[-0.15905217, 0.5985437, -0.78514193],
[-0.59360448, 0.57748297, 0.56048831]],
[[0.71232121, 0.68900052, 0.13370407],
[-0.69422699, 0.71968522, -0.01010355],
[-0.10318619, -0.085624, 0.99096979]]])
for i in range(6):
self.cam_mat[i] = self.cam_mat[i] @ R
self.cam_mat = torch.FloatTensor(self.cam_mat)
self.cameras_coordinate = [
[2.50436065 + x, -3.75589484 + y, 1.88800446 + z],
[4.02581981 + x, -2.56894275 + y, -3.29281609 + z],
[1.01348544 + x, 1.88043939 + y, -5.4273143 + z],
[-2.45261002 + x, 3.5962286 + y, -1.87506165 + z],
[-3.12155638 + x, 2.09254542 + y, 2.21770186 + z],
[-1.07692383 + x, -1.37631717 + y, 4.3081322 + z]
]
xx = -0.3
yy = 1.2
zz = 0.1
for i in range(6):
self.cameras_coordinate[i] = np.array(
self.cameras_coordinate[i]) @ R
self.cameras_coordinate[i][0] += xx
self.cameras_coordinate[i][1] += yy
self.cameras_coordinate[i][2] += zz
self.cameras_coordinate = torch.FloatTensor(self.cameras_coordinate)
if torch.cuda.is_available():
self.cam_mat = torch.FloatTensor(self.cam_mat).cuda()
self.cameras_coordinate = torch.FloatTensor(
self.cameras_coordinate).cuda()
# self.gif_writer = imageio.get_writer('example.gif', mode='I')
pass
class Renderer(nn.Module):
'''
上纹理+渲染
'''
def __init__(self, N=6, f_dim=256, point_num=1024):
super(Renderer, self).__init__()
'''
初始化参数:
'''
self.N = N
self.f_dim = f_dim
self.point_num = point_num
# DIB渲染器
# self.renderer = DIBRenderer(height=640, width=400, mode='Lambertian',camera_fov_y=66.96 * np.pi / 180.0)
self.renderer = DIBRenderer(height=256,
width=256,
mode='Lambertian',
camera_fov_y=66.96 * np.pi / 180.0)
self.renderer.set_look_at_parameters([0], [0], [0],
fovx=57.77316 * np.pi / 180.0,
fovy=44.95887 * np.pi / 180.0,
near=0.01,
far=10.0)
# 设置相机参数
# self.renderer.set_camera_parameters()
# 预定义相机外参
# 真实参数
# 相机全部往中心移动
# x = 0.597105
# y = 1.062068
# z = 1.111316
# x = -0.3
# y = 1.2
# z = 0.1
x = 0
y = 0
z = 0
self.cam_mat = np.array(
[[[0.57432211, 0.77105488, 0.27500633],
[-0.56542476, 0.13069975, 0.81437854],
[0.59198729, -0.62321099, 0.51103728]],
[[0.45602357, 0.72700674, 0.51332611],
[0.14605884, -0.63010819, 0.76264702],
[0.87790052, -0.2728092, -0.39352995]],
[[0.60918383, 0.52822546, 0.59150057],
[0.73834933, -0.64995523, -0.17999572],
[0.28937056, 0.54638454, -0.78595714]],
[[7.71746128e-01, 4.78767298e-01, 4.18556793e-01],
[4.76878378e-01, -2.72559500e-04, -8.78969248e-01],
[-4.20707651e-01, 8.77941797e-01, -2.28524119e-01]],
[[0.78888283, 0.55521065, 0.2634483],
[-0.15905217, 0.5985437, -0.78514193],
[-0.59360448, 0.57748297, 0.56048831]],
[[0.71232121, 0.68900052, 0.13370407],
[-0.69422699, 0.71968522, -0.01010355],
[-0.10318619, -0.085624, 0.99096979]]])
for i in range(6):
self.cam_mat[i] = self.cam_mat[i] @ R
self.cam_mat = torch.FloatTensor(self.cam_mat)
self.cameras_coordinate = [
[2.50436065 + x, -3.75589484 + y, 1.88800446 + z],
[4.02581981 + x, -2.56894275 + y, -3.29281609 + z],
[1.01348544 + x, 1.88043939 + y, -5.4273143 + z],
[-2.45261002 + x, 3.5962286 + y, -1.87506165 + z],
[-3.12155638 + x, 2.09254542 + y, 2.21770186 + z],
[-1.07692383 + x, -1.37631717 + y, 4.3081322 + z]
]
xx = -0.3
yy = 1.2
zz = 0.1
for i in range(6):
self.cameras_coordinate[i] = np.array(
self.cameras_coordinate[i]) @ R
self.cameras_coordinate[i][0] += xx
self.cameras_coordinate[i][1] += yy
self.cameras_coordinate[i][2] += zz
self.cameras_coordinate = torch.FloatTensor(self.cameras_coordinate)
if torch.cuda.is_available():
self.cam_mat = torch.FloatTensor(self.cam_mat).cuda()
self.cameras_coordinate = torch.FloatTensor(
self.cameras_coordinate).cuda()
# self.gif_writer = imageio.get_writer('example.gif', mode='I')
pass
def forward(self, vertex_positions, mesh_faces, input_uvs, input_texture,
mesh_face_textures):
'''
输入:
vertices : 顶点数*3
faces : 面数*3
uv : uv坐标值 : 顶点数x2
texture : NxCxHxW 图像
输出:
N个视角下的渲染图像:NxCxHxW
'''
images = []
img_probs = []
for i in range(self.N):
# N个视角下渲染,每次渲染一个视角下的batchsize张图片
# 设置相机参数
camera_view_mtx = self.cam_mat[i].repeat(vertex_positions.shape[0],
1, 1)
camera_view_shift = self.cameras_coordinate[i].repeat(
vertex_positions.shape[0], 1)
self.renderer.camera_params = [
camera_view_mtx, camera_view_shift,
self.renderer.camera_params[2]
]
predictions, img_prob, _ = self.renderer(
points=[vertex_positions, mesh_faces],
uv_bxpx2=input_uvs,
texture_bx3xthxtw=input_texture,
ft_fx3=mesh_face_textures)
predictions = torch.cat((predictions, img_prob),
dim=3).permute(0, 3, 1, 2)
temp = predictions.detach().cpu().numpy()[0]
# self.gif_writer.append_data((temp * 255).astype(np.uint8))
images.append(predictions)
img_probs.append(img_prob)
return images, img_probs
def get_camera_info(self):
return self.cam_mat, self.cameras_coordinate
import torch
from kaolin.rep import TriangleMesh
from kaolin.graphics import DIBRenderer
from config import DEVICE
renderer = DIBRenderer(128, 128)
class VertexRenderer:
def __init__(self):
pass
@classmethod
def render(cls, mesh, dist, elev, azim, colors=None):
isinstance(mesh, TriangleMesh)
dist, elev, azim = cls.check_camera_parameters(dist, elev, azim)
renderer.set_look_at_parameters([azim], [elev], [dist])
vertices = mesh.vertices.clone().to(DEVICE)[None]
faces = mesh.faces.clone().to(DEVICE)
colors = torch.ones_like(vertices).to(
DEVICE) if colors is None else colors
render_rgbs, render_alphas, face_norms = renderer.forward(
points=[vertices, faces], colors_bxpx3=colors)
return render_rgbs, render_alphas, face_norms
@staticmethod
def check_camera_parameters(dist, elev, azim):
if isinstance(dist, torch.Tensor):
def main():
args = parse_arguments()
###########################
# Load mesh
###########################
mesh = TriangleMesh.from_obj(args.mesh)
vertices = mesh.vertices.cuda()
faces = mesh.faces.int().cuda()
# Expand such that batch size = 1
vertices = vertices.unsqueeze(0)
###########################
# Normalize mesh position
###########################
vertices_max = vertices.max()
vertices_min = vertices.min()
vertices_middle = (vertices_max + vertices_min) / 2.
vertices = (vertices - vertices_middle) * MESH_SIZE
###########################
# Generate vertex color
###########################
if not args.use_texture:
vert_min = torch.min(vertices, dim=1, keepdims=True)[0]
vert_max = torch.max(vertices, dim=1, keepdims=True)[0]
colors = (vertices - vert_min) / (vert_max - vert_min)
###########################
# Generate texture mapping
###########################
if args.use_texture:
uv = get_spherical_coords_x(vertices[0].cpu().numpy())
uv = torch.from_numpy(uv).cuda()
# Expand such that batch size = 1
uv = uv.unsqueeze(0)
###########################
# Load texture
###########################
if args.use_texture:
# Load image as numpy array
texture = np.array(Image.open(args.texture))
# Convert numpy array to PyTorch tensor
texture = torch.from_numpy(texture).cuda()
# Convert from [0, 255] to [0, 1]
texture = texture.float() / 255.0
# Convert to NxCxHxW layout
texture = texture.permute(2, 0, 1).unsqueeze(0)
###########################
# Render
###########################
if args.use_texture:
renderer_mode = 'Lambertian'
else:
renderer_mode = 'VertexColor'
renderer = Renderer(HEIGHT, WIDTH, mode=renderer_mode)
loop = tqdm.tqdm(list(range(0, 360, 4)))
loop.set_description('Drawing')
os.makedirs(args.output_path, exist_ok=True)
writer = imageio.get_writer(os.path.join(args.output_path, 'example.gif'), mode='I')
for azimuth in loop:
renderer.set_look_at_parameters([90 - azimuth],
[CAMERA_ELEVATION],
[CAMERA_DISTANCE])
if args.use_texture:
predictions, _, _ = renderer(points=[vertices, faces.long()],
uv_bxpx2=uv,
texture_bx3xthxtw=texture)
else:
predictions, _, _ = renderer(points=[vertices, faces.long()],
colors_bxpx3=colors)
image = predictions.detach().cpu().numpy()[0]
writer.append_data((image * 255).astype(np.uint8))
writer.close()
batch_size=args.batchsize,
shuffle=False,
collate_fn=collate_fn,
num_workers=0)
# Model
mesh = kal.rep.TriangleMesh.from_obj('386.obj', enable_adjacency=True)
mesh.cuda()
normalize_adj(mesh)
initial_verts = mesh.vertices.clone()
camera_fov_y = 49.13434207744484 * np.pi / 180.0
cam_proj = perspectiveprojectionnp(camera_fov_y, 1.0)
cam_proj = torch.FloatTensor(cam_proj).cuda()
model = Encoder(4, 5, args.batchsize, 137, mesh.vertices.shape[0]).cuda()
renderer = Dib_Renderer(137, 137, mode='VertexColor')
model.load_state_dict(torch.load('log/{0}/best.pth'.format(args.expid)))
loss_epoch = 0.
f_epoch = 0.
num_batches = 0
num_items = 0
loss_fn = kal.metrics.point.chamfer_distance
model.eval()
with torch.no_grad():
for data in (tqdm(dataloader_val)):
# data creation
def visualisemask(vertices, faces, PdroneToCamera, b, isPredicted=True):
"""Set up renderer for visualising drone based on the predicted pose
PdroneToCamera: relative pose predicted by network or GT
"""
if not __KAOLIN_LOADED__:
return
device = vertices.get_device()
vertices = vertices.expand(b, vertices.size(1), vertices.size(2))
colors = torch.zeros(vertices.size()).to(device)
#setup color
vert_min = torch.min(vertices, dim=1, keepdims=True)[0]
vert_max = torch.max(vertices, dim=1, keepdims=True)[0]
if (isPredicted):
#colors[:,:,:2] = 0
colors[:, :, :3] = 1
else:
colors[:, :, 2:3] = 0
colors[:, :, 1] = 1
#get homogeneous coordinates for vertices
vertices = torch.torch.nn.functional.pad(vertices[:, :, ], [0, 1],
"constant", 1.0)
vertices = vertices.transpose(2, 1)
vertices = torch.matmul(PdroneToCamera, vertices)
#set up renderer
renderer = Renderer(320 * 1, 240 * 1)
vertices = vertices.transpose(2, 1)
b, _, _ = PdroneToCamera.size()
#set camera parameters
cameras = []
camera_rot_bx3x3 = torch.zeros((b, 3, 3), dtype=torch.float32).to(device)
camera_rot_bx3x3[:, 0, 0] = 1
camera_rot_bx3x3[:, 1, 1] = 1
camera_rot_bx3x3[:, 2, 2] = 1
cameras.append(camera_rot_bx3x3)
camera_pos_bx3 = torch.zeros((b, 3), dtype=torch.float32).to(device)
cameras.append(camera_pos_bx3)
camera_proj_3x1 = torch.zeros((3, 1), dtype=torch.float32).to(device)
camera_proj_3x1[:, :] = torch.from_numpy(
geometry.perspectiveprojectionnp(radians(50.8),
ratio=4. / 3.,
near=0.3,
far=750.0))
cameras.append(camera_proj_3x1)
renderer.set_camera_parameters(cameras)
#convert points from homogeneous
z_vec = vertices[..., -1:]
scale = torch.tensor(1.) / torch.clamp(z_vec, 0.000000001)
vertices = vertices[..., :-1]
#forward pass
predictions, mask, _ = renderer(points=[vertices, faces.long()],
colors_bxpx3=colors)
return predictions, mask
def main():
args = parse_arguments()
###########################
# Load mesh
###########################
filename = os.path.join(current_dir, args.filename_input)
#print(filename)
os.makedirs(args.output, exist_ok=True)
data_dir_1 = os.path.join(args.output, 'rendered_1.5_600',
filename.split('/')[6],
filename.split('/')[7])
mesh = TriangleMesh.from_obj(filename, with_vt=True, texture_res=5)
vertices = mesh.vertices
faces = mesh.faces.long()
face_textures = mesh.face_textures
textures = mesh.textures
uvs = mesh.uvs
## trying to corect for pytorch coordinates
#uvs[:, 1] = 1 - uvs[:, 1]
#uvs = uvs * 2 - 1
pfmtx = face2pfmtx(faces.numpy())
# Expand such that batch size = 1
vertices = vertices[None, :, :].cuda()
faces = faces[None, :, :].cuda()
face_textures = face_textures[None, :, :].cuda()
textures = textures[None, :, :, :].cuda()
uvs = uvs[None, :, :].cuda()
textures = textures.permute([0, 3, 1, 2])
#vertices = vertices.unsqueeze(0)
#print(vertices.shape, faces.shape, textures.shape, uvs.shape)
###########################
# Normalize mesh position
###########################
vertices_max = vertices.max()
vertices_min = vertices.min()
vertices_middle = (vertices_max + vertices_min) / 2.
vertices = (vertices - vertices_middle) * MESH_SIZE
###########################
# Generate vertex color
###########################
#vert_min = torch.min(vertices, dim=1, keepdims=True)[0]
#vert_max = torch.max(vertices, dim=1, keepdims=True)[0]
#colors = (vertices - vert_min) / (vert_max - vert_min)
###########################
# Mat, Light, and Shine #
###########################
bs = 1
material = np.array([[0.1, 0.1, 0.1], [1.0, 1.0, 1.0], [0.4, 0.4, 0.4]],
dtype=np.float32).reshape(-1, 3, 3)
shininess = np.array([100], dtype=np.float32).reshape(-1,
1) # this was 100
tfmat = torch.from_numpy(material).repeat(bs, 1, 1).cuda()
tfshi = torch.from_numpy(shininess).repeat(bs, 1).cuda()
lightdirect = np.array([0, 1, 0], dtype=np.float32).reshape(
(bs, 3)) #2 * np.random.rand(bs, 3).astype(np.float32) - 1
#lightdirect[:, 2] += 2
tflight = torch.from_numpy(lightdirect)
tflight_bx3 = tflight.cuda()
###########################
# Render
###########################
renderer = Renderer(HEIGHT, WIDTH, mode='Phong')
#renderer.renderer.set_smooth(pfmtx)
#loop = tqdm.tqdm(list(range(0, 180, 4)))
#loop.set_description('Drawing')
#writer = imageio.get_writer(os.path.join(args.output_path, 'example.gif'), mode='I')
#writer_sil = imageio.get_writer(os.path.join(args.output_path, 'example_sil.gif'), mode='I')
theta1 = np.round(np.random.uniform(0, 360, 20), 2)
theta2 = tqdm.tqdm(list(np.round(np.random.uniform(0, 360, 20), 2)))
os.makedirs(data_dir_1, exist_ok=True)
#net = resnet18(pretrained=True).cuda()
for num, azimuth in enumerate(theta2):
renderer.set_look_at_parameters([90 - azimuth], [theta1[num]],
[CAMERA_DISTANCE])
predictions, silhouette, _ = renderer(
points=[vertices, faces[0].long()],
uv_bxpx2=uvs,
ft_fx3=face_textures[0],
texture_bx3xthxtw=textures,
lightdirect_bx3=tflight_bx3,
material_bx3x3=tfmat,
shininess_bx1=tfshi)
image = predictions.detach().cpu().numpy()[0]
#imageio.imwrite('results/example.png', (255*image[:,:,:]).astype(np.uint8))
#writer.append_data((image * 255).astype(np.uint8))
mask = silhouette.detach().cpu().numpy()[0]
#print(mask.shape, image.shape)
whole_image = np.concatenate((image, mask), axis=2)
#print(whole_image.shape)
imageio.imwrite(
('%s/t1_%s_t2_%s.png' % (data_dir_1, theta1[num], azimuth)),
(255 * whole_image[:, :, :]).astype(np.uint8))
#imageio.imwrite('results/example_sil.png', (255*gray[:,:,:]).astype(np.uint8))
#writer_sil.append_data((gray * 255).astype(np.uint8))
#writer.close()
'''os.makedirs(data_dir_1,exist_ok=True)