class VisRenderer(object):
"""
Utility to render meshes using pytorch NMR
faces are F x 3 or 1 x F x 3 numpy
"""
def __init__(self, img_size, faces, t_size=3):
self.renderer = NeuralRenderer(img_size)
self.faces = Variable(torch.IntTensor(faces).cuda(),
requires_grad=False)
if self.faces.dim() == 2:
self.faces = torch.unsqueeze(self.faces, 0)
default_tex = np.ones(
(1, self.faces.shape[1], t_size, t_size, t_size, 3))
blue = np.array([156, 199, 234.]) / 255.
default_tex = default_tex * blue
# Could make each triangle different color
self.default_tex = Variable(torch.FloatTensor(default_tex).cuda(),
requires_grad=False)
# rot = transformations.quaternion_about_axis(np.pi/8, [1, 0, 0])
# This is median quaternion from sfm_pose
# rot = np.array([ 0.66553962, 0.31033762, -0.02249813, 0.01267084])
# This is the side view:
import cv2
R0 = cv2.Rodrigues(np.array([np.pi / 3, 0, 0]))[0]
R1 = cv2.Rodrigues(np.array([0, np.pi / 2, 0]))[0]
R = R1.dot(R0)
R = np.vstack((np.hstack((R, np.zeros((3, 1)))), np.array([0, 0, 0,
1])))
rot = transformations.quaternion_from_matrix(R, isprecise=True)
cam = np.hstack([0.75, 0, 0, rot])
self.default_cam = Variable(torch.FloatTensor(cam).cuda(),
requires_grad=False)
self.default_cam = torch.unsqueeze(self.default_cam, 0)
def __call__(self, verts, cams=None, texture=None, rend_mask=False):
"""
verts is |V| x 3 cuda torch Variable
cams is 7, cuda torch Variable
Returns N x N x 3 numpy
"""
#print("visrender call")
if texture is None:
texture = self.default_tex
elif texture.dim() == 5:
# Here input it F x T x T x T x 3 (instead of F x T x T x 3)
# So add batch dim.
texture = torch.unsqueeze(texture, 0)
if cams is None:
cams = self.default_cam
elif cams.dim() == 1:
cams = torch.unsqueeze(cams, 0)
if verts.dim() == 2:
verts = torch.unsqueeze(verts, 0)
#------------------------------ edited by parker
#------------------------------this is edited bird mesh----
f = open("edited_bird_mesh.off", "w")
f.write("OFF\n")
line = str(len(verts[0])) + " " + str(len(self.faces[0])) + " 0\n"
f.write(line)
mesh_x = np.empty(len(verts[0]))
mesh_y = np.empty(len(verts[0]))
mesh_z = np.empty(len(verts[0]))
# print("bird_vis verts:",verts[0])
for i in range(len(verts[0])):
line = str(float(verts[0][i][0])) + " " + str(float(
verts[0][i][1])) + " " + str(float(verts[0][i][2])) + "\n"
f.write(line)
for j in range(3):
if (j == 0):
mesh_x[i] = verts[0][i][j]
elif (j == 1):
mesh_y[i] = verts[0][i][j]
else:
mesh_z[i] = verts[0][i][j]
tri_i = np.empty(len(self.faces[0]))
tri_j = np.empty(len(self.faces[0]))
tri_k = np.empty(len(self.faces[0]))
for i in range(len(self.faces[0])):
line = str(3) + " " + str(int(self.faces[0][i][0])) + " " + str(
int(self.faces[0][i][1])) + " " + str(int(
self.faces[0][i][2])) + "\n"
f.write(line)
for j in range(3):
if (j == 0):
tri_i[i] = self.faces[0][i][j]
elif (j == 1):
tri_j[i] = self.faces[0][i][j]
else:
tri_k[i] = self.faces[0][i][j]
# fig = go.Figure(
# data=[go.Mesh3d(x=mesh_x, y=mesh_y, z=mesh_z, color='lightblue', opacity=0.5, i=tri_i, j=tri_j, k=tri_k)])
# fig.show()
f.close()
# ----------------------edited by parker-----------------
verts = asVariable(verts)
cams = asVariable(cams)
texture = asVariable(texture)
if rend_mask:
rend = self.renderer.forward(verts, self.faces, cams)
rend = rend.repeat(3, 1, 1)
rend = rend.unsqueeze(0)
else:
rend = self.renderer.forward(verts, self.faces, cams, texture)
rend = rend.data.cpu().numpy()[0].transpose((1, 2, 0))
rend = np.clip(rend, 0, 1) * 255.0
return rend.astype(np.uint8)
def rotated(self, vert, deg, axis=[0, 1, 0], cam=None, texture=None):
"""
vert is N x 3, torch FloatTensor (or Variable)
"""
import cv2
new_rot = cv2.Rodrigues(np.deg2rad(deg) * np.array(axis))[0]
new_rot = convert_as(torch.FloatTensor(new_rot), vert)
center = vert.mean(0)
new_vert = torch.t(torch.matmul(new_rot,
torch.t(vert - center))) + center
# new_vert = torch.matmul(vert - center, new_rot) + center
return self.__call__(new_vert, cams=cam, texture=texture)
def diff_vp(self,
verts,
cam=None,
angle=90,
axis=[1, 0, 0],
texture=None,
kp_verts=None,
new_ext=None,
extra_elev=False):
if cam is None:
cam = self.default_cam[0]
if new_ext is None:
new_ext = [0.6, 0, 0]
# Cam is 7D: [s, tx, ty, rot]
import cv2
cam = asVariable(cam)
quat = cam[-4:].view(1, 1, -1)
R = transformations.quaternion_matrix(
quat.squeeze().data.cpu().numpy())[:3, :3]
rad_angle = np.deg2rad(angle)
rotate_by = cv2.Rodrigues(rad_angle * np.array(axis))[0]
# new_R = R.dot(rotate_by)
new_R = rotate_by.dot(R)
if extra_elev:
# Left multiply the camera by 30deg on X.
R_elev = cv2.Rodrigues(np.array([np.pi / 9, 0, 0]))[0]
new_R = R_elev.dot(new_R)
# Make homogeneous
new_R = np.vstack(
[np.hstack((new_R, np.zeros((3, 1)))),
np.array([0, 0, 0, 1])])
new_quat = transformations.quaternion_from_matrix(new_R,
isprecise=True)
new_quat = Variable(torch.Tensor(new_quat).cuda(), requires_grad=False)
# new_cam = torch.cat([cam[:-4], new_quat], 0)
new_ext = Variable(torch.Tensor(new_ext).cuda(), requires_grad=False)
new_cam = torch.cat([new_ext, new_quat], 0)
rend_img = self.__call__(verts, cams=new_cam, texture=texture)
if kp_verts is None:
return rend_img
else:
kps = self.renderer.project_points(kp_verts.unsqueeze(0),
new_cam.unsqueeze(0))
kps = kps[0].data.cpu().numpy()
return kp2im(kps, rend_img, radius=1)
def set_bgcolor(self, color):
self.renderer.set_bgcolor(color)
def set_light_dir(self, direction, int_dir=0.8, int_amb=0.8):
renderer = self.renderer.renderer
renderer.light_direction = direction
renderer.light_intensity_directional = int_dir
renderer.light_intensity_ambient = int_amb
class VisRenderer(object):
"""
Utility to render meshes using pytorch NMR
faces are F x 3 or 1 x F x 3 numpy
"""
def __init__(self, img_size, faces, opts, t_size=3, uv_sampler=None):
self.opts = opts
# TODO junzhe option of renderer
if self.opts.renderer_opt == 'nmr':
from nnutils.nmr import NeuralRenderer
elif self.opts.renderer_opt == 'nmr_kaolin':
from nnutils.nmr_kaolin import NeuralRenderer
elif self.opts.renderer_opt == 'dibr_kaolin':
from nnutils.dibr_kaolin import NeuralRenderer
else:
raise NotImplementedError
self.renderer = NeuralRenderer(img_size, uv_sampler=uv_sampler)
self.faces = Variable(torch.IntTensor(faces).cuda(),
requires_grad=False)
if self.faces.dim() == 2:
self.faces = torch.unsqueeze(self.faces, 0)
default_tex = np.ones(
(1, self.faces.shape[1], t_size, t_size, t_size, 3))
blue = np.array([156, 199, 234.]) / 255.
default_tex = default_tex * blue
# Could make each triangle different color
self.default_tex = Variable(torch.FloatTensor(default_tex).cuda(),
requires_grad=False)
# rot = transformations.quaternion_about_axis(np.pi/8, [1, 0, 0])
# This is median quaternion from sfm_pose
# rot = np.array([ 0.66553962, 0.31033762, -0.02249813, 0.01267084])
# This is the side view:
import cv2
R0 = cv2.Rodrigues(np.array([np.pi / 3, 0, 0]))[0]
R1 = cv2.Rodrigues(np.array([0, np.pi / 2, 0]))[0]
R = R1.dot(R0)
R = np.vstack((np.hstack((R, np.zeros((3, 1)))), np.array([0, 0, 0,
1])))
rot = transformations.quaternion_from_matrix(R, isprecise=True)
cam = np.hstack([0.75, 0, 0, rot])
self.default_cam = Variable(torch.FloatTensor(cam).cuda(),
requires_grad=False)
self.default_cam = torch.unsqueeze(self.default_cam, 0)
def __call__(self, verts, cams=None, texture=None, rend_mask=False):
"""
verts is |V| x 3 cuda torch Variable
cams is 7, cuda torch Variable
Returns N x N x 3 numpy
"""
if texture is None:
texture = self.default_tex
elif texture.dim() == 5:
# Here input it F x T x T x T x 3 (instead of F x T x T x 3)
# So add batch dim.
texture = torch.unsqueeze(texture, 0)
if cams is None:
cams = self.default_cam
elif cams.dim() == 1:
cams = torch.unsqueeze(cams, 0)
if verts.dim() == 2:
verts = torch.unsqueeze(verts, 0)
verts = asVariable(verts)
cams = asVariable(cams)
texture = asVariable(texture)
if rend_mask:
rend = self.renderer.forward(verts, self.faces, cams)
rend = rend.repeat(3, 1, 1)
rend = rend.unsqueeze(0)
else:
rend = self.renderer.forward(verts, self.faces, cams, texture)
rend = rend.data.cpu().numpy()[0].transpose((1, 2, 0))
rend = np.clip(rend, 0, 1) * 255.0
return rend.astype(np.uint8)
def rotated(self, vert, deg, axis=[0, 1, 0], cam=None, texture=None):
"""
vert is N x 3, torch FloatTensor (or Variable)
"""
import cv2
new_rot = cv2.Rodrigues(np.deg2rad(deg) * np.array(axis))[0]
new_rot = convert_as(torch.FloatTensor(new_rot), vert)
center = vert.mean(0)
new_vert = torch.t(torch.matmul(new_rot,
torch.t(vert - center))) + center
# new_vert = torch.matmul(vert - center, new_rot) + center
return self.__call__(new_vert, cams=cam, texture=texture)
def diff_vp(self,
verts,
cam=None,
angle=90,
axis=[1, 0, 0],
texture=None,
kp_verts=None,
new_ext=None,
extra_elev=False):
if cam is None:
cam = self.default_cam[0]
if new_ext is None:
new_ext = [0.6, 0, 0]
# Cam is 7D: [s, tx, ty, rot]
import cv2
cam = asVariable(cam)
quat = cam[-4:].view(1, 1, -1)
R = transformations.quaternion_matrix(
quat.squeeze().data.cpu().numpy())[:3, :3]
rad_angle = np.deg2rad(angle)
rotate_by = cv2.Rodrigues(rad_angle * np.array(axis))[0]
# new_R = R.dot(rotate_by)
new_R = rotate_by.dot(R)
if extra_elev:
# Left multiply the camera by 30deg on X.
R_elev = cv2.Rodrigues(np.array([np.pi / 9, 0, 0]))[0]
new_R = R_elev.dot(new_R)
# Make homogeneous
new_R = np.vstack(
[np.hstack((new_R, np.zeros((3, 1)))),
np.array([0, 0, 0, 1])])
new_quat = transformations.quaternion_from_matrix(new_R,
isprecise=True)
new_quat = Variable(torch.Tensor(new_quat).cuda(), requires_grad=False)
# new_cam = torch.cat([cam[:-4], new_quat], 0)
new_ext = Variable(torch.Tensor(new_ext).cuda(), requires_grad=False)
new_cam = torch.cat([new_ext, new_quat], 0)
rend_img = self.__call__(verts, cams=new_cam, texture=texture)
if kp_verts is None:
return rend_img
else:
kps = self.renderer.project_points(kp_verts.unsqueeze(0),
new_cam.unsqueeze(0))
kps = kps[0].data.cpu().numpy()
return kp2im(kps, rend_img, radius=1)
def set_bgcolor(self, color):
self.renderer.set_bgcolor(color)
def set_light_dir(self, direction, int_dir=0.8, int_amb=0.8):
renderer = self.renderer.renderer
renderer.light_direction = direction
renderer.light_intensity_directional = int_dir
renderer.light_intensity_ambient = int_amb
