def __init__(self, w, h, f):
self.w = w
self.h = h
self.f = f
self.rn_d = DepthRenderer(frustum={
'near': 0.1,
'far': 10.,
'width': w,
'height': h
},
f=f)
class VisibilityChecker:
def __init__(self, w, h, f):
self.w = w
self.h = h
self.f = f
self.rn_d = DepthRenderer(frustum={'near': 0.1, 'far': 10., 'width': w, 'height': h}, f=f)
def vertex_visibility(self, camera, mask=None):
cam3d = ProjectPoints3D(**{k: getattr(camera, k) for k in camera.dterms if hasattr(camera, k)})
in_viewport = np.logical_and(
np.logical_and(np.round(camera.r[:, 0]) >= 0, np.round(camera.r[:, 0]) < self.w),
np.logical_and(np.round(camera.r[:, 1]) >= 0, np.round(camera.r[:, 1]) < self.h),
)
if not hasattr(self.rn_d, 'camera') or not np.all(self.rn_d.camera.r == camera.r):
self.rn_d.set(camera=camera)
depth = self.rn_d.r
proj = cam3d.r[in_viewport]
d = proj[:, 2]
idx = np.round(proj[:, [1, 0]].T).astype(np.int).tolist()
visible = np.zeros(cam3d.shape[0], dtype=np.bool)
visible[in_viewport] = np.abs(d - depth[tuple(idx)]) < 0.01
if mask is not None:
mask = cv2.erode(mask, np.ones((5, 5)))
visible[in_viewport] = np.logical_and(visible[in_viewport], mask[tuple(idx)])
return visible
def face_visibility(self, camera, mask=None):
v_vis = self.vertex_visibility(camera, mask)
return np.min(v_vis[self.f], axis=1)
def vertex_visibility_angle(self, camera):
n = VertNormals(camera.v, self.f)
v_cam = camera.v.r.dot(cv2.Rodrigues(camera.rt.r)[0]) + camera.t.r
n_cam = n.r.dot(cv2.Rodrigues(camera.rt.r)[0])
return np.sum(v_cam / (np.linalg.norm(v_cam, axis=1).reshape(-1, 1)) * -1 * n_cam, axis=1)
def face_visibility_angle(self, camera):
v_cam = camera.v.r.dot(cv2.Rodrigues(camera.rt.r)[0]) + camera.t.r
f_cam = v_cam[self.f]
f_norm = np.cross(f_cam[:, 0] - f_cam[:, 1], f_cam[:, 0] - f_cam[:, 2], axisa=1, axisb=1)
f_norm /= np.linalg.norm(f_norm, axis=1).reshape(-1, 1)
center = np.mean(f_cam, axis=1)
return np.sum(center / (np.linalg.norm(center, axis=1).reshape(-1, 1)) * -1 * f_norm, axis=1)
def _create_renderer(w=640,
h=480,
rt=np.zeros(3),
t=np.zeros(3),
f=None,
c=None,
k=None,
near=.5,
far=10.):
f = np.array([w, w]) / 2. if f is None else f
c = np.array([w, h]) / 2. if c is None else c
k = np.zeros(5) if k is None else k
rn = DepthRenderer()
rn.camera = ProjectPoints(rt=rt, t=t, f=f, c=c, k=k)
rn.frustum = {'near': near, 'far': far, 'height': h, 'width': w}
return rn
def render_bound(mesh, require_id = False, img_size=(448, 448), f=1000):
rn = DepthRenderer()
rn.camera = ProjectPoints(rt = np.zeros(3),
t = np.zeros(3),
f = np.array([f, f]),
c = np.array([img_size[1], img_size[0]])/2.,
k = np.zeros(5)
)
rn.frustum = {'near': .5, 'far': 10.,
'width': img_size[1], 'height': img_size[0]}
rn.v = mesh.points()
rn.f = mesh.face_vertex_indices()
rn.bgcolor = np.zeros(3)
if require_id is False:
return rn.boundarybool_image
else:
return rn.boundaryid_image
def depth_render(self,
max_dep,
verts,
cam=None,
far=None,
near=None,
img_size=None):
dRenderer = DepthRenderer()
dRenderer.camera = ProjectPoints(v=verts,
rt=np.zeros(3),
t=np.zeros(3),
f=cam[0] * np.ones(2),
c=cam[1:3],
k=np.zeros(5))
"""
cam is 3D [f, px, py]
"""
if img_size is not None:
h = img_size[0]
w = img_size[1]
else:
h = self.h
w = self.w
if cam is None:
cam = [self.flength, w / 2., h / 2.]
use_cam = ProjectPoints(
f=cam[0] * np.ones(2),
rt=np.zeros(3),
#rt=np.array([np.pi, 0, 0]),
t=np.zeros(3),
k=np.zeros(5),
c=cam[1:3])
if near is None:
near = np.maximum(np.min(verts[:, 2]) - 25, 0.1)
if far is None:
far = np.maximum(np.max(verts[:, 2]) + 25, 25)
#dRenderer.frustum={"near":1.,"far":5.,"width":self.width,"height":self.height}
dRenderer.frustum = {"near": near, "far": far, "width": w, "height": h}
dRenderer.v = verts
dRenderer.f = self.faces
depthArr = dRenderer.r.copy()
depthArr[depthArr > max_dep] = 0
depthArr = (depthArr / max_dep) * 255.0
depthArr = depthArr.astype(np.uint8)
return depthArr
def render_depth_v(verts, faces,
require_visi = False, img_size=(448, 448), f=1000):
rn = DepthRenderer()
rn.camera = ProjectPoints(rt = np.zeros(3),
t = np.zeros(3),
f = np.array([f, f]),
c = np.array([img_size[1], img_size[0]])/2.,
k = np.zeros(5)
)
rn.frustum = {'near': .5, 'far': 10.,
'width': img_size[1], 'height': img_size[0]}
rn.v = verts
rn.f = faces
rn.bgcolor = np.zeros(3)
if require_visi is True:
return rn.r, rn.visibility_image
else:
return rn.r
def render_sil(mesh, norm = True, img_size=(448, 448), f=1000):
rn = DepthRenderer()
rn.camera = ProjectPoints(rt = np.zeros(3),
t = np.zeros(3),
f = np.array([f, f]),
c = np.array([img_size[1], img_size[0]])/2.,
k = np.zeros(5)
)
rn.frustum = {'near': .5, 'far': 10.,
'width': img_size[1], 'height': img_size[0]}
rn.v = mesh.points()
rn.f = mesh.face_vertex_indices()
rn.bgcolor = np.zeros(3)
depth_map = rn.r
sil_img = np.zeros(depth_map.shape, dtype = np.uint8)
if norm is True:
sil_img[depth_map<10] = 1
else:
sil_img[depth_map<10] = 255
return sil_img
def get_3DSV(mesh):
from opendr.camera import ProjectPoints
from opendr.renderer import DepthRenderer
WIDTH, HEIGHT = 250, 250
camera = ProjectPoints(v=mesh.vertices,
f=np.array([WIDTH, WIDTH]),
c=np.array([WIDTH, HEIGHT]) / 2.,
t=np.array([0, 0, 2.5]),
rt=np.array([np.pi, 0, 0]),
k=np.zeros(5))
frustum = {'near': 1., 'far': 10., 'width': WIDTH, 'height': HEIGHT}
rn = DepthRenderer(camera=camera,
frustum=frustum,
f=mesh.faces,
overdraw=False)
points3d = camera.unproject_depth_image(rn.r)
points3d = points3d[points3d[:, :, 2] > np.min(points3d[:, :, 2]) + 0.01]
# print('sampled {} points.'.format(points3d.shape[0]))
return points3d
def get_3DSV(mesh):
from opendr.camera import ProjectPoints
from opendr.renderer import DepthRenderer
WIDTH, HEIGHT = 250, 250
rt = R.from_euler('xyz', [np.pi, 0, 0]).as_rotvec()
rt_mat = R.from_euler('xyz', [np.pi, 0, 0]).as_matrix()
camera = ProjectPoints(v=mesh.vertices, f=np.array([WIDTH, WIDTH]), c=np.array([WIDTH, HEIGHT]) / 2.,
t=np.array([0, 0, 3.0]), rt=rt, k=np.zeros(5))
frustum = {'near': 1., 'far': 10., 'width': WIDTH, 'height': HEIGHT}
rn = DepthRenderer(camera=camera, frustum=frustum, f=mesh.faces, overdraw=False)
# import cv2
depth_image = rn.depth_image.copy()
mask = depth_image < depth_image.max() - 0.01
depth_image[~mask] = 0
depth_image[mask] = 255 - (depth_image[mask] - depth_image[mask].min()) / (depth_image[mask].max() - depth_image[mask].min()) * 255
points3d = camera.unproject_depth_image(rn.r)
mask = points3d[:, :, 2] > np.min(points3d[:, :, 2]) + 0.01
points3d = points3d[mask]
return points3d, depth_image
isOpenGLCoords=True)
objKps = project3DPoints(camMat, objCornersTrans, isOpenGLCoords=True)
# draw 2D hand keypoints and object corners
imgAnno_gt = showHandJoints(img, handKps.copy(), lineThickness=2)
imgAnno_gt = showObjJoints(imgAnno_gt, objKps.copy(), lineThickness=2)
imgAnno_mano = showHandJoints(img,
handKps_mano.copy(),
lineThickness=2)
imgAnno_mano = showObjJoints(imgAnno_mano,
objKps.copy(),
lineThickness=2)
# render mano vertices
rn = DepthRenderer()
rn.camera = ProjectPoints(v=vertices[0],
rt=np.zeros(3),
t=np.zeros(3),
f=np.array([fx, fy]),
c=np.array([u0, v0]),
k=np.zeros(5))
rn.frustum = {
'near': 0.01,
'far': 1.5,
'width': img_w,
'height': img_h
}
rn.set(v=vertices[0], f=mano.faces, bgcolor=np.zeros(3))
# render
mano_rendered = rn.r