def render(self, r, t, r_type="quat"):
if r_type == "quat":
R = quat2mat(r)
elif r_type == "mat":
R = r
self.render_kernel["u_view"] = self._get_view_mtx(R, t)
app.run(framecount=0)
rgb_buffer = np.zeros((self.height, self.width, 4), dtype=np.float32)
gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT,
rgb_buffer)
rgb_gl = np.copy(rgb_buffer)
rgb_gl.shape = 480, 640, 4
rgb_gl = rgb_gl[::-1, :]
rgb_gl = np.round(rgb_gl[:, :, :3] * 255).astype(
np.uint8) # Convert to [0, 255]
bgr_gl = rgb_gl[:, :, [2, 1, 0]]
depth_buffer = np.zeros((self.height, self.width), dtype=np.float32)
gl.glReadPixels(0, 0, self.width, self.height, gl.GL_DEPTH_COMPONENT,
gl.GL_FLOAT, depth_buffer)
depth_gl = np.copy(depth_buffer)
depth_gl.shape = 480, 640
depth_gl = depth_gl[::-1, :]
depth_bg = depth_gl == 1
depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear -
(self.zFar - self.zNear) *
(2 * depth_gl - 1))
depth_gl[depth_bg] = 0
return bgr_gl, depth_gl
def render(self, cls_idx, r, t, r_type="quat", K=None):
if r_type == "quat":
R = quat2mat(r)
elif r_type == "mat":
R = r
self.cls_idx = cls_idx
self.render_kernel_list[cls_idx]["u_view"] = self._get_view_mtx(R, t)
if K is not None:
u_projection = self.my_compute_calib_proj(K, self.width,
self.height, self.zNear,
self.zFar)
self.render_kernel_list[cls_idx]["u_projection"] = np.copy(
u_projection)
# import time
# t = time.time()
app.run(framecount=0, framerate=0)
# print("render {} seconds/image".format(time.time()-t))
# app.run()
rgb_gl = np.flipud(self.rgb_buffer)
depth_gl = np.flipud(self.depth_buffer)
bgr_gl = rgb_gl[:, :, [2, 1, 0]] # convert to BGR format as cv2
bgr_gl *= 255
depth_bg = depth_gl == 1
depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear -
(self.zFar - self.zNear) *
(2 * depth_gl - 1))
depth_gl[depth_bg] = 0
return bgr_gl, depth_gl
def render(self, r, t, r_type='quat', K=None):
if r_type == 'quat':
R = quat2mat(r)
elif r_type == 'mat':
R = r
self.render_kernel['u_view'] = self._get_view_mtx(R, t)
if K is not None:
u_projection = self.my_compute_calib_proj(K, self.width,
self.height, self.zNear,
self.zFar)
self.render_kernel['u_projection'] = np.copy(u_projection)
app.run(framecount=0, framerate=0)
rgb_gl = np.flipud(self.rgb_buffer)
depth_gl = np.flipud(self.depth_buffer)
bgr_gl = rgb_gl[:, :, [2, 1, 0]] # convert to BGR format as cv2
bgr_gl *= 255
depth_bg = depth_gl == 1
depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear -
(self.zFar - self.zNear) *
(2 * depth_gl - 1))
depth_gl[depth_bg] = 0
return bgr_gl, depth_gl
def render(self,
r,
t,
light_position,
light_intensity,
brightness_k=0,
r_type='quat'):
'''
:param r:
:param t:
:param light_position:
:param light_intensity:
:param brightness_k: choose which brightness in __init__
:param r_type:
:return:
'''
if r_type == 'quat':
R = quat2mat(r)
elif r_type == 'mat':
R = r
self.brightness_k = brightness_k
self.render_kernels[brightness_k]['u_view'] = self._get_view_mtx(R, t)
self.render_kernels[brightness_k]['u_light_position'] = light_position
self.render_kernels[brightness_k]['u_normal'] = np.array(
np.matrix(
np.dot(
self.render_kernels[brightness_k]['u_view'].reshape(4, 4),
self.render_kernels[brightness_k]['u_model'].reshape(
4, 4))).I.T)
self.render_kernels[brightness_k][
'u_light_intensity'] = light_intensity
app.run(framecount=0)
rgb_buffer = np.zeros((self.height, self.width, 4), dtype=np.float32)
gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT,
rgb_buffer)
rgb_gl = np.copy(rgb_buffer)
rgb_gl.shape = 480, 640, 4
rgb_gl = rgb_gl[::-1, :]
rgb_gl = np.round(rgb_gl[:, :, :3] * 255).astype(
np.uint8) # Convert to [0, 255]
bgr_gl = rgb_gl[:, :, [2, 1, 0]]
depth_buffer = np.zeros((self.height, self.width), dtype=np.float32)
gl.glReadPixels(0, 0, self.width, self.height, gl.GL_DEPTH_COMPONENT,
gl.GL_FLOAT, depth_buffer)
depth_gl = np.copy(depth_buffer)
depth_gl.shape = 480, 640
depth_gl = depth_gl[::-1, :]
depth_bg = depth_gl == 1
depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear -
(self.zFar - self.zNear) *
(2 * depth_gl - 1))
depth_gl[depth_bg] = 0
return bgr_gl, depth_gl
exe1.arg_dict["rotation"][:] = mx.ndarray.array(v_rotation, ctx=ctx, dtype="float32")
exe1.arg_dict["translation"][:] = mx.ndarray.array(v_translation, ctx=ctx, dtype="float32")
exe1.arg_dict["pose_src"][:] = mx.ndarray.array(v_pose_src, ctx=ctx, dtype="float32")
exe1.forward(is_train=is_train)
simple_forward(exe1, v_point_cloud, v_rotation, v_translation, v_pose_src, ctx=ctx, is_train=True)
y1_raw = exe1.outputs[0]
y1 = y1_raw.asnumpy()
sum_y1 = get_output(y1)
# forward check
for iter in range(batch_size):
pc = np.squeeze(v_point_cloud[iter])
rot_q_delta = v_rotation[iter]
trans_delta = v_translation[iter]
pose_src = v_pose_src[iter]
rot_m_delta = quat2mat(rot_q_delta)
pose_est = RT_transform(pose_src, rot_q_delta, trans_delta, T_means, T_stds, rot_coord)
rot_m_tgt = pose_est[:3, :3]
trans_tgt = pose_est[:, 3]
pc_gt = np.matmul(rot_m_tgt, pc) + trans_tgt.reshape((3, 1))
pc_est = y1[iter]
assert (np.abs(pc_est - pc_gt).max(1) < 1e-4).all(), "forward check failed, {} > {}".format(
np.abs(pc_est - pc_gt).sum(), thresh
)
print("forward check succeed")
top_grad = mx.nd.zeros_like(y1_raw)
set_diff(top_grad)
exe1.backward(top_grad)
def render(
self,
r,
t,
light_position,
light_intensity,
brightness_k=0,
r_type="quat",
K=None,
):
"""
:param r:
:param t:
:param light_position:
:param light_intensity:
:param brightness_k: choose which brightness in __init__
:param r_type:
:return:
"""
if r_type == "quat":
R = quat2mat(r)
elif r_type == "mat":
R = r
self.brightness_k = brightness_k
self.render_kernels[brightness_k]["u_view"] = self._get_view_mtx(R, t)
self.render_kernels[brightness_k]["u_light_position"] = light_position
self.render_kernels[brightness_k]["u_normal"] = np.array(
np.matrix(
np.dot(
self.render_kernels[brightness_k]["u_view"].reshape(4, 4),
self.render_kernels[brightness_k]["u_model"].reshape(4, 4),
)).I.T)
self.render_kernels[brightness_k][
"u_light_intensity"] = light_intensity
if K is not None:
u_projection = self.my_compute_calib_proj(K, self.width,
self.height, self.zNear,
self.zFar)
self.render_kernels[brightness_k]["u_projection"] = np.copy(
u_projection)
app.run(framecount=0)
rgb_buffer = np.zeros((self.height, self.width, 4), dtype=np.float32)
gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT,
rgb_buffer)
rgb_gl = np.copy(rgb_buffer)
rgb_gl.shape = 480, 640, 4
rgb_gl = rgb_gl[::-1, :]
rgb_gl = np.round(rgb_gl[:, :, :3] * 255).astype(
np.uint8) # Convert to [0, 255]
bgr_gl = rgb_gl[:, :, [2, 1, 0]]
depth_buffer = np.zeros((self.height, self.width), dtype=np.float32)
gl.glReadPixels(
0,
0,
self.width,
self.height,
gl.GL_DEPTH_COMPONENT,
gl.GL_FLOAT,
depth_buffer,
)
depth_gl = np.copy(depth_buffer)
depth_gl.shape = 480, 640
depth_gl = depth_gl[::-1, :]
depth_bg = depth_gl == 1
depth_gl = (2 * self.zFar * self.zNear /
(self.zFar + self.zNear - (self.zFar - self.zNear) *
(2 * depth_gl - 1)))
depth_gl[depth_bg] = 0
return bgr_gl, depth_gl
