def render(self, r, t, r_type="quat"): if r_type == "quat": R = quat2mat(r) elif r_type == "mat": R = r self.render_kernel["u_view"] = self._get_view_mtx(R, t) app.run(framecount=0) rgb_buffer = np.zeros((self.height, self.width, 4), dtype=np.float32) gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT, rgb_buffer) rgb_gl = np.copy(rgb_buffer) rgb_gl.shape = 480, 640, 4 rgb_gl = rgb_gl[::-1, :] rgb_gl = np.round(rgb_gl[:, :, :3] * 255).astype( np.uint8) # Convert to [0, 255] bgr_gl = rgb_gl[:, :, [2, 1, 0]] depth_buffer = np.zeros((self.height, self.width), dtype=np.float32) gl.glReadPixels(0, 0, self.width, self.height, gl.GL_DEPTH_COMPONENT, gl.GL_FLOAT, depth_buffer) depth_gl = np.copy(depth_buffer) depth_gl.shape = 480, 640 depth_gl = depth_gl[::-1, :] depth_bg = depth_gl == 1 depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear - (self.zFar - self.zNear) * (2 * depth_gl - 1)) depth_gl[depth_bg] = 0 return bgr_gl, depth_gl
def render(self, cls_idx, r, t, r_type="quat", K=None): if r_type == "quat": R = quat2mat(r) elif r_type == "mat": R = r self.cls_idx = cls_idx self.render_kernel_list[cls_idx]["u_view"] = self._get_view_mtx(R, t) if K is not None: u_projection = self.my_compute_calib_proj(K, self.width, self.height, self.zNear, self.zFar) self.render_kernel_list[cls_idx]["u_projection"] = np.copy( u_projection) # import time # t = time.time() app.run(framecount=0, framerate=0) # print("render {} seconds/image".format(time.time()-t)) # app.run() rgb_gl = np.flipud(self.rgb_buffer) depth_gl = np.flipud(self.depth_buffer) bgr_gl = rgb_gl[:, :, [2, 1, 0]] # convert to BGR format as cv2 bgr_gl *= 255 depth_bg = depth_gl == 1 depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear - (self.zFar - self.zNear) * (2 * depth_gl - 1)) depth_gl[depth_bg] = 0 return bgr_gl, depth_gl
def render(self, r, t, r_type='quat', K=None): if r_type == 'quat': R = quat2mat(r) elif r_type == 'mat': R = r self.render_kernel['u_view'] = self._get_view_mtx(R, t) if K is not None: u_projection = self.my_compute_calib_proj(K, self.width, self.height, self.zNear, self.zFar) self.render_kernel['u_projection'] = np.copy(u_projection) app.run(framecount=0, framerate=0) rgb_gl = np.flipud(self.rgb_buffer) depth_gl = np.flipud(self.depth_buffer) bgr_gl = rgb_gl[:, :, [2, 1, 0]] # convert to BGR format as cv2 bgr_gl *= 255 depth_bg = depth_gl == 1 depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear - (self.zFar - self.zNear) * (2 * depth_gl - 1)) depth_gl[depth_bg] = 0 return bgr_gl, depth_gl
def render(self, r, t, light_position, light_intensity, brightness_k=0, r_type='quat'): ''' :param r: :param t: :param light_position: :param light_intensity: :param brightness_k: choose which brightness in __init__ :param r_type: :return: ''' if r_type == 'quat': R = quat2mat(r) elif r_type == 'mat': R = r self.brightness_k = brightness_k self.render_kernels[brightness_k]['u_view'] = self._get_view_mtx(R, t) self.render_kernels[brightness_k]['u_light_position'] = light_position self.render_kernels[brightness_k]['u_normal'] = np.array( np.matrix( np.dot( self.render_kernels[brightness_k]['u_view'].reshape(4, 4), self.render_kernels[brightness_k]['u_model'].reshape( 4, 4))).I.T) self.render_kernels[brightness_k][ 'u_light_intensity'] = light_intensity app.run(framecount=0) rgb_buffer = np.zeros((self.height, self.width, 4), dtype=np.float32) gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT, rgb_buffer) rgb_gl = np.copy(rgb_buffer) rgb_gl.shape = 480, 640, 4 rgb_gl = rgb_gl[::-1, :] rgb_gl = np.round(rgb_gl[:, :, :3] * 255).astype( np.uint8) # Convert to [0, 255] bgr_gl = rgb_gl[:, :, [2, 1, 0]] depth_buffer = np.zeros((self.height, self.width), dtype=np.float32) gl.glReadPixels(0, 0, self.width, self.height, gl.GL_DEPTH_COMPONENT, gl.GL_FLOAT, depth_buffer) depth_gl = np.copy(depth_buffer) depth_gl.shape = 480, 640 depth_gl = depth_gl[::-1, :] depth_bg = depth_gl == 1 depth_gl = 2 * self.zFar * self.zNear / (self.zFar + self.zNear - (self.zFar - self.zNear) * (2 * depth_gl - 1)) depth_gl[depth_bg] = 0 return bgr_gl, depth_gl
exe1.arg_dict["rotation"][:] = mx.ndarray.array(v_rotation, ctx=ctx, dtype="float32") exe1.arg_dict["translation"][:] = mx.ndarray.array(v_translation, ctx=ctx, dtype="float32") exe1.arg_dict["pose_src"][:] = mx.ndarray.array(v_pose_src, ctx=ctx, dtype="float32") exe1.forward(is_train=is_train) simple_forward(exe1, v_point_cloud, v_rotation, v_translation, v_pose_src, ctx=ctx, is_train=True) y1_raw = exe1.outputs[0] y1 = y1_raw.asnumpy() sum_y1 = get_output(y1) # forward check for iter in range(batch_size): pc = np.squeeze(v_point_cloud[iter]) rot_q_delta = v_rotation[iter] trans_delta = v_translation[iter] pose_src = v_pose_src[iter] rot_m_delta = quat2mat(rot_q_delta) pose_est = RT_transform(pose_src, rot_q_delta, trans_delta, T_means, T_stds, rot_coord) rot_m_tgt = pose_est[:3, :3] trans_tgt = pose_est[:, 3] pc_gt = np.matmul(rot_m_tgt, pc) + trans_tgt.reshape((3, 1)) pc_est = y1[iter] assert (np.abs(pc_est - pc_gt).max(1) < 1e-4).all(), "forward check failed, {} > {}".format( np.abs(pc_est - pc_gt).sum(), thresh ) print("forward check succeed") top_grad = mx.nd.zeros_like(y1_raw) set_diff(top_grad) exe1.backward(top_grad)
def render( self, r, t, light_position, light_intensity, brightness_k=0, r_type="quat", K=None, ): """ :param r: :param t: :param light_position: :param light_intensity: :param brightness_k: choose which brightness in __init__ :param r_type: :return: """ if r_type == "quat": R = quat2mat(r) elif r_type == "mat": R = r self.brightness_k = brightness_k self.render_kernels[brightness_k]["u_view"] = self._get_view_mtx(R, t) self.render_kernels[brightness_k]["u_light_position"] = light_position self.render_kernels[brightness_k]["u_normal"] = np.array( np.matrix( np.dot( self.render_kernels[brightness_k]["u_view"].reshape(4, 4), self.render_kernels[brightness_k]["u_model"].reshape(4, 4), )).I.T) self.render_kernels[brightness_k][ "u_light_intensity"] = light_intensity if K is not None: u_projection = self.my_compute_calib_proj(K, self.width, self.height, self.zNear, self.zFar) self.render_kernels[brightness_k]["u_projection"] = np.copy( u_projection) app.run(framecount=0) rgb_buffer = np.zeros((self.height, self.width, 4), dtype=np.float32) gl.glReadPixels(0, 0, self.width, self.height, gl.GL_RGBA, gl.GL_FLOAT, rgb_buffer) rgb_gl = np.copy(rgb_buffer) rgb_gl.shape = 480, 640, 4 rgb_gl = rgb_gl[::-1, :] rgb_gl = np.round(rgb_gl[:, :, :3] * 255).astype( np.uint8) # Convert to [0, 255] bgr_gl = rgb_gl[:, :, [2, 1, 0]] depth_buffer = np.zeros((self.height, self.width), dtype=np.float32) gl.glReadPixels( 0, 0, self.width, self.height, gl.GL_DEPTH_COMPONENT, gl.GL_FLOAT, depth_buffer, ) depth_gl = np.copy(depth_buffer) depth_gl.shape = 480, 640 depth_gl = depth_gl[::-1, :] depth_bg = depth_gl == 1 depth_gl = (2 * self.zFar * self.zNear / (self.zFar + self.zNear - (self.zFar - self.zNear) * (2 * depth_gl - 1))) depth_gl[depth_bg] = 0 return bgr_gl, depth_gl