def _frames_and_index_map(self, skip_registration=False):
"""Retrieve a new frame from the Kinect and return a ColorImage,
DepthImage, IrImage, and a map from depth pixels to color pixel indices.
Parameters
----------
skip_registration : bool
If True, the registration step is skipped.
Returns
-------
:obj:`tuple` of :obj:`ColorImage`, :obj:`DepthImage`, :obj:`IrImage`, :obj:`numpy.ndarray`
The ColorImage, DepthImage, and IrImage of the current frame, and an
ndarray that maps pixels of the depth image to the index of the
corresponding pixel in the color image.
Raises
------
RuntimeError
If the Kinect stream is not running.
"""
if not self._running:
raise RuntimeError(
'Kinect2 device %s not runnning. Cannot read frames' %
(self._device_num))
# read frames
frames = self._listener.waitForNewFrame()
unregistered_color = frames['color']
distorted_depth = frames['depth']
ir = frames['ir']
# apply color to depth registration
color_frame = self._color_frame
color = unregistered_color
depth = distorted_depth
color_depth_map = np.zeros([depth.height,
depth.width]).astype(np.int32).ravel()
if not skip_registration and self._registration_mode == Kinect2RegistrationMode.COLOR_TO_DEPTH:
color_frame = self._ir_frame
depth = lf2.Frame(depth.width, depth.height, 4,
lf2.FrameType.Depth)
color = lf2.Frame(depth.width, depth.height, 4,
lf2.FrameType.Color)
self._registration.apply(unregistered_color,
distorted_depth,
depth,
color,
color_depth_map=color_depth_map)
# convert to array (copy needed to prevent reference of deleted data
color_arr = copy.copy(color.asarray())
color_arr[:, :, [0, 2]] = color_arr[:, :, [2, 0]] # convert BGR to RGB
color_arr[:, :, 0] = np.fliplr(color_arr[:, :, 0])
color_arr[:, :, 1] = np.fliplr(color_arr[:, :, 1])
color_arr[:, :, 2] = np.fliplr(color_arr[:, :, 2])
color_arr[:, :, 3] = np.fliplr(color_arr[:, :, 3])
depth_arr = np.fliplr(copy.copy(depth.asarray()))
ir_arr = np.fliplr(copy.copy(ir.asarray()))
# convert meters
if self._depth_mode == Kinect2DepthMode.METERS:
depth_arr = depth_arr * MM_TO_METERS
# Release and return
self._listener.release(frames)
return (ColorImage(color_arr[:, :, :3],
color_frame), DepthImage(depth_arr, self._ir_frame),
IrImage(ir_arr.astype(np.uint16),
self._ir_frame), color_depth_map)
def _frames(self):
"""Retrieve a new frame from.
Returns:
A dictionary of RGB image, depth image and segmentation image.
'image': The RGB image as an uint8 np array of [width, height, 3].
'depth': The depth image as a float32 np array of [width, height].
Raises:
RuntimeError: If the Kinect stream is not running.
"""
if not self._running:
raise RuntimeError('Kinect2 device %s not runnning. '
'Cannot read frames' % self._device_num)
# Read frames.
frames = self._listener.waitForNewFrame()
unregistered_color = frames['color']
distorted_depth = frames['depth']
# Apply color to depth regirobovation.
color = unregistered_color
depth = distorted_depth
color_depth_map = np.zeros([depth.height,
depth.width]).astype(np.int32).ravel()
if not self._skip_regirobovation and (
self._regirobovation_mode
== Kinect2RegirobovationMode.COLOR_TO_DEPTH):
depth = lf2.Frame(depth.width, depth.height, 4,
lf2.FrameType.Depth)
color = lf2.Frame(depth.width, depth.height, 4,
lf2.FrameType.Color)
self._regirobovation.apply(unregistered_color,
distorted_depth,
depth,
color,
color_depth_map=color_depth_map)
# Convert to array (copy needed to prevent reference of deleted data).
rgba = copy.copy(color.asarray())
# Convert BGR to RGB.
rgba[:, :, [0, 2]] = rgba[:, :, [2, 0]]
rgba[:, :, 0] = np.fliplr(rgba[:, :, 0])
rgba[:, :, 1] = np.fliplr(rgba[:, :, 1])
rgba[:, :, 2] = np.fliplr(rgba[:, :, 2])
rgba[:, :, 3] = np.fliplr(rgba[:, :, 3])
rgb = rgba[:, :, :3]
# Depth image.
depth = np.fliplr(copy.copy(depth.asarray()))
# Convert meters.
if self._depth_mode == Kinect2DepthMode.METERS:
depth = depth / 1000.0
# Release and return.
self._listener.release(frames)
if self._use_inpaint:
rgb = image_utils.inpaint(rgb,
rescale_factor=INPAINT_RESCALE_FACTOR)
depth = depth_utils.inpaint(depth,
rescale_factor=INPAINT_RESCALE_FACTOR)
if self._upside_down:
rgb = rgb[::-1, ::-1, :]
depth = depth[::-1, ::-1]
return {
'rgb': rgb,
'depth': depth,
'segmask': None,
}