def __init__(self, invert_camera=False):
self.win_depth = "depth"
self.win_left = "left_rectified"
self.win_right = "right_rectified"
self.win_disparity = "disparity"
self.win_colormap = "colormap"
cv2.namedWindow(self.win_depth)
cv2.createTrackbar("num", self.win_depth, 2, 10, lambda x: None)
cv2.createTrackbar("blockSize", self.win_depth, 5, 255, lambda x: None)
self.ecv = image_processing.EventCv()
self.ecv.add_mouse_event(self.win_depth)
self.ecv.add_mouse_event(self.win_disparity)
self.ecv.add_mouse_event(self.win_colormap)
camera_height = camera_configs.camera_height
camera_width = camera_configs.camera_width
input_size = [camera_height, camera_width]
# self.pose = openpose.OpenPose(model_path, input_size=input_size)
if invert_camera:
self.camera1 = cv2.VideoCapture(1) # left camera1
self.camera2 = cv2.VideoCapture(0) # right camera2
else:
self.camera1 = cv2.VideoCapture(0) # left camera1
self.camera2 = cv2.VideoCapture(1) # right camera2
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
self.g = geometry_3d_pose.Geometry3DPose(camera_configs)
self.g.show_origin_pcd()
self.g.show_bone_line_pcd()
self.g.show_image_pcd(True)
self.g.show_desktop_pcd(True)
def __init__(self, invert_camera=False):
self.win_depth = "depth"
self.win_left = "left_rectified"
self.win_right = "right_rectified"
self.win_disparity = "disparity"
self.win_colormap = "colormap"
cv2.namedWindow(self.win_depth)
cv2.createTrackbar("num", self.win_depth, 2, 10, lambda x: None)
cv2.createTrackbar("blockSize", self.win_depth, 5, 255, lambda x: None)
self.ecv = image_processing.EventCv()
self.ecv.add_mouse_event(self.win_depth)
self.ecv.add_mouse_event(self.win_disparity)
self.ecv.add_mouse_event(self.win_colormap)
camera_height = camera_configs.camera_height
camera_width = camera_configs.camera_width
input_size = [camera_height, camera_width]
# self.pose = openpose.OpenPose(model_path, input_size=input_size)
if invert_camera:
self.camera1 = cv2.VideoCapture(1) # left camera1
self.camera2 = cv2.VideoCapture(0) # right camera2
else:
self.camera1 = cv2.VideoCapture(0) # left camera1
self.camera2 = cv2.VideoCapture(1) # right camera2
self.detector = UltraLightFaceDetector(model_path=None, network=None, )
self.g = open3d_visual.Open3DVisual(camera_intrinsic=camera_configs.camera_intrinsic,
depth_width=camera_configs.depth_width,
depth_height=camera_configs.camera_height,
depth_scale=camera_configs.depth_scale,
clipping_distance_in_meters=camera_configs.clipping_distance_in_meters)
self.g.show_origin_pcd()
self.g.show_bone_line_pcd()
self.g.show_image_pcd(True)
class_name = "ttt"
real_part = True
# real_part = False
dataset_root = "dataset/"
self.scale = 1
self.prefix = "v"
if real_part:
self.snapshot_dir = os.path.join(dataset_root, "real_part", class_name)
else:
self.snapshot_dir = os.path.join(dataset_root, "fake_part", class_name)
file_processing.create_dir(self.snapshot_dir, "color")
file_processing.create_dir(self.snapshot_dir, "depth")
# file_processing.create_dir(self.snapshot_dir, "ir")
file_processing.create_dir(self.snapshot_dir, "video")
video_name = file_processing.get_time()
self.save_video = os.path.join(self.snapshot_dir, "video", "{}_{}.avi".format(class_name, video_name))
if self.save_video:
self.video_writer = self.get_video_writer(self.save_video,
width=640,
height=480,
fps=20) # (424, 512, 4)
def __init__(self, calibration_file, width=640, height=480):
"""
:param calibration_file:
:param width:
:param height:
"""
self.config = camera_params.get_stereo_coefficients(
calibration_file, width, height)
self.pcd = open3d_visual.Open3DVisual(
camera_intrinsic=self.config["K1"],
depth_width=width,
depth_height=height)
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
class_name = "ttt"
real_part = True
# real_part = False
save_root = "dataset/"
self.scale = 1
self.prefix = "v"
if real_part:
self.save_dir = os.path.join(save_root, "real_part", class_name)
else:
self.save_dir = os.path.join(save_root, "fake_part", class_name)
file_processing.create_dir(self.save_dir, "color")
file_processing.create_dir(self.save_dir, "depth")
file_processing.create_dir(self.save_dir, "ir")
file_processing.create_dir(self.save_dir, "video")
video_name = file_processing.get_time()
self.save_l_video = os.path.join(
self.save_dir, "video",
"left_{}_{}.avi".format(class_name, video_name))
self.save_r_video = os.path.join(
self.save_dir, "video",
"right_{}_{}.avi".format(class_name, video_name))
if self.save_l_video:
self.video_l_writer = self.get_video_writer(self.save_l_video,
width=width,
height=height,
fps=30)
if self.save_r_video:
self.video_r_writer = self.get_video_writer(self.save_r_video,
width=width,
height=height,
fps=30)
self.pcd.show_image_pcd(True)
self.pcd.show_origin_pcd(True)
self.pcd.show_image_pcd(True)
def __init__(self):
model_path = "/media/dm/dm1/git/python-learning-notes/libs/ultra_ligh_face/face_detection_rbf.pth"
network = "RFB"
confidence_threshold = 0.85
nms_threshold = 0.3
top_k = 500
keep_top_k = 750
device = "cuda:0"
self.detector = UltraLightFaceDetector(model_path=model_path,
network=network,
confidence_threshold=confidence_threshold,
nms_threshold=nms_threshold,
top_k=top_k,
keep_top_k=keep_top_k,
device=device)
def __init__(self):
# Kinect runtime object
self.joint_count = PyKinectV2.JointType_Count # 25
self.kinect = PyKinectRuntime.PyKinectRuntime(
PyKinectV2.FrameSourceTypes_Body
| PyKinectV2.FrameSourceTypes_Color
| PyKinectV2.FrameSourceTypes_Depth)
self.depth_width, self.depth_height = self.kinect.depth_frame_desc.Width, self.kinect.depth_frame_desc.Height
self.color_width, self.color_height = self.kinect.color_frame_desc.Width, self.kinect.color_frame_desc.Height
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
self.g = geometry_3d_pose.Geometry3DPose(kinect_config)
def select_facebank_detect(image_dir, dest_dir, id_nums=None, detect_face=True):
if detect_face:
# model_path = "../../face_detection/face_detection_rbf.pth"
# model_path = "/media/dm/dm1/git/python-learning-notes/libs/ultra_ligh_face/face_detection_rbf.pth"
model_path = "/home/panjinquan/project/python-learning-notes//libs/ultra_ligh_face/face_detection_rbf.pth"
network = "RFB"
confidence_threshold = 0.85
nms_threshold = 0.3
top_k = 500
keep_top_k = 750
device = "cuda:0"
detector = UltraLightFaceDetector(model_path=model_path,
network=network,
confidence_threshold=confidence_threshold,
nms_threshold=nms_threshold,
top_k=top_k,
keep_top_k=keep_top_k,
device=device)
per_nums = 1
image_id = file_processing.get_sub_directory_list(image_dir)
nums_images = len(image_id)
print("have ID:{}".format(nums_images))
if id_nums:
id_nums = min(id_nums, nums_images)
image_id = image_id[:id_nums]
print("select ID:{}".format(len(image_id)))
for id in image_id:
image_list = file_processing.get_files_list(os.path.join(image_dir, id),
postfix=['*.jpg', "*.jpeg", '*.png', "*.JPG"])
count = 0
for src_path in image_list:
basename = os.path.basename(src_path)
if detect_face:
bgr_image = cv2.imread(src_path)
bboxes, scores, landms = detector.detect(bgr_image, isshow=True)
if not len(bboxes) == 1:
print("no face:{}".format(src_path))
continue
if count >= per_nums:
break
count += 1
dest_path = file_processing.create_dir(dest_dir, id, basename)
file_processing.copy_file(src_path, dest_path)
def __init__(self):
# Kinect runtime object
self.joint_count = PyKinectV2.JointType_Count # 25
self.kinect = PyKinectRuntime.PyKinectRuntime(
PyKinectV2.FrameSourceTypes_Body
| PyKinectV2.FrameSourceTypes_Color
| PyKinectV2.FrameSourceTypes_Infrared
| PyKinectV2.FrameSourceTypes_Depth)
self.depth_width, self.depth_height = self.kinect.depth_frame_desc.Width, self.kinect.depth_frame_desc.Height
self.color_width, self.color_height = self.kinect.color_frame_desc.Width, self.kinect.color_frame_desc.Height
self.ir_width, self.ir_height = self.kinect.infrared_frame_desc.Width, self.kinect.infrared_frame_desc.Height
self.g = geometry_3d_pose.Geometry3DPose(kinect_config)
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
class_name = "0"
# real_part = True
real_part = False
dataset_root = "dataset/"
self.scale = 2
self.prefix = "v1"
if real_part:
self.snapshot_dir = os.path.join(dataset_root, "real_part",
class_name)
else:
self.snapshot_dir = os.path.join(dataset_root, "fake_part",
class_name)
file_processing.create_dir(self.snapshot_dir, "color")
file_processing.create_dir(self.snapshot_dir, "depth")
file_processing.create_dir(self.snapshot_dir, "ir")
file_processing.create_dir(self.snapshot_dir, "video")
video_name = file_processing.get_time()
self.save_video = os.path.join(
self.snapshot_dir, "video",
"{}_{}.avi".format(class_name, video_name))
if self.save_video:
self.video_writer = self.get_video_writer(
self.save_video,
width=self.depth_width * self.scale,
height=self.depth_height * self.scale,
fps=20) # (424, 512, 4)
class Kinect2PointCloud():
def __init__(self):
# Kinect runtime object
self.joint_count = PyKinectV2.JointType_Count # 25
self.kinect = PyKinectRuntime.PyKinectRuntime(
PyKinectV2.FrameSourceTypes_Body
| PyKinectV2.FrameSourceTypes_Color
| PyKinectV2.FrameSourceTypes_Depth)
self.depth_width, self.depth_height = self.kinect.depth_frame_desc.Width, self.kinect.depth_frame_desc.Height
self.color_width, self.color_height = self.kinect.color_frame_desc.Width, self.kinect.color_frame_desc.Height
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
self.g = geometry_3d_pose.Geometry3DPose(kinect_config)
def start_capture(self, save_dir="", name=""):
self.g.show_origin_pcd(False)
self.g.show_bone_line_pcd(True)
self.g.show_image_pcd(False)
self.g.show_desktop_pcd(False)
save_dir = os.path.join(save_dir, name)
count = 0
save_freq = 3
while True:
# Get images from camera
if self.kinect.has_new_body_frame(
) and self.kinect.has_new_color_frame(
) and self.kinect.has_new_depth_frame():
body_frame = self.kinect.get_last_body_frame()
# ir_frame = self.kinect.get_last_infrared_frame()
color_frame = self.kinect.get_last_color_frame()
depth_frame = self.kinect.get_last_depth_frame()
# Reshape from 1D frame to 2D image
color_img = color_frame.reshape(
((self.color_height, self.color_width,
4))).astype(np.uint8)
depth_img = depth_frame.reshape(
((self.depth_height, self.depth_width))).astype(np.uint16)
align_color_img = utils.get_align_color_image(
self.kinect, color_img)
# front = self.detect_face(color_img)
body_joint2D, body_orientation = utils.get_body_joint2D(
body_frame, self.kinect, map_space="depth_space")
# if not front:
# body_joint2D = self.check_kpt_front(body_joint2D)
self.g.show(align_color_img, depth_img, body_joint2D)
body_joint2D = [d for d in body_joint2D if not d is None]
if body_joint2D and count % save_freq == 0:
self.save(save_dir, flag=count)
count += 1
def detect_face(self, image, front_tf=0.1):
image = cv2.cvtColor(image, cv2.COLOR_RGBA2RGB)
bboxes, scores, landms = self.detector.detect(image)
front = False
for s in scores:
if s > front_tf:
front = True
else:
front = False
break
image = image_processing.draw_image_bboxes_text(image,
bboxes,
scores,
color=(0, 0, 255))
image = image_processing.resize_image(image, resize_height=600)
cv2.imshow("Det", image)
cv2.waitKey(5)
return front
def check_kpt_front(self, body_joint2D):
pair_kpt = [(4, 8), (5, 9), (6, 10), (7, 11), (22, 24), (21, 23),
(12, 16), (13, 17), (14, 18), (15, 19)]
for k in range(len(body_joint2D)):
joint = body_joint2D[k]
if joint is None:
continue
for i, j in pair_kpt:
tmp_i = joint[i, :].copy()
tmp_j = joint[j, :].copy()
joint[i, :] = tmp_j
joint[j, :] = tmp_i
body_joint2D[k] = joint
return body_joint2D
def close(self):
self.kinect.close()
self.g.close()
def save(self, save_dir, flag):
if save_dir:
self.g.save_data(save_dir, flag)
class StereoCamera():
def __init__(self, invert_camera=False):
self.win_depth = "depth"
self.win_left = "left_rectified"
self.win_right = "right_rectified"
self.win_disparity = "disparity"
self.win_colormap = "colormap"
cv2.namedWindow(self.win_depth)
cv2.createTrackbar("num", self.win_depth, 2, 10, lambda x: None)
cv2.createTrackbar("blockSize", self.win_depth, 5, 255, lambda x: None)
self.ecv = image_processing.EventCv()
self.ecv.add_mouse_event(self.win_depth)
self.ecv.add_mouse_event(self.win_disparity)
self.ecv.add_mouse_event(self.win_colormap)
camera_height = camera_configs.camera_height
camera_width = camera_configs.camera_width
input_size = [camera_height, camera_width]
# self.pose = openpose.OpenPose(model_path, input_size=input_size)
if invert_camera:
self.camera1 = cv2.VideoCapture(1) # left camera1
self.camera2 = cv2.VideoCapture(0) # right camera2
else:
self.camera1 = cv2.VideoCapture(0) # left camera1
self.camera2 = cv2.VideoCapture(1) # right camera2
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
self.g = geometry_3d_pose.Geometry3DPose(camera_configs)
self.g.show_origin_pcd()
self.g.show_bone_line_pcd()
self.g.show_image_pcd(True)
self.g.show_desktop_pcd(True)
def get_rectify_image(self, image_left, image_right):
'''
畸变校正和立体校正
根据更正map对图片进行重构
获取用于畸变校正和立体校正的映射矩阵以及用于计算像素空间坐标的重投影矩阵
:param image_left:
:param image_right:
:return:
'''
left_rectified = cv2.remap(image_left, camera_configs.left_map1,
camera_configs.left_map2, cv2.INTER_LINEAR)
right_rectified = cv2.remap(image_right, camera_configs.right_map1,
camera_configs.right_map2,
cv2.INTER_LINEAR)
return left_rectified, right_rectified
def get_disparity_map(self,
left_rectified,
right_rectified,
num=1,
blockSize=1,
stereo_type="SGBM"):
'''
获得视差图
:param imgL:
:param imgR:
:param num:
:param blockSize:
:param stereo_type
:return:
'''
# 将图片置为灰度图，为StereoBM作准备
imgL = cv2.cvtColor(left_rectified, cv2.COLOR_BGR2GRAY)
imgR = cv2.cvtColor(right_rectified, cv2.COLOR_BGR2GRAY)
# 根据Block Maching方法生成差异图（opencv里也提供了SGBM/Semi-Global Block Matching算法，有兴趣可以试试）
if stereo_type == "BM":
stereo = cv2.StereoBM_create(numDisparities=16 * num,
blockSize=blockSize)
# stereo = cv2.StereoBM_create(numDisparities=16, blockSize=15)
elif stereo_type == "SGBM":
# SGBM匹配参数设置
if imgL.ndim == 2:
img_channels = 1
else:
img_channels = 3
# blockSize = 3
num = np.where(num > 0, num, 1)
param = {
'minDisparity': 0,
'numDisparities': 16 * num,
'blockSize': blockSize,
'P1': 2 * img_channels * blockSize,
'P2': 4 * img_channels * blockSize,
'disp12MaxDiff': 1,
'preFilterCap': 65,
'uniquenessRatio': 10,
'speckleWindowSize': 150,
'speckleRange': 2,
'mode': cv2.STEREO_SGBM_MODE_SGBM_3WAY
}
# 构建SGBM对象
stereo = cv2.StereoSGBM_create(**param)
# disparity_left = stereo.compute(imgL, imgR) # 计算左视差图
# disparity_right = stereo.compute(imgR, imgL) # 计算右视差图
disparity = stereo.compute(imgL, imgR)
# 除以16得到真实视差（因为SGBM算法得到的视差是×16的）
disparity[disparity < 0] = 0
# disparity.astype(np.float32) / 16.
disparity = np.divide(disparity.astype(np.float32), 16.)
return disparity
# 顺时针旋转90度
def RotateClockWise90(self, img):
trans_img = cv2.transpose(img)
new_img = cv2.flip(trans_img, 1)
return new_img
def get_depth(self, disparity, only_depth=True, scale=1.0):
'''
:param disparity:
:param only_depth:
:param scale:
:return: returm scale=1.0,距离,单位为毫米
'''
# 将图片扩展至3d空间中，其z方向的值则为当前的距离
depth = cv2.reprojectImageTo3D(disparity, camera_configs.Q)
if only_depth:
x, y, depth = cv2.split(depth)
depth = depth * scale
return depth
def start_capture(self):
snapshot_dir = "data/ov0575"
file_processing.create_dir(snapshot_dir)
count = 0
while True:
# 两个trackbar用来调节不同的参数查看效果
num = cv2.getTrackbarPos("num", "depth")
blockSize = cv2.getTrackbarPos("blockSize", "depth")
if blockSize % 2 == 0:
blockSize += 1
if blockSize < 5:
blockSize = 5
# 获取左右摄像头的数据
ret1, frameR = self.camera1.read()
ret2, frameL = self.camera2.read()
# frameL=self.RotateClockWise90(frameL)
# frameR=self.RotateClockWise90(frameR)
# 畸变校正和立体校正
self.left_rectified, self.right_rectified = self.get_rectify_image(
image_left=frameL, image_right=frameR)
# 获得视差图
self.disparity = self.get_disparity_map(self.left_rectified,
self.right_rectified, num,
blockSize)
# 计算像素点的3D坐标（左相机坐标系下）
self.depth = self.get_depth(self.disparity)
self.depth = np.asarray(self.depth, dtype=np.uint16)
# save_dir = "./data/data01"
# self.left_rectified, self.depth, _ = rgbd_data.load_body_joint_data(save_dir, count=40)
# self.g.show(self.left_rectified, self.depth, body_joint2D, body_rects=body_rects)
# self.g.show(frameL, self.depth, body_joint=None)
bboxes, scores, landms = self.detector.detect(self.left_rectified,
isshow=False)
self.show_landmark_boxes("Det", self.left_rectified, bboxes,
scores, landms)
self.display()
#
key = cv2.waitKey(1)
if key == ord("q"):
break
elif key == ord("s"):
count += 1
cv2.imwrite(
os.path.join(snapshot_dir, "left_{}.png".format(count)),
frameL)
cv2.imwrite(
os.path.join(snapshot_dir, "right_{}.png".format(count)),
frameR)
cv2.imwrite(
os.path.join(snapshot_dir,
"left_rectified_{}.png".format(count)),
self.left_rectified)
cv2.imwrite(
os.path.join(snapshot_dir,
"right_rectified_{}.png".format(count)),
self.right_rectified)
cv2.imwrite(
os.path.join(snapshot_dir, "depth_{}.png".format(count)),
self.depth)
@staticmethod
def show_landmark_boxes(win_name, image, bboxes, scores, landms):
'''
显示landmark和boxes
:param win_name:
:param image:
:param landmarks_list: [[x1, y1], [x2, y2]]
:param bboxes: [[ x1, y1, x2, y2],[ x1, y1, x2, y2]]
:return:
'''
image = image_processing.draw_landmark(image, landms, vis_id=True)
image = image_processing.draw_image_bboxes_text(image,
bboxes,
scores,
color=(0, 0, 255))
cv2.imshow(win_name, image)
def display(self, ):
self.ecv.update_image(self.depth)
# disparity_image = cv2.normalize(self.disparity, self.disparity, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX,
# dtype=cv2.CV_8U)
disparity_image = np.uint8(self.disparity.clip(1, 4080) /
16.) # 转换为uint8时，需要避免溢出255*16=4080
depth_colormap = cv2.applyColorMap(
cv2.convertScaleAbs(self.depth,
alpha=255 / camera_configs.clipping_distance),
cv2.COLORMAP_JET)
cv2.imshow(self.win_left, self.left_rectified)
cv2.imshow(self.win_right, self.right_rectified)
cv2.imshow(self.win_disparity, disparity_image)
cv2.imshow(self.win_depth, self.depth)
cv2.imshow(self.win_colormap, depth_colormap)
def close(self):
self.camera1.release()
self.camera2.release()
cv2.destroyAllWindows()
class StereoCamera():
def __init__(self, invert_camera=False):
self.win_depth = "depth"
self.win_left = "left_rectified"
self.win_right = "right_rectified"
self.win_disparity = "disparity"
self.win_colormap = "colormap"
cv2.namedWindow(self.win_depth)
cv2.createTrackbar("num", self.win_depth, 2, 10, lambda x: None)
cv2.createTrackbar("blockSize", self.win_depth, 5, 255, lambda x: None)
self.ecv = image_processing.EventCv()
self.ecv.add_mouse_event(self.win_depth)
self.ecv.add_mouse_event(self.win_disparity)
self.ecv.add_mouse_event(self.win_colormap)
camera_height = camera_configs.camera_height
camera_width = camera_configs.camera_width
input_size = [camera_height, camera_width]
# self.pose = openpose.OpenPose(model_path, input_size=input_size)
if invert_camera:
self.camera1 = cv2.VideoCapture(1) # left camera1
self.camera2 = cv2.VideoCapture(0) # right camera2
else:
self.camera1 = cv2.VideoCapture(0) # left camera1
self.camera2 = cv2.VideoCapture(1) # right camera2
self.detector = UltraLightFaceDetector(model_path=None, network=None, )
self.g = open3d_visual.Open3DVisual(camera_intrinsic=camera_configs.camera_intrinsic,
depth_width=camera_configs.depth_width,
depth_height=camera_configs.camera_height,
depth_scale=camera_configs.depth_scale,
clipping_distance_in_meters=camera_configs.clipping_distance_in_meters)
self.g.show_origin_pcd()
self.g.show_bone_line_pcd()
self.g.show_image_pcd(True)
class_name = "ttt"
real_part = True
# real_part = False
dataset_root = "dataset/"
self.scale = 1
self.prefix = "v"
if real_part:
self.snapshot_dir = os.path.join(dataset_root, "real_part", class_name)
else:
self.snapshot_dir = os.path.join(dataset_root, "fake_part", class_name)
file_processing.create_dir(self.snapshot_dir, "color")
file_processing.create_dir(self.snapshot_dir, "depth")
# file_processing.create_dir(self.snapshot_dir, "ir")
file_processing.create_dir(self.snapshot_dir, "video")
video_name = file_processing.get_time()
self.save_video = os.path.join(self.snapshot_dir, "video", "{}_{}.avi".format(class_name, video_name))
if self.save_video:
self.video_writer = self.get_video_writer(self.save_video,
width=640,
height=480,
fps=20) # (424, 512, 4)
@staticmethod
def get_video_writer(save_path, width, height, fps):
fourcc = cv2.VideoWriter_fourcc(*'XVID')
frameSize = (width, height)
video_writer = cv2.VideoWriter(save_path, fourcc, fps, frameSize)
print("video:width:{},height:{},fps:{}".format(width, height, fps))
return video_writer
def get_rectify_image(self, image_left, image_right):
'''
畸变校正和立体校正
根据更正map对图片进行重构
获取用于畸变校正和立体校正的映射矩阵以及用于计算像素空间坐标的重投影矩阵
:param image_left:
:param image_right:
:return:
'''
left_rectified = cv2.remap(image_left, camera_configs.left_map1, camera_configs.left_map2,
cv2.INTER_LINEAR)
right_rectified = cv2.remap(image_right, camera_configs.right_map1, camera_configs.right_map2,
cv2.INTER_LINEAR)
return left_rectified, right_rectified
def get_disparity_map(self, left_rectified, right_rectified, num=1, blockSize=1, stereo_type="SGBM"):
'''
获得视差图
:param imgL:
:param imgR:
:param num:
:param blockSize:
:param stereo_type
:return:
'''
# 将图片置为灰度图，为StereoBM作准备
imgL = cv2.cvtColor(left_rectified, cv2.COLOR_BGR2GRAY)
imgR = cv2.cvtColor(right_rectified, cv2.COLOR_BGR2GRAY)
# 根据Block Maching方法生成差异图（opencv里也提供了SGBM/Semi-Global Block Matching算法，有兴趣可以试试）
if stereo_type == "BM":
stereo = cv2.StereoBM_create(numDisparities=16 * num, blockSize=blockSize)
# stereo = cv2.StereoBM_create(numDisparities=16, blockSize=15)
elif stereo_type == "SGBM":
# SGBM匹配参数设置
if imgL.ndim == 2:
img_channels = 1
else:
img_channels = 3
# blockSize = 3
num = np.where(num > 0, num, 1)
param = {'minDisparity': 0,
'numDisparities': 16 * num,
'blockSize': blockSize,
'P1': 2 * img_channels * blockSize,
'P2': 4 * img_channels * blockSize,
'disp12MaxDiff': 1,
'preFilterCap': 65,
'uniquenessRatio': 10,
'speckleWindowSize': 150,
'speckleRange': 2,
'mode': cv2.STEREO_SGBM_MODE_SGBM_3WAY
}
# 构建SGBM对象
# print(param)
stereo = cv2.StereoSGBM_create(**param)
# disparity_left = stereo.compute(imgL, imgR) # 计算左视差图
# disparity_right = stereo.compute(imgR, imgL) # 计算右视差图
disparity = stereo.compute(imgL, imgR)
# 除以16得到真实视差（因为SGBM算法得到的视差是×16的）
disparity[disparity < 0] = 0
# disparity.astype(np.float32) / 16.
disparity = np.divide(disparity.astype(np.float32), 16.)
return disparity
# 顺时针旋转90度
def RotateClockWise90(self, img):
trans_img = cv2.transpose(img)
new_img = cv2.flip(trans_img, 1)
return new_img
def get_depth(self, disparity, scale=1.0, method=False):
'''
reprojectImageTo3D(disparity, Q),输入的Q,单位必须是毫米(mm)
:param disparity:
:param only_depth:
:param scale:
:return: returm scale=1.0,距离,单位为毫米
'''
# 将图片扩展至3d空间中，其z方向的值则为当前的距离
if method:
depth = cv2.reprojectImageTo3D(disparity, camera_configs.Q)
x, y, depth = cv2.split(depth)
else:
# baseline = 21.50635
# fx = 419.29128272967574
baseline = abs(camera_configs.T[0])
# fx = abs(camera_configs.left_camera_matrix[0][0])
fx = abs(camera_configs.Q[2, 3])
depth = (fx * baseline) / disparity
depth = depth * scale
depth = np.asarray(depth, dtype=np.uint16)
return depth
def start_capture(self):
snapshot_dir = "data/ov0575"
file_processing.create_dir(snapshot_dir)
self.count = 0
while True:
# 两个trackbar用来调节不同的参数查看效果
num = cv2.getTrackbarPos("num", "depth")
blockSize = cv2.getTrackbarPos("blockSize", "depth")
if blockSize % 2 == 0:
blockSize += 1
if blockSize < 5:
blockSize = 5
# 获取左右摄像头的数据
ret1, frameR = self.camera1.read() # <class 'tuple'>: (480, 640, 3)
ret2, frameL = self.camera2.read() # <class 'tuple'>: (480, 640, 3)
# frameL=self.RotateClockWise90(frameL)
# frameR=self.RotateClockWise90(frameR)
# 畸变校正和立体校正
self.left_rectified, self.right_rectified = self.get_rectify_image(image_left=frameL, image_right=frameR)
# 获得视差图
self.disparity = self.get_disparity_map(self.left_rectified, self.right_rectified, num, blockSize)
# 计算像素点的3D坐标（左相机坐标系下）
self.depth = self.get_depth(self.disparity)
self.depth = np.asarray(self.depth, dtype=np.uint16)
# save_dir = "./data/data01"
# self.left_rectified, self.depth, _ = rgbd_data.load_body_joint_data(save_dir, count=40)
# self.g.show(self.left_rectified, self.depth, body_joint2D, body_rects=body_rects)
# self.g.show(frameL, self.depth, body_joint=None)
bboxes, scores, landms = self.detector.detect(self.left_rectified, isshow=False)
self.show_landmark_boxes("Det", self.left_rectified, bboxes, scores, landms)
self.display(bboxes)
@staticmethod
def show_landmark_boxes(win_name, image, bboxes, scores, landms):
'''
显示landmark和boxes
:param win_name:
:param image:
:param landmarks_list: [[x1, y1], [x2, y2]]
:param bboxes: [[ x1, y1, x2, y2],[ x1, y1, x2, y2]]
:return:
'''
image = image_processing.draw_landmark(image, landms, vis_id=True)
image = image_processing.draw_image_bboxes_text(image, bboxes, scores, color=(0, 0, 255))
cv2.imshow(win_name, image)
def display(self, bboxes):
self.g.show(color_image=self.left_rectified, depth_image=self.depth)
self.ecv.update_image(self.depth)
# disparity_image = cv2.normalize(self.disparity, self.disparity, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX,
# dtype=cv2.CV_8U)
disparity_image = np.uint8(self.disparity.clip(1, 4080) / 16.) # 转换为uint8时，需要避免溢出255*16=4080
depth_colormap = cv2.applyColorMap(
cv2.convertScaleAbs(self.depth, alpha=255 / camera_configs.clipping_distance),
cv2.COLORMAP_JET)
cv2.imshow(self.win_left, self.left_rectified)
cv2.imshow(self.win_right, self.right_rectified)
cv2.imshow(self.win_disparity, disparity_image)
cv2.imshow(self.win_depth, self.depth)
cv2.imshow(self.win_colormap, depth_colormap)
cv2.waitKey(20)
self.count += 1
# self.flag = cv2.waitKey(20) & 0xFF == ord('s')
freq = 3
self.save_videos(self.left_rectified, bboxes)
if self.snapshot_dir and len(bboxes) > 0 and self.count % freq == 0:
print("save image:{}".format(self.count))
# self.flag = True
# pre = "{}.png".format(self.count)
prefix = "{}_{}.png".format(self.prefix, self.count)
cv2.imwrite(os.path.join(self.snapshot_dir, "depth/{}".format(prefix)), self.depth)
cv2.imwrite(os.path.join(self.snapshot_dir, "color/{}".format(prefix)), self.left_rectified)
# cv2.imwrite(os.path.join(self.snapshot_dir, "ir/{}".format(prefix)), ir_frame)
def save_videos(self, color_img, bboxes):
"""
<class 'tuple'>: (480, 640, 3)
:param color_img:
:param bboxes:
:return:
"""
if self.save_video and len(bboxes) > 0:
# image_processing.cv_show_image("color_img", color_img, waitKey=1)
# color_img = cv2.cvtColor(color_img, cv2.COLOR_RGBA2RGB) # 将BGR转为RGB
self.video_writer.write(color_img)
def close(self):
self.camera1.release()
self.camera2.release()
cv2.destroyAllWindows()
class Kinect2PointCloud():
def __init__(self):
# Kinect runtime object
self.joint_count = PyKinectV2.JointType_Count # 25
self.kinect = PyKinectRuntime.PyKinectRuntime(
PyKinectV2.FrameSourceTypes_Body
| PyKinectV2.FrameSourceTypes_Color
| PyKinectV2.FrameSourceTypes_Infrared
| PyKinectV2.FrameSourceTypes_Depth)
self.depth_width, self.depth_height = self.kinect.depth_frame_desc.Width, self.kinect.depth_frame_desc.Height
self.color_width, self.color_height = self.kinect.color_frame_desc.Width, self.kinect.color_frame_desc.Height
self.ir_width, self.ir_height = self.kinect.infrared_frame_desc.Width, self.kinect.infrared_frame_desc.Height
self.g = geometry_3d_pose.Geometry3DPose(kinect_config)
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
class_name = "0"
# real_part = True
real_part = False
dataset_root = "dataset/"
self.scale = 2
self.prefix = "v1"
if real_part:
self.snapshot_dir = os.path.join(dataset_root, "real_part",
class_name)
else:
self.snapshot_dir = os.path.join(dataset_root, "fake_part",
class_name)
file_processing.create_dir(self.snapshot_dir, "color")
file_processing.create_dir(self.snapshot_dir, "depth")
file_processing.create_dir(self.snapshot_dir, "ir")
file_processing.create_dir(self.snapshot_dir, "video")
video_name = file_processing.get_time()
self.save_video = os.path.join(
self.snapshot_dir, "video",
"{}_{}.avi".format(class_name, video_name))
if self.save_video:
self.video_writer = self.get_video_writer(
self.save_video,
width=self.depth_width * self.scale,
height=self.depth_height * self.scale,
fps=20) # (424, 512, 4)
@staticmethod
def get_video_writer(save_path, width, height, fps):
fourcc = cv2.VideoWriter_fourcc(*'XVID')
frameSize = (width, height)
video_writer = cv2.VideoWriter(save_path, fourcc, fps, frameSize)
print("video:width:{},height:{},fps:{}".format(width, height, fps))
return video_writer
def start_capture(self):
self.g.show_origin_pcd()
self.g.show_bone_line_pcd()
self.g.show_image_pcd(False)
self.g.show_desktop_pcd(True)
self.count = 0
while True:
# Get images from camera
if self.kinect.has_new_body_frame(
) and self.kinect.has_new_color_frame(
) and self.kinect.has_new_depth_frame():
body_frame = self.kinect.get_last_body_frame()
ir_frame = self.kinect.get_last_infrared_frame()
color_frame = self.kinect.get_last_color_frame()
depth_frame = self.kinect.get_last_depth_frame()
# Reshape from 1D frame to 2D image
color_img = color_frame.reshape(
((self.color_height, self.color_width,
4))).astype(np.uint8)
ir_frame = self.get_infrared_frame(ir_frame)
self.ir_frame1 = np.asarray(ir_frame / 2, np.uint8)
depth_img = depth_frame.reshape(
((self.depth_height, self.depth_width))).astype(np.uint16)
align_color_img = utils.get_align_color_image(
self.kinect, color_img)
bgr_image = cv2.cvtColor(align_color_img, cv2.COLOR_RGBA2BGR)
bboxes, scores, landms = self.detector.detect(bgr_image,
isshow=False)
self.show_landmark_boxes("Det", align_color_img, bboxes,
scores, landms)
# body_joint2D, body_orientation = utils.get_body_joint2D(body_frame,
# self.kinect,
# map_space="depth_space")
# self.g.show(align_color_img, depth_img, body_joint2D,joint_count=PyKinectV2.JointType_Count)
# self.g.show(align_color_img, depth_img, body_joint2D)
self.show(color_img, align_color_img, depth_img, ir_frame,
bboxes)
def get_infrared_frame(self, ir_frame):
"""
:param ir_frame:
:return:
"""
ir_frame = ir_frame.reshape(((self.ir_height, self.ir_width)))
ir_frame = np.uint8(ir_frame.clip(1, 4000) / 16.)
# ir_frame = np.uint8(ir_frame.clip(1, 4000))
ir_frame = np.dstack((ir_frame, ir_frame, ir_frame))
# ir_frame = img_as_float(ir_frame)
# ir_frame = exposure.adjust_gamma(image, 0.5) # 调亮
return ir_frame
@staticmethod
def show_landmark_boxes(win_name, image, bboxes, scores, landms):
'''
显示landmark和boxes
:param win_name:
:param image:
:param landmarks_list: [[x1, y1], [x2, y2]]
:param bboxes: [[ x1, y1, x2, y2],[ x1, y1, x2, y2]]
:return:
'''
image = image_processing.draw_landmark(image, landms, vis_id=True)
image = image_processing.draw_image_bboxes_text(image,
bboxes,
scores,
color=(0, 0, 255))
image_processing.cv_show_image(win_name, image, waitKey=1)
def show(self, color_img, align_color_img, depth_img, ir_frame, bboxes):
"""
:param color_img: <class 'tuple'>: (1080, 1920, 4),uint8
:param align_color_img: <class 'tuple'>: (424, 512, 4),uint8
:param depth_img: <class 'tuple'>: (424, 512),uint16
:param ir_frame: <class 'tuple'>: (424, 512, 3),uint8
:return:
"""
# align_color_img = cv2.cvtColor(align_color_img, cv2.COLOR_RGBA2RGB) # 将BGR转为RGB
image_processing.addMouseCallback("depth_img", param=depth_img)
image_processing.addMouseCallback("align_color_img", param=depth_img)
# image_processing.cv_show_image("color_img", color_img, waitKey=1)
image_processing.cv_show_image("align_color_img",
align_color_img,
waitKey=1)
image_processing.cv_show_image("depth_img", depth_img, waitKey=1)
image_processing.cv_show_image("ir_frame", ir_frame, waitKey=1)
image_processing.cv_show_image("ir_frame1", self.ir_frame1, waitKey=1)
self.count += 1
freq = 3
# self.flag = cv2.waitKey(20) & 0xFF == ord('s')
if self.snapshot_dir and len(bboxes) > 0 and self.count % freq == 0:
print("save image:{}".format(self.count))
# self.flag = True
# pre = "{}.png".format(self.count)
prefix = "{}_{}.png".format(self.prefix, self.count)
cv2.imwrite(
os.path.join(self.snapshot_dir, "depth/{}".format(prefix)),
depth_img)
cv2.imwrite(
os.path.join(self.snapshot_dir, "color/{}".format(prefix)),
align_color_img)
cv2.imwrite(
os.path.join(self.snapshot_dir, "ir/{}".format(prefix)),
ir_frame)
cv2.waitKey(20)
self.save_videos(color_img, align_color_img, depth_img, ir_frame,
bboxes)
def save_videos(self, color_img, align_color_img, depth_img, ir_frame,
bboxes):
"""
(424, 512, 4)
:param color_img:
:param align_color_img:
:param depth_img:
:param ir_frame:
:param bboxes:
:return:
"""
if self.save_video and len(bboxes) > 0:
color_img = image_processing.resize_image(
color_img, resize_height=self.depth_height * self.scale)
# <class 'tuple'>: (424, 753, 4)
# <class 'tuple'>: (848, 1024, 3)
color_img = image_processing.center_crop(
color_img,
crop_size=[
self.depth_height * self.scale,
self.depth_width * self.scale
])
# image_processing.cv_show_image("color_img", color_img, waitKey=1)
color_img = cv2.cvtColor(color_img,
cv2.COLOR_RGBA2RGB) # 将BGR转为RGB
self.video_writer.write(color_img)
def close(self):
self.kinect.close()
self.g.close()
class CVVideo():
def __init__(self):
model_path = "/media/dm/dm1/git/python-learning-notes/libs/ultra_ligh_face/face_detection_rbf.pth"
network = "RFB"
confidence_threshold = 0.85
nms_threshold = 0.3
top_k = 500
keep_top_k = 750
device = "cuda:0"
self.detector = UltraLightFaceDetector(model_path=model_path,
network=network,
confidence_threshold=confidence_threshold,
nms_threshold=nms_threshold,
top_k=top_k,
keep_top_k=keep_top_k,
device=device)
def start_capture(self, video_path, save_video=None, detect_freq=1):
"""
start capture video
:param video_path: *.avi,*.mp4,...
:param save_video: *.avi
:param detect_freq:
:return:
"""
video_cap = image_processing.get_video_capture(video_path,width=640, height=480)
width, height, numFrames, fps = image_processing.get_video_info(video_cap)
if save_video:
self.video_writer = image_processing.get_video_writer(save_video, width, height, fps)
# freq = int(fps / detect_freq)
count = 0
while True:
isSuccess, frame = video_cap.read()
if not isSuccess:
break
if count % detect_freq == 0:
# frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
out_frame = self.do_something(frame)
if save_video:
self.write_video(out_frame)
count += 1
video_cap.release()
def write_video(self, img):
self.video_writer.write(img)
# def do_something(self, frame):
# frame = np.rot90(frame, -1)
# h, w, d = frame.shape
# resize_height = int(h / 2)
# frame = image_processing.resize_image(frame, resize_height=resize_height)
# bboxes_list = [[0, 200, w, h]]
# frame = image_processing.get_bboxes_image(frame, bboxes_list)[0]
# out_frame = np.asarray(frame)
# cv2.imshow("image", out_frame)
# cv2.imwrite("image.png",out_frame)
# cv2.waitKey(0)
# return out_frame
def do_something(self, frame):
bgr_image = frame.copy()
bboxes, scores, landms = self.detector.detect(bgr_image, isshow=False)
bgr_image = image_processing.draw_landmark(bgr_image, landms, vis_id=True)
bgr_image = image_processing.draw_image_bboxes_text(bgr_image, bboxes, scores, color=(0, 0, 255))
cv2.imshow("image", bgr_image)
cv2.moveWindow("image", 0, 0)
cv2.waitKey(10)
return frame
class StereoDepth(object):
"""
"""
def __init__(self, calibration_file, width=640, height=480):
"""
:param calibration_file:
:param width:
:param height:
"""
self.config = camera_params.get_stereo_coefficients(
calibration_file, width, height)
self.pcd = open3d_visual.Open3DVisual(
camera_intrinsic=self.config["K1"],
depth_width=width,
depth_height=height)
self.detector = UltraLightFaceDetector(
model_path=None,
network=None,
)
class_name = "ttt"
real_part = True
# real_part = False
save_root = "dataset/"
self.scale = 1
self.prefix = "v"
if real_part:
self.save_dir = os.path.join(save_root, "real_part", class_name)
else:
self.save_dir = os.path.join(save_root, "fake_part", class_name)
file_processing.create_dir(self.save_dir, "color")
file_processing.create_dir(self.save_dir, "depth")
file_processing.create_dir(self.save_dir, "ir")
file_processing.create_dir(self.save_dir, "video")
video_name = file_processing.get_time()
self.save_l_video = os.path.join(
self.save_dir, "video",
"left_{}_{}.avi".format(class_name, video_name))
self.save_r_video = os.path.join(
self.save_dir, "video",
"right_{}_{}.avi".format(class_name, video_name))
if self.save_l_video:
self.video_l_writer = self.get_video_writer(self.save_l_video,
width=width,
height=height,
fps=30)
if self.save_r_video:
self.video_r_writer = self.get_video_writer(self.save_r_video,
width=width,
height=height,
fps=30)
self.pcd.show_image_pcd(True)
self.pcd.show_origin_pcd(True)
self.pcd.show_image_pcd(True)
def capture(self, left_source, right_source):
cap_left = cv2.VideoCapture(left_source)
cap_right = cv2.VideoCapture(right_source)
if not cap_left.isOpened() and not cap_right.isOpened(
): # If we can't get images from both sources, error
print("Can't opened the streams!")
sys.exit(-9)
# Change the resolution in need
cap_right.set(cv2.CAP_PROP_FRAME_WIDTH, 640) # float
cap_right.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) # float
cap_left.set(cv2.CAP_PROP_FRAME_WIDTH, 640) # float
cap_left.set(cv2.CAP_PROP_FRAME_HEIGHT, 480) # float
width, height, numFrames, fps = self.get_video_info(cap_left)
self.count = 1
while True: # Loop until 'q' pressed or stream ends
# Grab&retreive for sync images
if not (cap_left.grab() and cap_right.grab()):
print("No more frames")
break
self.count += 1
self.task(cap_left, cap_right)
if cv2.waitKey(1) & 0xFF == ord(
'q'): # Get key to stop stream. Press q for exit
break
# Release the sources.
cap_left.release()
cap_right.release()
cv2.destroyAllWindows()
@staticmethod
def get_video_info(video_cap):
width = int(video_cap.get(cv2.CAP_PROP_FRAME_WIDTH))
height = int(video_cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
numFrames = int(video_cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = int(video_cap.get(cv2.CAP_PROP_FPS))
print("video:width:{},height:{},fps:{}".format(width, height, fps))
return width, height, numFrames, fps
@staticmethod
def get_video_writer(save_path, width, height, fps):
fourcc = cv2.VideoWriter_fourcc(*'XVID')
frameSize = (width, height)
video_writer = cv2.VideoWriter(save_path, fourcc, fps, frameSize)
print("video:width:{},height:{},fps:{}".format(width, height, fps))
return video_writer
def get_depth(self, disparity, scale=1.0, method=True):
'''
reprojectImageTo3D(disparity, Q),输入的Q,单位必须是毫米(mm)
:param disparity:
:param only_depth:
:param scale:
:return: returm scale=1.0,距离,单位为毫米
'''
# 将图片扩展至3d空间中，其z方向的值则为当前的距离
if method:
depth = cv2.reprojectImageTo3D(disparity,
self.config["Q"]) # 单位必须是毫米(mm)
x, y, depth = cv2.split(depth)
else:
# baseline = 21.50635
# fx = 419.29128272967574
baseline = abs(self.config["T"][0])
# fx = abs(self.config["K1"][0, 0])
fx = abs(self.config["Q"][2, 3])
depth = (fx * baseline) / disparity
depth = depth * scale
depth = np.asarray(depth, dtype=np.uint16)
return depth
@staticmethod
def get_depth_colormap(depth, clipping_distance=1500):
depth = np.clip(depth, 0, clipping_distance)
depth_colormap = cv2.applyColorMap(
# cv2.convertScaleAbs(depth, alpha=255 / clipping_distance),
cv2.convertScaleAbs(depth, alpha=1),
cv2.COLORMAP_JET)
return depth_colormap
def get_disparity(self, imgL, imgR):
""" Depth map calculation. Works with SGBM and WLS. Need rectified images, returns depth map ( left to right disparity ) """
# SGBM Parameters -----------------
window_size = 1 # wsize default 3; 5; 7 for SGBM reduced size image; 15 for SGBM full size image (1300px and above); 5 Works nicely
param = {
'minDisparity': 0,
'numDisparities': 32,
'blockSize': 5,
'P1': 10,
'P2': 20,
'disp12MaxDiff': 1,
'preFilterCap': 65,
'uniquenessRatio': 10,
'speckleWindowSize': 150,
'speckleRange': 2,
'mode': 2
}
left_matcher = cv2.StereoSGBM_create(**param)
# left_matcher = cv2.StereoSGBM_create(
# minDisparity=-1,
# numDisparities=5*16, # max_disp has to be dividable by 16 f. E. HH 192, 256
# blockSize=window_size,
# P1=8 * 3 * window_size,
# # wsize default 3; 5; 7 for SGBM reduced size image; 15 for SGBM full size image (1300px and above); 5 Works nicely
# P2=32 * 3 * window_size,
# disp12MaxDiff=12,
# uniquenessRatio=10,
# speckleWindowSize=50,
# speckleRange=32,
# preFilterCap=63,
# mode=cv2.STEREO_SGBM_MODE_SGBM_3WAY
# )
right_matcher = cv2.ximgproc.createRightMatcher(left_matcher)
# FILTER Parameters
lmbda = 8000
sigma = 1.3
wls_filter = cv2.ximgproc.createDisparityWLSFilter(
matcher_left=left_matcher)
wls_filter.setLambda(lmbda)
wls_filter.setSigmaColor(sigma)
displ = left_matcher.compute(imgL, imgR) # .astype(np.float32)/16
dispr = right_matcher.compute(imgR, imgL) # .astype(np.float32)/16
displ = np.int16(displ)
dispr = np.int16(dispr)
filteredImg = wls_filter.filter(
displ, imgL, None, dispr) # important to put "imgL" here!!!
filteredImg = cv2.normalize(src=filteredImg,
dst=filteredImg,
beta=0,
alpha=255,
norm_type=cv2.NORM_MINMAX)
filteredImg = np.uint8(filteredImg)
# 除以16得到真实视差（因为SGBM算法得到的视差是×16的）
displ[displ < 0] = 0
# disparity.astype(np.float32) / 16.
displ = np.divide(displ.astype(np.float32), 16.)
return filteredImg, displ
def get_rectify_image(self, l_frame, r_frame):
'''
畸变校正和立体校正
根据更正map对图片进行重构
获取用于畸变校正和立体校正的映射矩阵以及用于计算像素空间坐标的重投影矩阵
:param l_frame:
:param r_frame:
:return:
'''
# left_rectified = cv2.remap(l_frame,
# self.config["left_map_x"],
# self.config["left_map_y"],
# cv2.INTER_LINEAR)
# right_rectified = cv2.remap(r_frame,
# self.config["right_map_x"],
# self.config["right_map_y"],
# cv2.INTER_LINEAR)
left_rectified = cv2.remap(l_frame, self.config["left_map_x"],
self.config["left_map_y"], cv2.INTER_LINEAR,
cv2.BORDER_CONSTANT)
right_rectified = cv2.remap(r_frame, self.config["right_map_x"],
self.config["right_map_y"],
cv2.INTER_LINEAR, cv2.BORDER_CONSTANT)
return left_rectified, right_rectified
def task(self, cap_left, cap_right):
"""
:param cap_left:
:param cap_right:
:return:
"""
_, l_frame = cap_left.retrieve()
_, r_frame = cap_right.retrieve()
# 畸变校正和立体校正
left_rectified, right_rectified = self.get_rectify_image(
l_frame=l_frame, r_frame=r_frame)
# We need grayscale for disparity map.
gray_left = cv2.cvtColor(left_rectified, cv2.COLOR_BGR2GRAY)
gray_right = cv2.cvtColor(right_rectified, cv2.COLOR_BGR2GRAY)
# Get the disparity map
disparity_image, displ = self.get_disparity(gray_left, gray_right)
depth = self.get_depth(disparity=disparity_image)
bboxes, scores, landms = self.detector.detect(l_frame, isshow=False)
self.show(l_frame, r_frame, disparity_image, depth, bboxes, scores)
self.pcd.show(color_image=l_frame, depth_image=depth)
def show(self, l_frame, r_frame, disparity_image, depth, bboxes, scores):
"""
:param l_frame:
:param r_frame:
:param disparity_image:
:param depth:
:return:
"""
depth_colormap = self.get_depth_colormap(depth, clipping_distance=4500)
image_processing.addMouseCallback("depth_colormap", depth)
image_processing.addMouseCallback("left", depth)
image_processing.addMouseCallback("Disparity", disparity_image)
det_img = image_processing.draw_image_bboxes_text(l_frame,
bboxes,
scores,
color=(0, 0, 255))
# Show the images
cv2.imshow('det_img', det_img)
cv2.imshow('left', l_frame)
cv2.imshow('right', r_frame)
cv2.imshow('Disparity', disparity_image)
cv2.imshow('depth_colormap', depth_colormap)
if self.count <= 2:
cv2.moveWindow("det_img", 0, 0)
cv2.moveWindow("left", 700, 0)
cv2.moveWindow("right", 1400, 0)
cv2.moveWindow("Disparity", 0, 600)
cv2.moveWindow("depth_colormap", 700, 600)
cv2.waitKey(10)
self.save(l_img=l_frame, r_img=r_frame, depth_img=depth, bboxes=bboxes)
def save(self, l_img, r_img, depth_img, bboxes, freq=3):
if self.save_dir and len(bboxes) > 0 and self.count % freq == 0:
print("save image:{}".format(self.count))
prefix = "{}_{}.png".format(self.prefix, self.count)
cv2.imwrite(os.path.join(self.save_dir, "depth/{}".format(prefix)),
depth_img)
cv2.imwrite(os.path.join(self.save_dir, "color/{}".format(prefix)),
l_img)
cv2.imwrite(os.path.join(self.save_dir, "ir/{}".format(prefix)),
r_img)
if self.save_l_video and len(bboxes) > 0:
# image_processing.cv_show_image("color_img", color_img, waitKey=1)
# color_img = cv2.cvtColor(color_img, cv2.COLOR_RGBA2RGB) # 将BGR转为RGB
self.video_l_writer.write(l_img)
if self.save_r_video and len(bboxes) > 0:
# image_processing.cv_show_image("color_img", color_img, waitKey=1)
# color_img = cv2.cvtColor(color_img, cv2.COLOR_RGBA2RGB) # 将BGR转为RGB
self.video_r_writer.write(r_img)