def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imwrite('/Users/Utente/Desktop/pose/out.jpg', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 0 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses print("draw", img.dtype, img.shape, img.min(), img.max()) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) print(img.min(), img.max()) for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu:#run the model "net" in cuda if not specified to run specifically in CPU # gets in here since by default store true is false. Hence uses GPU for inference by default net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) # cv2.resize(img,(720,1280)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', cv2.resize(img,(2800,1800))) key = cv2.waitKey(33) if key == 27: # esc return
def __call__(self, img, height_size=256): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(self.model, img, height_size, stride, upsample_ratio, False) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imwrite("/data1/qilei_chen/DEVELOPMENTS/test1.jpg", img) return current_poses
def visualize_prediction(self, image): orig_img = image.copy() if not np.array_equal(self.image, image): self.image = image self.__inference_fast(self.image, self.height_size, self.stride, self.upsample_ratio) current_poses = [] for n in range(len(self.pose_entries)): if len(self.pose_entries[n]) == 0: continue pose_keypoints = np.ones( (self.num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(self.num_keypoints): if self.pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( self.all_keypoints[int(self.pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( self.all_keypoints[int(self.pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, self.pose_entries[n][18]) current_poses.append(pose) # if self.track: # previous_poses = [] # track_poses(previous_poses, current_poses, smooth=smooth) # previous_poses = current_poses for pose in current_poses: pose.draw(image) image = cv2.addWeighted(orig_img, 0.6, image, 0.4, 0) # plt.imshow(image) # plt.show() return image
def run_demo(net, height_size, cpu, track, smooth, image_provider=None, use_realsense_cam=False, openvino=False): if not image_provider and not use_realsense_cam: raise ValueError('Either `image_provider` or `use_realsense_cam` must be provided') if use_realsense_cam: pipeline = init_realsense() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 while True: # get frame from source provided if use_realsense_cam: frames = pipeline.wait_for_frames() color_frame = frames.get_color_frame() img = np.asanyarray(color_frame.get_data()) else: img = next(image_provider) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu, openvino=openvino) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids, arm): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] stateMachines = {} for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) """ kpt_names = ['nose', 'neck', 'r_sho', 'r_elb', 'r_wri', 'l_sho', 'l_elb', 'l_wri', 'r_hip', 'r_knee', 'r_ank', 'l_hip', 'l_knee', 'l_ank', 'r_eye', 'l_eye', 'r_ear', 'l_ear'] r_elb-3, r-wri-4, l_elb-6, l_wri-7 """ # print('ID: {}'.format(n)) # print('\tRight elbow: {}, right wrist: {}'.format(pose_keypoints[3], pose_keypoints[4])) # print('\tLeft elbow: {}, left wrist: {}'.format(pose_keypoints[6], pose_keypoints[7])) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses, threshold=3) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) for pose in current_poses: if pose.id not in stateMachines.keys(): stateMachines[pose.id] = StateMachine(pose.id, pose.keypoints, arm) #print('ID {} detected'.format(pose.id)) continue # call stateMachine methods stateMachines[pose.id].update(pose.keypoints, img) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 mean_time = 0 video_time = 0 prep_displacement = 0 prep_time = 0 for img in image_provider: tik1 = cv2.getTickCount() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if (pose.bbox[0] + pose.bbox[2] / 2 > 355 and (pose.bbox[0] + pose.bbox[2] / 2 < 555)): current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.rectangle(img, (330, 330), (430, 230), (0, 0, 255), 2) prev_pose = [] for pose in current_poses: if track: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if (pose.keypoints[4][0] > 330 and pose.keypoints[4][0] < 430) and (pose.keypoints[4][1] > 230 and pose.keypoints[4][1] < 330) and \ (pose.keypoints[7][0] > 330 and pose.keypoints[7][0] < 430) and (pose.keypoints[7][1] > 230 and pose.keypoints[7][1] < 330): cv2.rectangle( img, (pose.keypoints[4][0] - 10, pose.keypoints[4][1] - 10), (pose.keypoints[4][0] + 10, pose.keypoints[4][1] + 10), (0, 255, 0), 2) cv2.rectangle( img, (pose.keypoints[7][0] - 10, pose.keypoints[7][1] - 10), (pose.keypoints[7][0] + 10, pose.keypoints[7][1] + 10), (0, 255, 0), 2) if bool(previous_poses): prev_pose = previous_poses[0] prep_displacement += ( (pose.keypoints[4][0] - prev_pose.keypoints[4][0])**2 + (pose.keypoints[4][1] - prev_pose.keypoints[4][1])** 2)**0.5 prep_displacement += ( (pose.keypoints[7][0] - prev_pose.keypoints[7][0])**2 + (pose.keypoints[7][1] - prev_pose.keypoints[7][1])** 2)**0.5 prep_time += (cv2.getTickCount() - tik1) / cv2.getTickFrequency() previous_poses = current_poses current_time = (cv2.getTickCount() - tik1) / cv2.getTickFrequency() video_time += current_time cv2.putText(img, "displacement: %d" % int(prep_displacement), (20, 20), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1) cv2.putText(img, "prep time (s): %.1f" % (prep_time / 2), (20, 40), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1) cv2.putText(img, "video time (s): %.1f" % (video_time / 2), (20, 60), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1) if mean_time == 0: mean_time = current_time else: mean_time = mean_time * 0.95 + current_time * 0.05 #cv2.putText(img, 'FPS: {}'.format(int(1 / mean_time * 10) / 10), # (40, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) cv2.imshow('Chicken Prep Test Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 1: delay = 0 else: delay = 1
def run_demo(net, action_net, image_provider, height_size, cpu, boxList): net = net.eval() print(torch.cuda.device_count()) print(torch.cuda.is_available()) a = torch.Tensor(5, 3) a = a.cuda() print(a) if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18 i = 0 for img in image_provider: # 遍历图像集 orig_img = img.copy() # copy 一份 # print(i) fallFlag = 0 if i % 1 == 0: heatmaps, pafs, scale, pad = infer_fast( net, img, height_size, stride, upsample_ratio, cpu) # 返回热图,paf,输入模型图象相比原始图像缩放倍数,输入模型图像padding尺寸 total_keypoints_num = 0 all_keypoints_by_type = [ ] # all_keypoints_by_type为18个list,每个list包含Ni个当前点的x、y坐标,当前点热图值,当前点在所有特征点中的index for kpt_idx in range( num_keypoints): # 19th for bg 第19个为背景,之考虑前18个关节点 total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True ) # 得到所有分配的人(前18维为每个人各个关节点在所有关节点中的索引,后两唯为每个人得分及每个人关节点数量),及所有关节点信息 for kpt_id in range(all_keypoints.shape[0]): # 依次将每个关节点信息缩放回原始图像上 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): # 依次遍历找到的每个人 if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) posebox = (int(pose.bbox[0]), int(pose.bbox[1]), int(pose.bbox[0]) + int(pose.bbox[2]), int(pose.bbox[1]) + int(pose.bbox[3])) coincideValue = coincide(boxList, posebox) print(posebox) print('coincideValue:' + str(coincideValue)) if len( pose.getKeyPoints() ) >= 10 and coincideValue >= 0.3 and pose.lowerHalfFlag < 3: # 当人体的点数大于10个的时候算作一个人,同时判断yolov5的框和pose的框是否有交集并且占比30%,同时要有下半身 current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img, is_save=True, show_draw=True) crown_proportion = pose.bbox[2] / pose.bbox[3] #宽高比 pose = action_detect(action_net, pose, crown_proportion) #判断摔倒还是正常 if pose.pose_action == 'fall': cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 0, 255), thickness=3) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) fallFlag = 1 else: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0)) # fallFlag = 1 if fallFlag == 1: t = time.time() # cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img) print('我保存照片了') img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # 保存识别后的照片 # cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img) # print('我保存照片了') # cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1) i += 1 cv2.destroyAllWindows()
def run_demo(net,action_net, image_provider, height_size, cpu): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts i = 0 for img in image_provider: orig_img = img.copy() # print(i) if i % 1 == 0: heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if len(pose.getKeyPoints()) >= 10: current_poses.append(pose) # current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img,show_draw=True) crown_proportion = pose.bbox[2]/pose.bbox[3] #宽高比 pose = action_detect(action_net,pose,crown_proportion) if pose.pose_action == 'fall': cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 0, 255),thickness=3) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) else: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0)) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1) i += 1 cv2.destroyAllWindows()
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) # import ipdb; ipdb.set_trace() # print(current_poses) current_poses[-1].keypoints if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow("Lightweight Human Pose Estimation Python Demo", img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def anime_frame(rgb, env, size=None, useSigmod=False, useTwice=False): if env is None: # return init_pose("./checkpoint_iter_370000.pth") # return init_pose("./default_checkpoints/R.pth") return init_pose("./refine4_checkpoints/checkpoint_iter_14000.pth") net, previous_poses = env stride = 8 upsample_ratio = 4 heatmaps, pafs, scale, pad = infer_fast(net, rgb, 368, stride, upsample_ratio) num_keypoints = Pose.num_kpts total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[ 1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[ 0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) # if track: track_poses(previous_poses, current_poses, smooth=True) previous_poses = current_poses env[1] = previous_poses # return rgb, env # print(rgb.min(), rgb.max()) # img = rgb.squeeze(0).permute(1, 2, 0).cpu().numpy()[:, :, ::-1]# * 255 # img = img.squeeze(0).permute(1, 2, 0).cpu().numpy()[:, :, ::-1]# * 255 # img += 0.5 # img *= 255 # img = img.astype(np.uint8) img = np.zeros((rgb.shape[2], rgb.shape[3], rgb.shape[1]), dtype=np.uint8) show_info = True for pose in current_poses: pose.draw(img, show_info) show_info = False # break # for pose in current_poses: # cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), # (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) # if track: # cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), # cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) img = img[:, :, ::-1] #/ 255 # print(img.shape, img.dtype, img.min(), img.max()) img = torch.FloatTensor(img.astype(np.float32)) img /= 255 img = img.permute(2, 0, 1).unsqueeze(0).cuda() output = rgb * 0.6 + img * 0.4 # output = img return output, env
def run_demo(net, image_provider, height_size=256, cpu=False, track_ids=False): net = net.eval() if not cpu: net = net.cuda() print("use cuda") stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18 #Initialize previous_pose_kpts = [] graph_x, graph_y = [], [] result = [-1, -1, -1, -1, -1] count = 0 start_frame, end_frame = 1000000, -1 completed_half = False total_len_frame = 0 one_cycle_kpts = [] for i, img in enumerate(image_provider): img = cv2.resize(img, (600, 600)) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) #total_keypoints_num = 18 pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) #### pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) #Select joints pose_keypoints = np.concatenate( (pose_keypoints[2], pose_keypoints[5], pose_keypoints[8], pose_keypoints[10], pose_keypoints[11], pose_keypoints[13])).reshape(-1, 2) #Analyze posture previous_pose_kpts.append(pose_keypoints) liftoneleg = LiftOneLeg(previous_pose_kpts) #Wrong angle, leg_status = liftoneleg.check_leg_up_down() #Update status and count leg_status, completed_half, count_update, start_frame_update, end_frame_update= \ liftoneleg.count_repetition(angle, leg_status, completed_half, count, i, start_frame, end_frame) if (count_update == count + 1): print("count : %d" % count) one_cycle_kpts.append(previous_pose_kpts[start_frame:]) result = test_per_frame( previous_pose_kpts[start_frame - total_len_frame:end_frame - total_len_frame], LABEL) total_len_frame += len(previous_pose_kpts) previous_pose_kpts = [] count, start_frame, end_frame = count_update, start_frame_update, end_frame_update #To plot angle graph if int(angle) != 90: graph_x.append(i) graph_y.append(angle) #Put text on the screen cv2.putText(img, 'count : {}'.format(count), (10, 520), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 2) cv2.putText(img, "Rsho-Lsho :%3.2f" % (result[0]), (10, 550), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) cv2.putText( img, "Lsho-Lhip :%3.2f, Lhip-Lank :%3.2f" % (result[1], result[2]), (10, 570), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) cv2.putText(img, "Rhip-Rank :%3.2f" % (result[3]), (10, 590), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) cv2.putText( img, '3 align :{}'.format(liftoneleg.check_if_3points_are_aligned()), (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 2) cv2.putText( img, 'shoulder :{}'.format(liftoneleg.check_if_shoulders_are_aligned()), (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 2) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return return graph_x, graph_y
def run(net, image_provider, height_size, cpu, track, smooth): ts1 = time.time() net = setup(net, cpu) stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 ts2 = time.time() print("setup took ", ts2 - ts1, " s") print(ts2) print(ts1) for img in image_provider: ts1 = time.time() orig_img = img.copy() print('Original Dimensions : ', orig_img.shape) ts_netw1 = time.time() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) ts_netw2 = time.time() print("network time", (ts_netw2 - ts_netw1) * 1000, "ms") total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) ts2 = time.time() cv2.putText(img, '%.2fms' % ((ts2 - ts1) * 1000.0), (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 1: delay = 0 else: delay = 1
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) # Debug code for j in range(0, 16): heatmap = heatmaps[:,:,j] heatmap = heatmap.reshape((128,128,1)) heatmapimg = np.array(heatmap * 255, dtype = np.uint8) heatmap = cv2.applyColorMap(heatmapimg, cv2.COLORMAP_JET) #heatmap = heatmap/255 cv2.imwrite('hmtestpadh_'+str(j)+'.jpg', heatmap) #h_img = TF.to_pil_image(heatmaps) #h_img.save('img_heatmap', 'JPEG') print('heatmaps shape: {}'.format(heatmaps.shape)) np.savetxt('heatmaps.txt', heatmaps.reshape(-1)) rows, cols, depths = np.nonzero(pafs) print(pafs[rows, cols, depths]) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) # Debug code print('total_keypoints_num: {} {} {}\n'.format(all_keypoints_by_type, total_keypoints_num, kpt_idx)) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(0) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 dict_id_color = {} for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses dict_id_color_r = {} for id_ in dict_id_color.keys(): flag_track = False for pose in current_poses: if id_ == pose.id: flag_track = True break if flag_track: dict_id_color_r[pose.id] = dict_id_color[pose.id] dict_id_color = dict_id_color_r track_thr = 0 for pose in current_poses: # print('pose.id : ',pose.id) if pose.id not in dict_id_color.keys(): R_ = random.randint(30,255) G_ = random.randint(30,255) B_ = random.randint(30,255) dict_id_color[pose.id] = [[B_,G_,R_],1] else: dict_id_color[pose.id][1] += 1 if dict_id_color[pose.id][1]>track_thr: pose.draw(img,color_x = dict_id_color[pose.id][0]) img = cv2.addWeighted(orig_img, 0.3, img, 0.7, 0) for pose in current_poses: if dict_id_color[pose.id][1]>track_thr: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0),4) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.namedWindow('Lightweight Human Pose Estimation Python Demo', 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids): # , filename): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18개 previous_poses = [] c = 0 ttt = 0 idxx = 0 csv_dict = { 'frame_number': [], 'driver_index': [], 'is_driver_flag': [] } #,'state' : []} driver_find_flag = False weird_state_flag = False extractor = Extractor('default_checkpoints/ckpt.t7', True) find_class = Find_assault(extractor) for img in image_provider: is_driver_flag = False t5 = time.time() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): ##사이즈 변환 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # 운전자를 못 찾았으면 find_driver 에 들어감. if driver_find_flag is False: driver_find_flag, find_driver_count, find_state = find_class.find_driver( current_poses, orig_img) cv2.putText(img, "Driver_find_count : " + str(find_driver_count), (0, 20), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) cv2.putText(img, "State : Find_Driver", (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) else: # print("idxx : ", idxx) is_driver_flag, driver_index, weird_state_count, weird_state_flag = find_class.is_driver( current_poses, orig_img) cv2.putText(img, "Weird_State_Count : " + str(weird_state_count), (0, 20), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) if weird_state_flag: cv2.putText(img, 'State : ABNORMAL', (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) else: cv2.putText(img, "State : Driver_Found", (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) # print("Driver_index :", driver_index) # print("Driver_Flag : ", is_driver_flag) # csv_dict['frame_number'].append(idxx) # csv_dict['driver_index'].append(driver_index) # csv_dict['is_driver_flag'].append(is_driver_flag) # csv_dict['state'].append(state) if track_ids == True: ##Track Poses propagate_ids(previous_poses, current_poses) previous_poses = current_poses index_counter = 0 for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if is_driver_flag and index_counter == driver_index: cv2.putText(img, 'DRIVER', (pose.bbox[0] + 100, pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) index_counter += 1 tt = time.time() fps = 1 / (tt - ttt) print('fps=', fps) ttt = time.time() str_ = "FPS : %0.2f" % fps cv2.putText(img, str_, (0, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) # cv2_imshow(img) cv2.imwrite('output/two_2/' + str(idxx) + '.png', img) idxx += 1 key = cv2.waitKey(1) if key == 27: # esc return df = pd.DataFrame(csv_dict)
def run_demo(net, image_provider, height_size, cpu, track, smooth, com): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # +1 for Hidden COM previous_poses = [] # original delay 33 # 0 = pause / wait for input indefinetly delay = 0 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM found_kpts = [] C_pts = [] BOS = [[-1, -1], [-1, -1]] for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) found_kpts.append(kpt_id) # print(kpt_id, pose_keypoints[kpt_id], Pose.kpt_names[kpt_id]) # ====== HOLY GRAIL ========= if com: COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints) pose_keypoints[-1] = COM pose = Pose(pose_keypoints, pose_entries[n][18], C_pts, BOS) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_inference(net, image_provider, height_size, cpu, track, smooth, no_display, json_view=False): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 100 if isinstance(image_provider, ImageReader): delay = 0 for img in image_provider: heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n, pose_entry in enumerate(pose_entries): if len(pose_entry) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entry[kpt_id] != -1.0: pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entry[kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entry[kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entry[18]) current_poses.append(pose) if json_view == True: return current_poses if not no_display: if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (32, 202, 252)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('PoseCamera', img) key = cv2.waitKey(delay) if key == 27: return
def run_demo(net, image_provider, height_size, cpu, track_ids): #, filename): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts #18개 previous_poses = [] c = 0 ttt = 0 for img in image_provider: t5 = time.time() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): ##사이즈 변환 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: ##Track Poses propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) tt = time.time() fps = 1 / (tt - ttt) print('fps=', fps) ttt = time.time() str = "FPS : %0.2f" % fps cv2.putText(img, str, (0, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) # cv2.imwrite('PAF_'+filename[image_provider.idx -1], img) key = cv2.waitKey(1) if key == 27: # esc return
def run_on_image(net, height_size, cpu, track, smooth, img, stride, upsample_ratio, num_keypoints, threshold): global previous_poses orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) score = 0 total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: # cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), r_hand_center, r_hand_width, l_hand_center, l_hand_width, = detect_hand( pose) if -1 not in r_hand_center: cv2.circle(img, (r_hand_center[0], r_hand_center[1]), 5, (255, 0, 0), 5) cv2.rectangle(img, (r_hand_center[0] - r_hand_width, r_hand_center[1] - r_hand_width), (r_hand_center[0] + r_hand_width, r_hand_center[1] + r_hand_width), (0, 255, 255)) if -1 not in l_hand_center: cv2.circle(img, (l_hand_center[0], l_hand_center[1]), 5, (255, 0, 0), 5) cv2.rectangle(img, (l_hand_center[0] - l_hand_width, l_hand_center[1] - l_hand_width), (l_hand_center[0] + l_hand_width, l_hand_center[1] + l_hand_width), (0, 255, 255)) face_center, face_width = detect_face(pose) if -1 not in face_center: cv2.rectangle( img, (face_center[0] - face_width, face_center[1] - face_width), (face_center[0] + face_width, face_center[1] + face_width), (0, 0, 255)) # (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (face_center[0] - face_width, face_center[1] - face_width - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if -1 not in r_hand_center: x, y, h, w, score = detect_touch(face_center, face_width, r_hand_center, r_hand_width) if h != 0: cv2.rectangle(img, (x, y), (x + h, y + w), (255, 0, 255)) cv2.putText(img, f'Score: {score:0.2f}', (x, y - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 0)) if -1 not in l_hand_center: x, y, h, w, score = detect_touch(face_center, face_width, l_hand_center, l_hand_width) if h != 0: cv2.rectangle(img, (x, y), (x + h, y + w), (255, 0, 255)) cv2.putText(img, f'Score: {score:0.2f}', (x, y - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 0)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) delay = 1 detect = False key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33 return score > threshold
def get_skel_coords(net, image_provider, height_size=256, cpu=False, track=1, smooth=1): # text_file = open("skel/"+ args.save_txt, "w") net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 img = image_provider # for img in image_provider: #print(img.shape) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # for pose in current_poses: if len(current_poses) != 0: # n = text_file.write(coords) cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) # break #cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) if len(current_poses) != 0: #print ("not zero", img) return current_poses[0].return_coords(), img else: #print ("zero") return [], np.array((1, 1, 1), np.uint8) key = cv2.waitKey(delay) if key == 27: # esc text_file.close() return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = num_keys previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][num_keypoints]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0, 0, 255)) for p in range(pose_entries.shape[0]): allkey=True landmarks=[] if(len(pose_entries[p])>0): if pose_entries[p][0]==-1 or pose_entries[p][14]==-1 or pose_entries[p][15]==-1 or (pose_entries[p][16]==-1 and pose_entries[p][17]==-1): allkey=False if allkey: landmarks.append(int(all_keypoints[int(pose_entries[p][0])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][0])][1])) landmarks.append(int(all_keypoints[int(pose_entries[p][14])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][14])][1])) landmarks.append(int(all_keypoints[int(pose_entries[p][15])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][15])][1])) if pose_entries[p][17]==-1: ear="right" landmarks.append(int(all_keypoints[int(pose_entries[p][16])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][16])][1])) else: ear="left" landmarks.append(int(all_keypoints[int(pose_entries[p][17])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][17])][1])) if len(landmarks)>0: imgpts, modelpts, rotate_degree, center = face_orientation(img, landmarks,ear) cv2.line(img, center, tuple(imgpts[1].ravel()), (0,255,0), 3) #GREEN cv2.line(img, center, tuple(imgpts[0].ravel()), (255,0,), 3) #BLUE cv2.line(img, center, tuple(imgpts[2].ravel()), (0,0,255), 3) #RED #for j in range(len(rotate_degree)): #cv2.putText(img, ('{:05.2f}').format(float(rotate_degree[j])), (10, 30 + (50 * j)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), thickness=2, lineType=2) #cv2.imwrite('/Users/Utente/Desktop/poseFINAL/out.jpg', img) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(net, image_provider, output_dir, height_size, cpu, track, smooth, generate_video): net = net.eval() if not cpu: net = net.cuda() if generate_video: video_output_path = os.path.join(output_dir, output_dir.split("/")[-1] +"-annotations" + ".mp4") #(1080, 1920, 3) video_out = cv2.VideoWriter(video_output_path, cv2.VideoWriter_fourcc('m','p','4','v'), 10, (1920,1080)) stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 for idx, img in enumerate(image_provider): orig_img = img.copy() frame_file = os.path.join(output_dir, output_dir.split("/")[-1]+f"_{idx:06}.txt") with open(frame_file, "w") as frame_f: print("Input the model is", orig_img.shape) #print(output_path) heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses annotations_hand = [] annotations_head = [] for pose in current_poses: pose.draw(img) print(pose.hands_bbox, pose.head_bbox) for hand_bbox in pose.hands_bbox: x_center = (hand_bbox[0][0] + hand_bbox[1][0]) / (2*1920) y_center = (hand_bbox[0][1] + hand_bbox[1][1]) / (2*1080) x_offset = (hand_bbox[1][0] - hand_bbox[0][0]) / 1920 y_offset = (hand_bbox[1][1] - hand_bbox[0][1]) / 1080 if x_center < 0.0: x_offset = x_offset - x_center x_center = 0 hand_bbox_scaled = (x_center, y_center, x_offset, y_offset) print(hand_bbox_scaled) frame_f.write("8 " + ' '.join(map(str, hand_bbox_scaled)) + "\n") annotations_hand.append(hand_bbox_scaled) x_center = (pose.head_bbox[0][0] + pose.head_bbox[1][0]) / (2*1920) y_center = (pose.head_bbox[0][1] + pose.head_bbox[1][1]) / (2*1080) x_offset = (pose.head_bbox[1][0] - pose.head_bbox[0][0]) / 1920 y_offset = (pose.head_bbox[1][1] - pose.head_bbox[0][1]) / 1080 if x_center < 0.0: x_offset = x_offset - x_center x_center = 0 head_bbox_scaled = (x_center, y_center, x_offset, y_offset) print(head_bbox_scaled) frame_f.write("9 " + ' '.join(map(str, head_bbox_scaled)) + "\n") annotations_head.append(head_bbox_scaled) img = cv2.addWeighted(orig_img, 0.3, img, 0.7, 0) for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) img_filename = os.path.join(output_dir, output_dir.split("/")[-1] + f"_{idx:06}.jpg") print(img_filename) cv2.imwrite(img_filename, orig_img) if generate_video: video_out.write(img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33 if video_out: video_out.release()
def callback(self, data): try: cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) ## Rescale Image size rescale_factor = 1 width = int(cv_image.shape[1] * rescale_factor) height = int(cv_image.shape[0] * rescale_factor) dim = (width, height) resized_img = cv2.resize(cv_image, dim) net = PoseEstimationWithMobileNet() checkpoint = torch.load( "/home/zheng/lightweight-human-pose-estimation.pytorch/checkpoint_iter_370000.pth", map_location='cpu') load_state(net, checkpoint) height_size = 256 net = net.eval() net = net.cuda() net.eval() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 # img = cv2.imread("/home/zheng/lightweight-human-pose-estimation.pytorch/data/image_1400.jpg") img = asarray(cv_image) orig_img = img heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu="store_true") total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] ## Collect all keypoint in numpy array to send it to Ros" pose_keypoints_ros_data = np.zeros(16) my_array_for_publishing = Float32MultiArray() #### pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(8): if pose_entries[0][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[0][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[0][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[0][18]) current_poses.append(pose) pose_keypoints_ros_data[2 * kpt_id] = pose.keypoints[kpt_id][0] pose_keypoints_ros_data[2 * kpt_id + 1] = pose.keypoints[kpt_id][1] for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) my_array_for_publishing.data = [ pose_keypoints_ros_data[0], pose_keypoints_ros_data[1], pose_keypoints_ros_data[2], pose_keypoints_ros_data[3], pose_keypoints_ros_data[4], pose_keypoints_ros_data[5], pose_keypoints_ros_data[6], pose_keypoints_ros_data[7], pose_keypoints_ros_data[8], pose_keypoints_ros_data[9], pose_keypoints_ros_data[10], pose_keypoints_ros_data[11], pose_keypoints_ros_data[12], pose_keypoints_ros_data[13], pose_keypoints_ros_data[14], pose_keypoints_ros_data[15], ] # cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) self.image_pub.publish(self.bridge.cv2_to_imgmsg(img, "bgr8")) self.keypts_pub.publish(my_array_for_publishing) # cv2.imwrite('/home/zheng/Bureau/image_1400_key.jpg',img) cv2.waitKey(2)
def run_demo(model, image_provider, height_size, cpu, track, smooth, file): """[summary] Args: model ([type]): [description] image_provider ([type]): [description] height_size ([type]): [description] cpu ([type]): [description] track ([type]): [description] smooth ([type]): [description] file ([type]): [description] Returns: [type]: [description] """ model = model.eval() if not cpu: model = model.cuda() point_list = [] stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] # 保存视频 fps = image_provider.fps width = image_provider.width height = image_provider.height fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', 'V') # video_saver = cv2.VideoWriter('TESTV.mp4', fourcc, fps, (height, width)) save_video_path = os.path.join(os.getcwd(), 'video_output') if not os.path.exists(save_video_path): os.mkdir(save_video_path) save_video_name = os.path.join(save_video_path, file + '.mp4') video_saver = cv2.VideoWriter(save_video_name, fourcc, fps, (width, height)) for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(model, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for pose_entry in pose_entries: if len(pose_entry) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entry[kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entry[kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entry[kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entry[18]) current_poses.append(pose) # save keypoint in list key_point_list = pose_keypoints.flatten().tolist() point_list.append(key_point_list) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) video_saver.write(img) return point_list
def run_demo(net, image_provider1, image_provider2, height_size, cpu, track, smooth, com): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # +1 for Hidden COM previous_poses = [] # original delay 33 # 0 = pause / wait for input indefinetly delay = 33 total_provider = zip(image_provider1, image_provider2) for img1, img2 in total_provider: orig_img1 = img1.copy() orig_img2 = img2.copy() heatmaps1, pafs1, scale1, pad1 = infer_fast(net, img1, height_size, stride, upsample_ratio, cpu) heatmaps2, pafs2, scale2, pad2 = infer_fast(net, img2, height_size, stride, upsample_ratio, cpu) total_keypoints_num1 = 0 total_keypoints_num2 = 0 all_keypoints_by_type1 = [] all_keypoints_by_type2 = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num1 += extract_keypoints(heatmaps1[:, :, kpt_idx], all_keypoints_by_type1, total_keypoints_num1) total_keypoints_num2 += extract_keypoints(heatmaps2[:, :, kpt_idx], all_keypoints_by_type2, total_keypoints_num2) pose_entries1, all_keypoints1 = group_keypoints(all_keypoints_by_type1, pafs1, demo=True) pose_entries2, all_keypoints2 = group_keypoints(all_keypoints_by_type2, pafs2, demo=True) for kpt_id in range(all_keypoints1.shape[0]): all_keypoints1[kpt_id, 0] = (all_keypoints1[kpt_id, 0] * stride / upsample_ratio - pad1[1]) / scale1 all_keypoints1[kpt_id, 1] = (all_keypoints1[kpt_id, 1] * stride / upsample_ratio - pad1[0]) / scale1 for kpt_id in range(all_keypoints2.shape[0]): all_keypoints2[kpt_id, 0] = (all_keypoints2[kpt_id, 0] * stride / upsample_ratio - pad2[1]) / scale2 all_keypoints2[kpt_id, 1] = (all_keypoints2[kpt_id, 1] * stride / upsample_ratio - pad2[0]) / scale2 current_poses1 = [] current_poses2 = [] for n in range(len(pose_entries1)): if len(pose_entries1[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM found_kpts = [] C_pts = [] BOS = [[-1, -1], [-1, -1]] for kpt_id in range(num_keypoints): if pose_entries1[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints1[int(pose_entries1[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints1[int(pose_entries1[n][kpt_id]), 1]) found_kpts.append(kpt_id) if com: COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints) pose_keypoints[-1] = COM pose = Pose(pose_keypoints, pose_entries1[n][18], C_pts, BOS) current_poses1.append(pose) for n in range(len(pose_entries2)): if len(pose_entries2[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM found_kpts = [] C_pts = [] BOS = [[-1, -1], [-1, -1]] for kpt_id in range(num_keypoints): if pose_entries2[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints2[int(pose_entries2[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints2[int(pose_entries2[n][kpt_id]), 1]) found_kpts.append(kpt_id) if com: COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints) pose_keypoints[-1] = COM pose = Pose(pose_keypoints, pose_entries2[n][18], C_pts, BOS) current_poses2.append(pose) #if track: #track_poses(previous_poses, current_poses, smooth=smooth) #previous_poses = current_poses for pose in current_poses1: pose.draw(img1) for pose in current_poses2: pose.draw(img2) img1 = cv2.addWeighted(orig_img1, 0.6, img1, 0.4, 0) img2 = cv2.addWeighted(orig_img2, 0.6, img2, 0.4, 0) #for pose in current_poses: #cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), #(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) #if track: #cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), #cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Demo, Feed 1', img1) cv2.imshow('Demo, Feed 2', img2) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track, smooth): show_time = True net = net.eval() if not cpu: net = net.cuda() track = True print("track:" + str(track)) stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 for img, img_dir in image_provider: orig_img = img.copy() start_time = datetime.datetime.now() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) end_time = datetime.datetime.now() if show_time: print("pose time:") print((end_time - start_time).microseconds / 1000) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) #cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) print(img_dir) img_save_dir = img_dir.replace("15_1", "15_1_lightpose") #cv2.imwrite("/data2/qilei_chen/DATA/ShanghaiAutograding/gangganpingheng_images_240/15_1/right_light_pose/"+os.path.basename(img_dir),img) cv2.imwrite(img_save_dir, img) '''
def run_demo(net, action_net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts with open( "D:/py/openpose_lightweight/performance_evaluation/action_result.txt", "a") as f: for img, img_name, label in image_provider: if label == 'fall': label_ = 0 else: label_ = 1 heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if len(pose.getKeyPoints()) >= 12: current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img) crown_proportion = pose.bbox[2] / pose.bbox[3] # 宽高比 pose = action_detect(action_net, pose, crown_proportion) cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) f.write( f"{label_} {pose.action_fall} {pose.action_normal}\n") f.flush() break cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1)
def run(net, img, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] kpt_names = Pose.kpt_names l_wri = kpt_names.index('l_wri') r_wri = kpt_names.index('r_wri') left_wrists = [] right_wrists = [] orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] # gather keypoints for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) # rescale keypoints pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale # get all poses position current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) # entries is confidence pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) # paint on pose.draw( img) # draw lines and circles to show the human skeleton found img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: left_wrists.append(pose.keypoints[l_wri]) right_wrists.append(pose.keypoints[r_wri]) # print(pose.keypoints[l_wri], pose.keypoints[r_wri]) # found: left wrist and right wrist return img, current_poses, left_wrists, right_wrists