def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imwrite('/Users/Utente/Desktop/pose/out.jpg', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 0 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses print("draw", img.dtype, img.shape, img.min(), img.max()) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) print(img.min(), img.max()) for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu:#run the model "net" in cuda if not specified to run specifically in CPU # gets in here since by default store true is false. Hence uses GPU for inference by default net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) # cv2.resize(img,(720,1280)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', cv2.resize(img,(2800,1800))) key = cv2.waitKey(33) if key == 27: # esc return
def extract_pose(heatmaps, pafs, scale, pad, num_keypoints, stride, upsample_ratio): total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) return current_poses
def get_final_keypoint_positions(all_keypoints_by_type, pafs, num_keypoints, stride, scale, upsample_ratio, pad): pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) return current_poses
def parse_poses(inference_results, input_scale, stride, fx, is_video=False): global previous_poses_2d poses_2d = get_root_relative_poses(inference_results) poses_2d_scaled = [] for pose_2d in poses_2d: num_kpt = (pose_2d.shape[0] - 1) // 3 pose_2d_scaled = np.ones(pose_2d.shape[0], dtype=np.float32) * -1 for kpt_id in range(num_kpt): if pose_2d[kpt_id * 3] != -1: if pose_2d[kpt_id * 3] >0 and pose_2d[kpt_id * 3 + 1]>0: pose_2d_scaled[kpt_id * 3] = pose_2d[kpt_id * 3] * stride / input_scale pose_2d_scaled[kpt_id * 3 + 1] = pose_2d[kpt_id * 3 + 1] * stride / input_scale pose_2d_scaled[kpt_id * 3 + 2] = pose_2d[kpt_id * 3 + 2] # print(previous_poses_2d) pose_2d_scaled[-1] = pose_2d[-1] poses_2d_scaled.append(pose_2d_scaled) if is_video: # track poses ids current_poses_2d = [] for pose_2d_scaled,pre_pose in zip(poses_2d_scaled,previous_poses_2d): pose_keypoints = np.ones((Pose.num_kpts, 2), dtype=np.int32) * -1 for kpt_id in range(Pose.num_kpts): if pose_2d_scaled[kpt_id * 3] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0:2] = pose_2d_scaled[kpt_id * 3:kpt_id * 3 + 2].astype(np.int32) else: pose_keypoints[kpt_id, 0:2] = pre_pose.keypoints[kpt_id, 0:2].astype(np.int32) pose = Pose(pose_keypoints, pose_2d_scaled[-1]) current_poses_2d.append(pose) propagate_ids(previous_poses_2d, current_poses_2d) previous_poses_2d = current_poses_2d return np.array(poses_2d_scaled)
def _dect_pose(self, **kwargs): """Detect poses. Arguments: img {ndarray}: input image. model {PoseEstimationWithMobileNet}: initialized OpenPose model. previous_poses {list}: previous poses for tracking mode. Returns: list: detected poses. """ img = kwargs.get('img', None) model = kwargs.get('model', None) previous_poses = kwargs.get('previous_poses', None) use_cuda = kwargs.get('use_cuda', False) track = self.__params.track smooth = self.__params.smooth stride = self.__params.stride upsample_ratio = self.__params.upsample_ratio num_keypoints = Pose.num_kpts #orig_img = img.copy() heatmaps, pafs, scale, pad = self._infer_fast(model=model, img=img, use_cuda=use_cuda) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses return current_poses
def __call__(self, img, height_size=256): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(self.model, img, height_size, stride, upsample_ratio, False) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imwrite("/data1/qilei_chen/DEVELOPMENTS/test1.jpg", img) return current_poses
def visualize_prediction(self, image): orig_img = image.copy() if not np.array_equal(self.image, image): self.image = image self.__inference_fast(self.image, self.height_size, self.stride, self.upsample_ratio) current_poses = [] for n in range(len(self.pose_entries)): if len(self.pose_entries[n]) == 0: continue pose_keypoints = np.ones( (self.num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(self.num_keypoints): if self.pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( self.all_keypoints[int(self.pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( self.all_keypoints[int(self.pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, self.pose_entries[n][18]) current_poses.append(pose) # if self.track: # previous_poses = [] # track_poses(previous_poses, current_poses, smooth=smooth) # previous_poses = current_poses for pose in current_poses: pose.draw(image) image = cv2.addWeighted(orig_img, 0.6, image, 0.4, 0) # plt.imshow(image) # plt.show() return image
def pose_detector(img, net, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) import ipdb # ipdb.set_trace() total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) return current_poses
def track_poses(self, previous_poses, current_poses, threshold=3, smooth=False): """Propagate poses ids from previous frame results. Id is propagated, if there are at least `threshold` similar keypoints between pose from previous frame and current. If correspondence between pose on previous and current frame was established, pose keypoints are smoothed. :param previous_poses: poses from previous frame with ids :param current_poses: poses from current frame to assign ids :param threshold: minimal number of similar keypoints between poses :param smooth: smooth pose keypoints between frames :return: None """ current_poses = sorted(current_poses, key=lambda pose: pose.confidence, reverse=True) # match confident poses first mask = np.ones(len(previous_poses), dtype=np.int32) for current_pose in current_poses: best_matched_id = None best_matched_pose_id = None best_matched_iou = 0 for id, previous_pose in enumerate(previous_poses): if not mask[id]: continue iou = self.get_similarity(current_pose, previous_pose) if iou > best_matched_iou: best_matched_iou = iou best_matched_pose_id = previous_pose.id best_matched_id = id if best_matched_iou >= threshold: mask[best_matched_id] = 0 else: # pose not similar to any previous best_matched_pose_id = None current_pose.update_id(best_matched_pose_id) if smooth: for kpt_id in range(Pose.num_kpts): if current_pose.keypoints[kpt_id, 0] == -1: continue # reuse filter if previous pose has valid filter if (best_matched_pose_id is not None and previous_poses[best_matched_id].keypoints[kpt_id, 0] != -1): current_pose.filters[kpt_id] = previous_poses[best_matched_id].filters[kpt_id] current_pose.keypoints[kpt_id, 0] = current_pose.filters[kpt_id][0](current_pose.keypoints[kpt_id, 0]) current_pose.keypoints[kpt_id, 1] = current_pose.filters[kpt_id][1](current_pose.keypoints[kpt_id, 1]) current_pose.bbox = Pose.get_bbox(current_pose.keypoints)
def getPose(self, height_size=256, stride=8, upsample_ratio=4, num_keypoints=18): heatmaps, pafs, scale, pad = infer_fast(net, self.frametmp, height_size, stride, upsample_ratio, 0) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if current_poses != [] and len(current_poses) != 0: self.current_pose = get_max_human(current_poses) else: self.current_pose = None
def parse_poses_2d(inference_results, input_scale, stride, fx, is_video=False): global previous_poses_2d poses_3d, poses_2d, features_shape = get_root_relative_poses( inference_results) poses_2d_scaled = [] for pose_2d in poses_2d: num_kpt = (pose_2d.shape[0] - 1) // 3 pose_2d_scaled = np.ones( pose_2d.shape[0], dtype=np.float32) * -1 # +1 for pose confidence for kpt_id in range(num_kpt): if pose_2d[kpt_id * 3] != -1: pose_2d_scaled[kpt_id * 3] = int(pose_2d[kpt_id * 3] * stride / input_scale) pose_2d_scaled[kpt_id * 3 + 1] = int(pose_2d[kpt_id * 3 + 1] * stride / input_scale) pose_2d_scaled[kpt_id * 3 + 2] = pose_2d[kpt_id * 3 + 2] pose_2d_scaled[-1] = pose_2d[-1] poses_2d_scaled.append(pose_2d_scaled) if is_video: # track poses ids current_poses_2d = [] for pose_id in range(len(poses_2d_scaled)): pose_keypoints = np.ones((Pose.num_kpts, 2), dtype=np.int32) * -1 for kpt_id in range(Pose.num_kpts): if poses_2d_scaled[pose_id][kpt_id * 3] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( poses_2d_scaled[pose_id][kpt_id * 3 + 0]) pose_keypoints[kpt_id, 1] = int( poses_2d_scaled[pose_id][kpt_id * 3 + 1]) pose = Pose(pose_keypoints, poses_2d_scaled[pose_id][-1]) current_poses_2d.append(pose) propagate_ids(previous_poses_2d, current_poses_2d) previous_poses_2d = current_poses_2d return np.array(poses_2d_scaled)
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) # import ipdb; ipdb.set_trace() # print(current_poses) current_poses[-1].keypoints if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow("Lightweight Human Pose Estimation Python Demo", img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def find_temperature(net, image_provider = VideoReader(), send = False, cpu = False): height_size = 256 stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts data = {} frames = image_provider.next_frame() img = np.array(frames[0]) depth = np.array(frames[1]) """ Estimate the pose and find the person in the middle """ heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] distMin = 310 midPose = None for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) dist = abs(pose.keypoints[0][0] - image_provider.width/2) if dist < distMin: distMin = dist midPose = pose current_poses.append(pose) """ Find the temperature of each exposed parts of the body """ if midPose != None: for n in range(len(Pose.kpt_names)): if midPose.keypoints[n][0] != 0 or midPose.keypoints[n][1] != 0: data[Pose.kpt_names[n]] = get_temperature( midPose, n, depth ) return data
def callback(self, data): try: cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) ## Rescale Image size rescale_factor = 1 width = int(cv_image.shape[1] * rescale_factor) height = int(cv_image.shape[0] * rescale_factor) dim = (width, height) resized_img = cv2.resize(cv_image, dim) net = PoseEstimationWithMobileNet() checkpoint = torch.load( "/home/zheng/lightweight-human-pose-estimation.pytorch/checkpoint_iter_370000.pth", map_location='cpu') load_state(net, checkpoint) height_size = 256 net = net.eval() net = net.cuda() net.eval() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 # img = cv2.imread("/home/zheng/lightweight-human-pose-estimation.pytorch/data/image_1400.jpg") img = asarray(cv_image) orig_img = img heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu="store_true") total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] ## Collect all keypoint in numpy array to send it to Ros" pose_keypoints_ros_data = np.zeros(16) my_array_for_publishing = Float32MultiArray() #### pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(8): if pose_entries[0][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[0][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[0][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[0][18]) current_poses.append(pose) pose_keypoints_ros_data[2 * kpt_id] = pose.keypoints[kpt_id][0] pose_keypoints_ros_data[2 * kpt_id + 1] = pose.keypoints[kpt_id][1] for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) my_array_for_publishing.data = [ pose_keypoints_ros_data[0], pose_keypoints_ros_data[1], pose_keypoints_ros_data[2], pose_keypoints_ros_data[3], pose_keypoints_ros_data[4], pose_keypoints_ros_data[5], pose_keypoints_ros_data[6], pose_keypoints_ros_data[7], pose_keypoints_ros_data[8], pose_keypoints_ros_data[9], pose_keypoints_ros_data[10], pose_keypoints_ros_data[11], pose_keypoints_ros_data[12], pose_keypoints_ros_data[13], pose_keypoints_ros_data[14], pose_keypoints_ros_data[15], ] # cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) self.image_pub.publish(self.bridge.cv2_to_imgmsg(img, "bgr8")) self.keypts_pub.publish(my_array_for_publishing) # cv2.imwrite('/home/zheng/Bureau/image_1400_key.jpg',img) cv2.waitKey(2)
def parse_poses(inference_results, input_scale, stride, fx, is_video=False): global previous_poses_2d poses_3d, poses_2d, features_shape = get_root_relative_poses( inference_results) poses_2d_scaled = [] for pose_2d in poses_2d: num_kpt = (pose_2d.shape[0] - 1) // 3 pose_2d_scaled = np.ones( pose_2d.shape[0], dtype=np.float32) * -1 # +1 for pose confidence for kpt_id in range(num_kpt): if pose_2d[kpt_id * 3] != -1: pose_2d_scaled[kpt_id * 3] = int(pose_2d[kpt_id * 3] * stride / input_scale) pose_2d_scaled[kpt_id * 3 + 1] = int(pose_2d[kpt_id * 3 + 1] * stride / input_scale) pose_2d_scaled[kpt_id * 3 + 2] = pose_2d[kpt_id * 3 + 2] pose_2d_scaled[-1] = pose_2d[-1] poses_2d_scaled.append(pose_2d_scaled) if is_video: # track poses ids current_poses_2d = [] for pose_id in range(len(poses_2d_scaled)): pose_keypoints = np.ones((Pose.num_kpts, 2), dtype=np.int32) * -1 for kpt_id in range(Pose.num_kpts): if poses_2d_scaled[pose_id][kpt_id * 3] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( poses_2d_scaled[pose_id][kpt_id * 3 + 0]) pose_keypoints[kpt_id, 1] = int( poses_2d_scaled[pose_id][kpt_id * 3 + 1]) pose = Pose(pose_keypoints, poses_2d_scaled[pose_id][-1]) current_poses_2d.append(pose) propagate_ids(previous_poses_2d, current_poses_2d) previous_poses_2d = current_poses_2d translated_poses_3d = [] # translate poses for pose_id in range(len(poses_3d)): pose_3d = poses_3d[pose_id].reshape((-1, 4)).transpose() pose_2d = poses_2d[pose_id][:-1].reshape((-1, 3)).transpose() num_valid = np.count_nonzero(pose_2d[2] != -1) pose_3d_valid = np.zeros((3, num_valid), dtype=np.float32) pose_2d_valid = np.zeros((2, num_valid), dtype=np.float32) valid_id = 0 for kpt_id in range(pose_3d.shape[1]): if pose_2d[2, kpt_id] == -1: continue pose_3d_valid[:, valid_id] = pose_3d[0:3, kpt_id] pose_2d_valid[:, valid_id] = pose_2d[0:2, kpt_id] valid_id += 1 pose_2d_valid[0] = pose_2d_valid[0] - features_shape[2] / 2 pose_2d_valid[1] = pose_2d_valid[1] - features_shape[1] / 2 mean_3d = np.expand_dims(pose_3d_valid.mean(axis=1), axis=1) mean_2d = np.expand_dims(pose_2d_valid.mean(axis=1), axis=1) numerator = np.trace( np.dot((pose_3d_valid[:2, :] - mean_3d[:2, :]).transpose(), pose_3d_valid[:2, :] - mean_3d[:2, :])).sum() numerator = np.sqrt(numerator) denominator = np.sqrt( np.trace( np.dot((pose_2d_valid[:2, :] - mean_2d[:2, :]).transpose(), pose_2d_valid[:2, :] - mean_2d[:2, :])).sum()) mean_2d = np.array( [mean_2d[0, 0], mean_2d[1, 0], fx * input_scale / stride]) mean_3d = np.array([mean_3d[0, 0], mean_3d[1, 0], 0]) translation = numerator / denominator * mean_2d - mean_3d if is_video: translation = current_poses_2d[pose_id].filter(translation) for kpt_id in range(19): pose_3d[0, kpt_id] = pose_3d[0, kpt_id] + translation[0] pose_3d[1, kpt_id] = pose_3d[1, kpt_id] + translation[1] pose_3d[2, kpt_id] = pose_3d[2, kpt_id] + translation[2] translated_poses_3d.append(pose_3d.transpose().reshape(-1)) return np.array(translated_poses_3d), np.array(poses_2d_scaled)
def run_demo(net,action_net, image_provider, height_size, cpu): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts i = 0 for img in image_provider: orig_img = img.copy() # print(i) if i % 1 == 0: heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if len(pose.getKeyPoints()) >= 10: current_poses.append(pose) # current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img,show_draw=True) crown_proportion = pose.bbox[2]/pose.bbox[3] #宽高比 pose = action_detect(action_net,pose,crown_proportion) if pose.pose_action == 'fall': cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 0, 255),thickness=3) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) else: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0)) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1) i += 1 cv2.destroyAllWindows()
def run_demo(net, image_provider, height_size, cpu, track_ids, arm): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] stateMachines = {} for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) """ kpt_names = ['nose', 'neck', 'r_sho', 'r_elb', 'r_wri', 'l_sho', 'l_elb', 'l_wri', 'r_hip', 'r_knee', 'r_ank', 'l_hip', 'l_knee', 'l_ank', 'r_eye', 'l_eye', 'r_ear', 'l_ear'] r_elb-3, r-wri-4, l_elb-6, l_wri-7 """ # print('ID: {}'.format(n)) # print('\tRight elbow: {}, right wrist: {}'.format(pose_keypoints[3], pose_keypoints[4])) # print('\tLeft elbow: {}, left wrist: {}'.format(pose_keypoints[6], pose_keypoints[7])) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses, threshold=3) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) for pose in current_poses: if pose.id not in stateMachines.keys(): stateMachines[pose.id] = StateMachine(pose.id, pose.keypoints, arm) #print('ID {} detected'.format(pose.id)) continue # call stateMachine methods stateMachines[pose.id].update(pose.keypoints, img) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(net, height_size, track, smooth, record_vid, camera_type): net = net.eval() net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] ##Tarit defined slope_threshold = 0.4 ear_slope_threshold = 0.5 eye_ear_slope_threshold = 0.5 not_detected = (-1, -1) sleep_confirmation_time = 60 #in seconds #flags to detect whether the person is sleeping or not sleeping = False timer_started = False time_notified = 0 selected_pose = None while True: #msg, frame = receiver.receive(timeout = 60.0) #img = cv2.imdecode(np.frombuffer(frame, dtype='uint8'), -1) img = cap.read() if camera_type == "jetson": img = img[1300:1780, 1320:1960] #start_time = time.time() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses '''for pose in current_poses: pose.draw(img)''' ##find longest_nect_to_nose_dst and select that pose longest_nect_to_nose_dst = 0 for pose in current_poses: nose = tuple(pose.keypoints[0]) neck = tuple(pose.keypoints[1]) ##pythagoras nect_to_nose_dst = pow((pow(abs(nose[0] - neck[0]), 2)) + (pow(abs(nose[1] - neck[1]), 2)), 1 / 2) if nect_to_nose_dst > longest_nect_to_nose_dst: longest_nect_to_nose_dst = nect_to_nose_dst selected_pose = pose if selected_pose is not None: selected_pose.draw(img) nose = tuple(selected_pose.keypoints[0]) neck = tuple(selected_pose.keypoints[1]) l_ear = tuple(selected_pose.keypoints[16]) r_ear = tuple(selected_pose.keypoints[17]) l_eye = tuple(selected_pose.keypoints[15]) r_eye = tuple(selected_pose.keypoints[14]) #print(cal_slope(l_eye,l_ear),cal_slope(r_eye,r_ear)) ##detect if the person back if facing to the camera if nose == (-1, -1): if l_ear != not_detected and r_ear != not_detected: ear_slope = abs(l_ear[1] - r_ear[1]) / abs(l_ear[0] - r_ear[0]) cv2.circle(img, l_ear, 5, (255, 0, 0), 3) cv2.circle(img, r_ear, 5, (0, 255, 0), 3) if ear_slope > ear_slope_threshold: sleeping = True print("sleeping") else: sleeping = False else: ##out of condition, can't detect sleeping = False else: cv2.circle(img, nose, 5, (255, 0, 0), 3) cv2.circle(img, neck, 5, (0, 255, 0), 3) slope_inverse = (nose[0] - neck[0]) / (nose[1] - neck[1]) l_ear_eye_slope = cal_slope(l_eye, l_ear) r_ear_eye_slope = cal_slope(r_eye, r_ear) #increase the slope_threshold if the person is turning their head #print(pose.keypoints[16],pose.keypoints[17]) #print ear location if l_ear == (-1, -1) or r_ear == (-1, -1): slope_threshold = 1 print("one ear missing , Increasing slope_threshold") else: slope_threshold = 0.4 if abs(slope_inverse) > slope_threshold: #cv2.putText(img,"".join([str(pose.id),"sleeping"]),(20,50),cv2.FONT_HERSHEY_COMPLEX,2,(255,0,0),3) print("Sleeping (neck bend more than threshold)") #cv2.putText(img,"sleeping",(20,50),cv2.FONT_HERSHEY_COMPLEX,2,(255,0,0),3) sleeping = True elif l_eye == not_detected or r_eye == not_detected: sleeping = True print("Sleeping (not seeing both eyes)") elif l_ear_eye_slope < -0.6 or r_ear_eye_slope > 0.6 or l_ear_eye_slope > eye_ear_slope_threshold or r_ear_eye_slope < -eye_ear_slope_threshold: sleeping = True print("Sleeping (ears higher/lower than eyes)") else: print("Not sleeping") sleeping = False if sleeping: if not timer_started: t_start_sleep = time.time() timer_started = True else: if time.time() - t_start_sleep > sleep_confirmation_time: print("sending line message") pic_name = "".join( ["log_data/", str(time_notified), ".jpg"]) cv2.imwrite(pic_name, img) #lineNotify("Elderly sleeping %d"%time_notified) notifyFile("Elderly sleeping %d" % time_notified, pic_name) time_notified += 1 timer_started = False sleeping = False else: timer_started = False #song = AudioSegment.from_mp3("Alarm_Clock_Sound.mp3") #play(song) img = cv2.addWeighted(orig_img, 0.6, img, 0.6, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Sleep detector', img) if record_vid: out_raw.write(orig_img) out_pose.write(img) #print((1/(time.time()-start_time))) if cv2.waitKey(1) == 27: # esc #receiver.close() cap.stop() if record_vid: out_raw.release() out_pose.release() return
def run_demo(net, action_net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts with open( "D:/py/openpose_lightweight/performance_evaluation/action_result.txt", "a") as f: for img, img_name, label in image_provider: if label == 'fall': label_ = 0 else: label_ = 1 heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if len(pose.getKeyPoints()) >= 12: current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img) crown_proportion = pose.bbox[2] / pose.bbox[3] # 宽高比 pose = action_detect(action_net, pose, crown_proportion) cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) f.write( f"{label_} {pose.action_fall} {pose.action_normal}\n") f.flush() break cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1)
def run_demo(net, image_provider, height_size=256, cpu=False, track_ids=False): net = net.eval() if not cpu: net = net.cuda() print("use cuda") stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18 #Initialize previous_pose_kpts = [] graph_x, graph_y = [], [] result = [-1, -1, -1, -1, -1] count = 0 start_frame, end_frame = 1000000, -1 completed_half = False total_len_frame = 0 one_cycle_kpts = [] for i, img in enumerate(image_provider): img = cv2.resize(img, (600, 600)) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) #total_keypoints_num = 18 pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) #### pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) #Select joints pose_keypoints = np.concatenate( (pose_keypoints[2], pose_keypoints[5], pose_keypoints[8], pose_keypoints[10], pose_keypoints[11], pose_keypoints[13])).reshape(-1, 2) #Analyze posture previous_pose_kpts.append(pose_keypoints) liftoneleg = LiftOneLeg(previous_pose_kpts) #Wrong angle, leg_status = liftoneleg.check_leg_up_down() #Update status and count leg_status, completed_half, count_update, start_frame_update, end_frame_update= \ liftoneleg.count_repetition(angle, leg_status, completed_half, count, i, start_frame, end_frame) if (count_update == count + 1): print("count : %d" % count) one_cycle_kpts.append(previous_pose_kpts[start_frame:]) result = test_per_frame( previous_pose_kpts[start_frame - total_len_frame:end_frame - total_len_frame], LABEL) total_len_frame += len(previous_pose_kpts) previous_pose_kpts = [] count, start_frame, end_frame = count_update, start_frame_update, end_frame_update #To plot angle graph if int(angle) != 90: graph_x.append(i) graph_y.append(angle) #Put text on the screen cv2.putText(img, 'count : {}'.format(count), (10, 520), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 2) cv2.putText(img, "Rsho-Lsho :%3.2f" % (result[0]), (10, 550), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) cv2.putText( img, "Lsho-Lhip :%3.2f, Lhip-Lank :%3.2f" % (result[1], result[2]), (10, 570), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) cv2.putText(img, "Rhip-Rank :%3.2f" % (result[3]), (10, 590), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255, 255, 0), 2) cv2.putText( img, '3 align :{}'.format(liftoneleg.check_if_3points_are_aligned()), (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 2) cv2.putText( img, 'shoulder :{}'.format(liftoneleg.check_if_shoulders_are_aligned()), (10, 100), cv2.FONT_HERSHEY_SIMPLEX, 1.5, (255, 255, 255), 2) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return return graph_x, graph_y
def run_demo(net, action_net, image_provider, height_size, cpu, boxList): net = net.eval() print(torch.cuda.device_count()) print(torch.cuda.is_available()) a = torch.Tensor(5, 3) a = a.cuda() print(a) if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18 i = 0 for img in image_provider: # 遍历图像集 orig_img = img.copy() # copy 一份 # print(i) fallFlag = 0 if i % 1 == 0: heatmaps, pafs, scale, pad = infer_fast( net, img, height_size, stride, upsample_ratio, cpu) # 返回热图,paf,输入模型图象相比原始图像缩放倍数,输入模型图像padding尺寸 total_keypoints_num = 0 all_keypoints_by_type = [ ] # all_keypoints_by_type为18个list,每个list包含Ni个当前点的x、y坐标,当前点热图值,当前点在所有特征点中的index for kpt_idx in range( num_keypoints): # 19th for bg 第19个为背景,之考虑前18个关节点 total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True ) # 得到所有分配的人(前18维为每个人各个关节点在所有关节点中的索引,后两唯为每个人得分及每个人关节点数量),及所有关节点信息 for kpt_id in range(all_keypoints.shape[0]): # 依次将每个关节点信息缩放回原始图像上 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): # 依次遍历找到的每个人 if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) posebox = (int(pose.bbox[0]), int(pose.bbox[1]), int(pose.bbox[0]) + int(pose.bbox[2]), int(pose.bbox[1]) + int(pose.bbox[3])) coincideValue = coincide(boxList, posebox) print(posebox) print('coincideValue:' + str(coincideValue)) if len( pose.getKeyPoints() ) >= 10 and coincideValue >= 0.3 and pose.lowerHalfFlag < 3: # 当人体的点数大于10个的时候算作一个人,同时判断yolov5的框和pose的框是否有交集并且占比30%,同时要有下半身 current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img, is_save=True, show_draw=True) crown_proportion = pose.bbox[2] / pose.bbox[3] #宽高比 pose = action_detect(action_net, pose, crown_proportion) #判断摔倒还是正常 if pose.pose_action == 'fall': cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 0, 255), thickness=3) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) fallFlag = 1 else: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0)) # fallFlag = 1 if fallFlag == 1: t = time.time() # cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img) print('我保存照片了') img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # 保存识别后的照片 # cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img) # print('我保存照片了') # cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1) i += 1 cv2.destroyAllWindows()
def run_on_image(net, height_size, cpu, track, smooth, img, stride, upsample_ratio, num_keypoints, threshold): global previous_poses orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) score = 0 total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: # cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), r_hand_center, r_hand_width, l_hand_center, l_hand_width, = detect_hand( pose) if -1 not in r_hand_center: cv2.circle(img, (r_hand_center[0], r_hand_center[1]), 5, (255, 0, 0), 5) cv2.rectangle(img, (r_hand_center[0] - r_hand_width, r_hand_center[1] - r_hand_width), (r_hand_center[0] + r_hand_width, r_hand_center[1] + r_hand_width), (0, 255, 255)) if -1 not in l_hand_center: cv2.circle(img, (l_hand_center[0], l_hand_center[1]), 5, (255, 0, 0), 5) cv2.rectangle(img, (l_hand_center[0] - l_hand_width, l_hand_center[1] - l_hand_width), (l_hand_center[0] + l_hand_width, l_hand_center[1] + l_hand_width), (0, 255, 255)) face_center, face_width = detect_face(pose) if -1 not in face_center: cv2.rectangle( img, (face_center[0] - face_width, face_center[1] - face_width), (face_center[0] + face_width, face_center[1] + face_width), (0, 0, 255)) # (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (face_center[0] - face_width, face_center[1] - face_width - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if -1 not in r_hand_center: x, y, h, w, score = detect_touch(face_center, face_width, r_hand_center, r_hand_width) if h != 0: cv2.rectangle(img, (x, y), (x + h, y + w), (255, 0, 255)) cv2.putText(img, f'Score: {score:0.2f}', (x, y - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 0)) if -1 not in l_hand_center: x, y, h, w, score = detect_touch(face_center, face_width, l_hand_center, l_hand_width) if h != 0: cv2.rectangle(img, (x, y), (x + h, y + w), (255, 0, 255)) cv2.putText(img, f'Score: {score:0.2f}', (x, y - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (255, 255, 0)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) delay = 1 detect = False key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33 return score > threshold
def run_demo(net, image_provider1, image_provider2, height_size, cpu, track, smooth, com): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # +1 for Hidden COM previous_poses = [] # original delay 33 # 0 = pause / wait for input indefinetly delay = 33 total_provider = zip(image_provider1, image_provider2) for img1, img2 in total_provider: orig_img1 = img1.copy() orig_img2 = img2.copy() heatmaps1, pafs1, scale1, pad1 = infer_fast(net, img1, height_size, stride, upsample_ratio, cpu) heatmaps2, pafs2, scale2, pad2 = infer_fast(net, img2, height_size, stride, upsample_ratio, cpu) total_keypoints_num1 = 0 total_keypoints_num2 = 0 all_keypoints_by_type1 = [] all_keypoints_by_type2 = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num1 += extract_keypoints(heatmaps1[:, :, kpt_idx], all_keypoints_by_type1, total_keypoints_num1) total_keypoints_num2 += extract_keypoints(heatmaps2[:, :, kpt_idx], all_keypoints_by_type2, total_keypoints_num2) pose_entries1, all_keypoints1 = group_keypoints(all_keypoints_by_type1, pafs1, demo=True) pose_entries2, all_keypoints2 = group_keypoints(all_keypoints_by_type2, pafs2, demo=True) for kpt_id in range(all_keypoints1.shape[0]): all_keypoints1[kpt_id, 0] = (all_keypoints1[kpt_id, 0] * stride / upsample_ratio - pad1[1]) / scale1 all_keypoints1[kpt_id, 1] = (all_keypoints1[kpt_id, 1] * stride / upsample_ratio - pad1[0]) / scale1 for kpt_id in range(all_keypoints2.shape[0]): all_keypoints2[kpt_id, 0] = (all_keypoints2[kpt_id, 0] * stride / upsample_ratio - pad2[1]) / scale2 all_keypoints2[kpt_id, 1] = (all_keypoints2[kpt_id, 1] * stride / upsample_ratio - pad2[0]) / scale2 current_poses1 = [] current_poses2 = [] for n in range(len(pose_entries1)): if len(pose_entries1[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM found_kpts = [] C_pts = [] BOS = [[-1, -1], [-1, -1]] for kpt_id in range(num_keypoints): if pose_entries1[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints1[int(pose_entries1[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints1[int(pose_entries1[n][kpt_id]), 1]) found_kpts.append(kpt_id) if com: COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints) pose_keypoints[-1] = COM pose = Pose(pose_keypoints, pose_entries1[n][18], C_pts, BOS) current_poses1.append(pose) for n in range(len(pose_entries2)): if len(pose_entries2[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM found_kpts = [] C_pts = [] BOS = [[-1, -1], [-1, -1]] for kpt_id in range(num_keypoints): if pose_entries2[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints2[int(pose_entries2[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints2[int(pose_entries2[n][kpt_id]), 1]) found_kpts.append(kpt_id) if com: COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints) pose_keypoints[-1] = COM pose = Pose(pose_keypoints, pose_entries2[n][18], C_pts, BOS) current_poses2.append(pose) #if track: #track_poses(previous_poses, current_poses, smooth=smooth) #previous_poses = current_poses for pose in current_poses1: pose.draw(img1) for pose in current_poses2: pose.draw(img2) img1 = cv2.addWeighted(orig_img1, 0.6, img1, 0.4, 0) img2 = cv2.addWeighted(orig_img2, 0.6, img2, 0.4, 0) #for pose in current_poses: #cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), #(pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) #if track: #cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), #cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Demo, Feed 1', img1) cv2.imshow('Demo, Feed 2', img2) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] list_of_elbow_angles = [] list_of_shoulder_angles = [] list_of_hip_angles = [] list_of_knee_angles = [] rep_count = 0 rep_not_counted = True rep_start = False rep_end = False activity = None rotate = True rotate_90 = True rotate_270 = False for img in image_provider: if rotate: if rotate_90: img = imutils.rotate_bound(img, 90) if rotate_270: img = imutils.rotate_bound(img, 270) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) # pose entries -> Array showing number of poses, and which keypoint in each pose for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] # do not iterate over all poses, instead only choose the one with largest bounding box (pose.bbox) for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) # import pdb;pdb.set_trace() pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) final_pose = pose.get_pose() draw(final_pose, img, names = False, lines = True, angles = [5, 6, 7]) # """ # - Find out sleeping position (or not standing? Easier to write rules for standing) # - # """ # Figure out if standing or sleeping neck = get_coordinates(final_pose, 1) hip = average_coordinates(final_pose, 8, 11) knee = average_coordinates(final_pose, 9, 12) # checking if standing by comparing 'y' coordinates # neck is not good. Choose something which is below hips in sleeping but not in standing if neck[1] > hip[1] and hip[1] > knee[1]: rotate = False else: rotate = True # checking which side to rotate by comparing x coordinates print('neck[0], hip[0], neck[1], hip[1]',neck[0], hip[0], neck[1], hip[1]) if neck[0] < hip[0]: rotate_90 = True rotate_270 = False else: rotate_90 = False rotate_270 = True # print('rotate, rotate_90, rotate_270',rotate, rotate_90, rotate_270) continue if activity == None: activity, list_of_elbow_angles = get_activity(final_pose, list_of_elbow_angles) continue else: pass pushups = Pushups(final_pose, list_of_elbow_angles) try: corrections = pushups.all_corrections() for problem, correction in corrections.items(): if problem == 'lazy_pushup': list_of_elbow_angles.append(correction[0]) # if rep_not_counted: start = correction[2] down = correction[3] # print('1] start, down, rep_start, rep_end, rep_count, elbow angle', \ # start, down, rep_start, rep_end, rep_count, correction[0]) print('Activity : {} ||| Correction : {} ||| Rep count : {}'.format(activity, correction[1], rep_count), end = '\r') # print('Correction : {} ||| Rep count : {}'.format(correction[1], rep_count), end = '\r') if start == True and down == True and rep_start == False: start_index = len(list_of_elbow_angles) - 1 rep_start = True rep_end = False rep_counted = False if start == False and down == False and rep_end == False: stop_index = len(list_of_elbow_angles) - 1 interval = [] # import pdb; pdb.set_trace() for angle in list_of_elbow_angles[start_index: stop_index]: if angle < pushups.lazy_pushup_threshold and not rep_counted: rep_count = rep_count + 1 rep_counted = True rep_end = True rep_start = False squats = Squats(final_pose, list_of_hip_angles) print("Ran Squats") try: corrections = squats.all_corrections() for problem, correction in corrections.items(): if problem == 'squat_depth': list_of_hip_angles.append(correction[0]) # if rep_not_counted: start = correction[2] down = correction[3] print('1] start, down, rep_start, rep_end, rep_count, squat angle', \ start, down, rep_start, rep_end, rep_count, correction[0]) if start == True and down == True and rep_start == False: start_index = len(list_of_hip_angles) - 1 rep_start = True rep_end = False rep_counted = False if start == False and down == False and rep_end == False: stop_index = len(list_of_hip_angles) - 1 interval = [] # import pdb; pdb.set_trace() for angle in list_of_hip_angles[start_index: stop_index]: if angle < squats.squat_depth_angle_threshold and not rep_counted: rep_count = rep_count + 1 rep_counted = True rep_end = True rep_start = False print('2] start, down, rep_start, rep_end, rep_count, squat angle', \ start, down, rep_start, rep_end, rep_count, correction[0]) except: pass img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) # cv2.imwrite('output.jpg', img) key = cv2.waitKey(1) if key == 27: # esc return
def get_skel_coords(net, image_provider, height_size=256, cpu=False, track=1, smooth=1): # text_file = open("skel/"+ args.save_txt, "w") net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 img = image_provider # for img in image_provider: #print(img.shape) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # for pose in current_poses: if len(current_poses) != 0: # n = text_file.write(coords) cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) # break #cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) if len(current_poses) != 0: #print ("not zero", img) return current_poses[0].return_coords(), img else: #print ("zero") return [], np.array((1, 1, 1), np.uint8) key = cv2.waitKey(delay) if key == 27: # esc text_file.close() return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_inference(net, image_provider, height_size, cpu, track, smooth, no_display, json_view=False): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 100 if isinstance(image_provider, ImageReader): delay = 0 for img in image_provider: heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n, pose_entry in enumerate(pose_entries): if len(pose_entry) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entry[kpt_id] != -1.0: pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entry[kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entry[kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entry[18]) current_poses.append(pose) if json_view == True: return current_poses if not no_display: if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (32, 202, 252)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('PoseCamera', img) key = cv2.waitKey(delay) if key == 27: return
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = num_keys previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][num_keypoints]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0, 0, 255)) for p in range(pose_entries.shape[0]): allkey=True landmarks=[] if(len(pose_entries[p])>0): if pose_entries[p][0]==-1 or pose_entries[p][14]==-1 or pose_entries[p][15]==-1 or (pose_entries[p][16]==-1 and pose_entries[p][17]==-1): allkey=False if allkey: landmarks.append(int(all_keypoints[int(pose_entries[p][0])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][0])][1])) landmarks.append(int(all_keypoints[int(pose_entries[p][14])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][14])][1])) landmarks.append(int(all_keypoints[int(pose_entries[p][15])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][15])][1])) if pose_entries[p][17]==-1: ear="right" landmarks.append(int(all_keypoints[int(pose_entries[p][16])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][16])][1])) else: ear="left" landmarks.append(int(all_keypoints[int(pose_entries[p][17])][0])) landmarks.append(int(all_keypoints[int(pose_entries[p][17])][1])) if len(landmarks)>0: imgpts, modelpts, rotate_degree, center = face_orientation(img, landmarks,ear) cv2.line(img, center, tuple(imgpts[1].ravel()), (0,255,0), 3) #GREEN cv2.line(img, center, tuple(imgpts[0].ravel()), (255,0,), 3) #BLUE cv2.line(img, center, tuple(imgpts[2].ravel()), (0,0,255), 3) #RED #for j in range(len(rotate_degree)): #cv2.putText(img, ('{:05.2f}').format(float(rotate_degree[j])), (10, 30 + (50 * j)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), thickness=2, lineType=2) #cv2.imwrite('/Users/Utente/Desktop/poseFINAL/out.jpg', img) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(model, image_provider, height_size, cpu, track, smooth, file): """[summary] Args: model ([type]): [description] image_provider ([type]): [description] height_size ([type]): [description] cpu ([type]): [description] track ([type]): [description] smooth ([type]): [description] file ([type]): [description] Returns: [type]: [description] """ model = model.eval() if not cpu: model = model.cuda() point_list = [] stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] # 保存视频 fps = image_provider.fps width = image_provider.width height = image_provider.height fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', 'V') # video_saver = cv2.VideoWriter('TESTV.mp4', fourcc, fps, (height, width)) save_video_path = os.path.join(os.getcwd(), 'video_output') if not os.path.exists(save_video_path): os.mkdir(save_video_path) save_video_name = os.path.join(save_video_path, file + '.mp4') video_saver = cv2.VideoWriter(save_video_name, fourcc, fps, (width, height)) for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(model, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for pose_entry in pose_entries: if len(pose_entry) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entry[kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entry[kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entry[kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entry[18]) current_poses.append(pose) # save keypoint in list key_point_list = pose_keypoints.flatten().tolist() point_list.append(key_point_list) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) video_saver.write(img) return point_list