def run_demo(net, image_provider, height_size, cpu): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 color = [0, 224, 255] # buf = [] finish = False begun = False count = 0 avgwes, avgesh, avgshk, avgseh, avghorizontal, avgsek = 0., 0., 0., 0., 0., 0. badpose_startt = 0 pose_startt = 0 total_bad_duration = 0 lower_hip_time = 0 averaging_duration = 3 message_time_gap = 7 reset_time_gap = 5 all_checks = (Checker(message_time_gap, reset_time_gap, bodyDownCheck, "Raise your body up"), Checker(message_time_gap, reset_time_gap, kneeDownCheck, "Raise your knees"), Checker(message_time_gap, reset_time_gap, backHighCheck, "Lower your back")) avg_accuracy = AverageAccuracy() #Stop the workout if the an incorrect posture lasts for more than 15 seconds max_bad_pose_duration = 20 last_time_time = time.time() knee_touching_ground = False for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(18): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale # print(len(pose_entries)) # print("parts__") kpLocations = dict() # print("Pose entries: ", len(pose_entries)) for n in range(len(pose_entries)): # print("pose entries length ", len(pose_entries[n])) if len(pose_entries[n]) < 10: continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Shoulder", "Ankle"): continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Elbow", "Hip"): continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Shoulder", "Hip"): continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Shoulder", "Knee"): continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Shoulder", "Ankle"): continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Elbow", "Hip"): continue if not checkDistance(pose_entries[n], all_keypoints, "left", "Elbow", "Knee"): continue # print("Is horizontal : " , horizontal) left = isFacingLeft(pose_entries[n], all_keypoints) print("Is facing left ", left) anglehorizontal = horizontalAngle(pose_entries[n], all_keypoints) displayText("Hor: " + str(anglehorizontal), 50, 20, img) anglewes, angleesh, angleshk, angleseh, anglesek = getPlankAngles( pose_entries[n], all_keypoints, img) cont = False for angl in (anglewes, angleesh, angleshk, angleseh, anglesek): if math.isnan(angl): cont = True print("nan found") break if cont: continue avghorizontal += anglehorizontal avgwes += anglewes avgesh += angleesh avgshk += angleshk avgseh += angleseh avgsek += anglesek count += 1 if count == averaging_duration: count = 0 avgwes /= averaging_duration avgesh /= averaging_duration avgshk /= averaging_duration avgseh /= averaging_duration avghorizontal /= averaging_duration avgsek /= averaging_duration horizontal = checkHorizontal(avghorizontal) # print("Is horizontal ", horizontal) if horizontal: # print("Avg Angles: wrist elbow shoulder: {0} elbow shoulder hips: {1} shoulder hips knee: {2} shoulder elbow hip {3}".format(avgwes, avgesh, avgshk, avgseh)) correct_pose, wes, esh, shk, seh = checkCorrectPlank( avgwes, avgesh, avgshk, avgseh) if not begun and correct_pose: begun = True pose_startt = time.time() last_time_time = time.time() tts.BotSpeak( "Now that you are in position, I'll start the timer." ) botlog_queue.put( "Now that you are in position, I'll start the timer." ) if begun: accuracy = getAccuracy(avgwes, avgesh, avgshk, avgseh, avgsek) accuracy_queue.put(accuracy) avg_accuracy.put(accuracy) displayText(accuracy, 10, 120, img) correct_pose = True for check in all_checks: if not check.check(avgwes, avgesh, avgshk, avgseh, avgsek): # if check.display_message: # tts.BotSpeak(check.message) if check.display_message: tts.BotSpeak(check.message) botlog_queue.put(check.message) displayText(check.message, 20, 100, img, (0, 255, 0)) correct_pose = False break d = time.time() - last_time_time if d > 10: seconds_over = int(time.time() - pose_startt) tts.BotSpeak( "{0} seconds over".format(seconds_over)) botlog_queue.put( "{0} seconds over".format(seconds_over)) last_time_time = time.time() if not correct_pose: # if not shk: # if lower_hip_time > 0: # d = time.time() - lower_hip_time # if d > 5: # tts.BotSpeak("Lower your hips") # lower_hip_time = 0 # else: # lower_hip_time = time.time() # else: # lower_hip_time = 0 if badpose_startt != 0: duration = time.time() - badpose_startt displayText( "badpose duration: " + str(duration), 0, 50, img) if duration > max_bad_pose_duration: finish = True # if duration > 20: # displayText("Pose broken for 20", 0, 40, img) else: badpose_startt = time.time() else: lower_hip_time = 0 if badpose_startt != 0: total_bad_duration += time.time( ) - badpose_startt badpose_startt = 0 displayText("Correct Pose", 0, 30, img) print("Correct Pose: ", correct_pose) elif begun: finish = True avgwes, avgesh, avgshk, avgseh, avghorizontal, avgsek = 0., 0., 0., 0., 0., 0. if finish: if badpose_startt > 0: total_bad_duration += time.time() - badpose_startt duration = time.time() - pose_startt good_duration = duration - total_bad_duration tts.BotSpeak("You held an incorrect position for too long.") botlog_queue.put( "You held an incorrect position for too long.") tts.BotSpeak("Your workout is now complete") botlog_queue.put("Your workout is now complete") tts.BotSpeak("Your average accuracy was {0}".format( int(avg_accuracy.get()))) botlog_queue.put("Your average accuracy was {0}".format( int(avg_accuracy.get()))) tts.BotSpeak( "You held the correct plank position for {0} seconds and your workout lasted for {1} seconds." .format(int(good_duration), int(duration))) botlog_queue.put( "You held the correct plank position for {0} seconds and your workout lasted for {1} seconds." .format(int(good_duration), int(duration))) # tts.BotSpeak("end") print("-" * 20) print("Total duration: ", good_duration) return # print("true_count {0} count {1} finish {2} begun {3} ".format(true_count, count, finish, begun)) # triplet = ("left", "Shoulder", "Hip", "Ankle") # # triplet = ("left", "Wrist", "Elbow", "Shoulder") # print(getAngle(pose_entries[n], all_keypoints, *triplet)) # print("wes", getAngle(pose_entries[n], all_keypoints, "left", "Wrist", "Elbow", "Shoulder")) # print("esh", getAngle(pose_entries[n], all_keypoints, "left", "Elbow", "Shoulder", "Hip")) # drawLinesTriplets(pose_entries[n], all_keypoints, img, color, *triplet) # print("trip2", getAngle(pose_entries[n], all_keypoints, "", "leftShoulder", "chestCenter", "rightShoulder")/180) for part_id in range(len(BODY_PARTS_PAF_IDS) - 2): kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0] global_kpt_a_id = pose_entries[n][kpt_a_id] if global_kpt_a_id != -1: x_a, y_a = all_keypoints[int(global_kpt_a_id), 0:2] cv2.circle(img, (int(x_a), int(y_a)), 3, color, -1) # cv2.putText(img, str(kpt_a_id), # (int(x_a), int(y_a)), # font, # fontScale, # fontColor, # lineType) kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1] global_kpt_b_id = pose_entries[n][kpt_b_id] if global_kpt_b_id != -1: x_b, y_b = all_keypoints[int(global_kpt_b_id), 0:2] cv2.circle(img, (int(x_b), int(y_b)), 3, color, -1) if global_kpt_a_id != -1 and global_kpt_b_id != -1: cv2.line(img, (int(x_a), int(y_a)), (int(x_b), int(y_b)), color, 2) # print(part_id, (x_a, y_a), (x_b, y_b)) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) # cv2.imwrite("output.jpeg", img) key = cv2.waitKey(33) if key == 27: # esc # tts.BotSpeak("end") return
def run_demo(net, action_net, image_provider, height_size, cpu, boxList): net = net.eval() print(torch.cuda.device_count()) print(torch.cuda.is_available()) a = torch.Tensor(5, 3) a = a.cuda() print(a) if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18 i = 0 for img in image_provider: # 遍历图像集 orig_img = img.copy() # copy 一份 # print(i) fallFlag = 0 if i % 1 == 0: heatmaps, pafs, scale, pad = infer_fast( net, img, height_size, stride, upsample_ratio, cpu) # 返回热图,paf,输入模型图象相比原始图像缩放倍数,输入模型图像padding尺寸 total_keypoints_num = 0 all_keypoints_by_type = [ ] # all_keypoints_by_type为18个list,每个list包含Ni个当前点的x、y坐标,当前点热图值,当前点在所有特征点中的index for kpt_idx in range( num_keypoints): # 19th for bg 第19个为背景,之考虑前18个关节点 total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True ) # 得到所有分配的人(前18维为每个人各个关节点在所有关节点中的索引,后两唯为每个人得分及每个人关节点数量),及所有关节点信息 for kpt_id in range(all_keypoints.shape[0]): # 依次将每个关节点信息缩放回原始图像上 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): # 依次遍历找到的每个人 if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) posebox = (int(pose.bbox[0]), int(pose.bbox[1]), int(pose.bbox[0]) + int(pose.bbox[2]), int(pose.bbox[1]) + int(pose.bbox[3])) coincideValue = coincide(boxList, posebox) print(posebox) print('coincideValue:' + str(coincideValue)) if len( pose.getKeyPoints() ) >= 10 and coincideValue >= 0.3 and pose.lowerHalfFlag < 3: # 当人体的点数大于10个的时候算作一个人,同时判断yolov5的框和pose的框是否有交集并且占比30%,同时要有下半身 current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img, is_save=True, show_draw=True) crown_proportion = pose.bbox[2] / pose.bbox[3] #宽高比 pose = action_detect(action_net, pose, crown_proportion) #判断摔倒还是正常 if pose.pose_action == 'fall': cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 0, 255), thickness=3) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) fallFlag = 1 else: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0)) # fallFlag = 1 if fallFlag == 1: t = time.time() # cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img) print('我保存照片了') img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # 保存识别后的照片 # cv2.imwrite(f'C:/zqr/project/yolov5_openpose/Image/{t}.jpg', img) # print('我保存照片了') # cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1) i += 1 cv2.destroyAllWindows()
def run_demo(net, image_provider, height_size, cpu, track_ids, arm): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] stateMachines = {} for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) """ kpt_names = ['nose', 'neck', 'r_sho', 'r_elb', 'r_wri', 'l_sho', 'l_elb', 'l_wri', 'r_hip', 'r_knee', 'r_ank', 'l_hip', 'l_knee', 'l_ank', 'r_eye', 'l_eye', 'r_ear', 'l_ear'] r_elb-3, r-wri-4, l_elb-6, l_wri-7 """ # print('ID: {}'.format(n)) # print('\tRight elbow: {}, right wrist: {}'.format(pose_keypoints[3], pose_keypoints[4])) # print('\tLeft elbow: {}, left wrist: {}'.format(pose_keypoints[6], pose_keypoints[7])) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses, threshold=3) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) for pose in current_poses: if pose.id not in stateMachines.keys(): stateMachines[pose.id] = StateMachine(pose.id, pose.keypoints, arm) #print('ID {} detected'.format(pose.id)) continue # call stateMachine methods stateMachines[pose.id].update(pose.keypoints, img) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) # print("showing") cv2.imwrite(f"outputs/demo-{image_provider.idx}.jpg", img)
def run_demo(model, image_provider, height_size, cpu, track, smooth, file): """[summary] Args: model ([type]): [description] image_provider ([type]): [description] height_size ([type]): [description] cpu ([type]): [description] track ([type]): [description] smooth ([type]): [description] file ([type]): [description] Returns: [type]: [description] """ model = model.eval() if not cpu: model = model.cuda() point_list = [] stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] # 保存视频 fps = image_provider.fps width = image_provider.width height = image_provider.height fourcc = cv2.VideoWriter_fourcc('M', 'P', '4', 'V') # video_saver = cv2.VideoWriter('TESTV.mp4', fourcc, fps, (height, width)) save_video_path = os.path.join(os.getcwd(), 'video_output') if not os.path.exists(save_video_path): os.mkdir(save_video_path) save_video_name = os.path.join(save_video_path, file + '.mp4') video_saver = cv2.VideoWriter(save_video_name, fourcc, fps, (width, height)) for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(model, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for pose_entry in pose_entries: if len(pose_entry) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entry[kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entry[kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entry[kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entry[18]) current_poses.append(pose) # save keypoint in list key_point_list = pose_keypoints.flatten().tolist() point_list.append(key_point_list) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) video_saver.write(img) return point_list
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) # Debug code for j in range(0, 16): heatmap = heatmaps[:,:,j] heatmap = heatmap.reshape((128,128,1)) heatmapimg = np.array(heatmap * 255, dtype = np.uint8) heatmap = cv2.applyColorMap(heatmapimg, cv2.COLORMAP_JET) #heatmap = heatmap/255 cv2.imwrite('hmtestpadh_'+str(j)+'.jpg', heatmap) #h_img = TF.to_pil_image(heatmaps) #h_img.save('img_heatmap', 'JPEG') print('heatmaps shape: {}'.format(heatmaps.shape)) np.savetxt('heatmaps.txt', heatmaps.reshape(-1)) rows, cols, depths = np.nonzero(pafs) print(pafs[rows, cols, depths]) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) # Debug code print('total_keypoints_num: {} {} {}\n'.format(all_keypoints_by_type, total_keypoints_num, kpt_idx)) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(0) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track_ids): #, filename): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts #18개 previous_poses = [] c = 0 ttt = 0 for img in image_provider: t5 = time.time() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): ##사이즈 변환 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: ##Track Poses propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) tt = time.time() fps = 1 / (tt - ttt) print('fps=', fps) ttt = time.time() str = "FPS : %0.2f" % fps cv2.putText(img, str, (0, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) # cv2.imwrite('PAF_'+filename[image_provider.idx -1], img) key = cv2.waitKey(1) if key == 27: # esc return
def run_demo(net, image_provider, height_size, cpu, track, smooth): show_time = True net = net.eval() if not cpu: net = net.cuda() track = True print("track:" + str(track)) stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 for img, img_dir in image_provider: orig_img = img.copy() start_time = datetime.datetime.now() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) end_time = datetime.datetime.now() if show_time: print("pose time:") print((end_time - start_time).microseconds / 1000) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) #cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) print(img_dir) img_save_dir = img_dir.replace("15_1", "15_1_lightpose") #cv2.imwrite("/data2/qilei_chen/DATA/ShanghaiAutograding/gangganpingheng_images_240/15_1/right_light_pose/"+os.path.basename(img_dir),img) cv2.imwrite(img_save_dir, img) '''
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 mean_time = 0 video_time = 0 prep_displacement = 0 prep_time = 0 for img in image_provider: tik1 = cv2.getTickCount() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if (pose.bbox[0] + pose.bbox[2] / 2 > 355 and (pose.bbox[0] + pose.bbox[2] / 2 < 555)): current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.rectangle(img, (330, 330), (430, 230), (0, 0, 255), 2) prev_pose = [] for pose in current_poses: if track: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if (pose.keypoints[4][0] > 330 and pose.keypoints[4][0] < 430) and (pose.keypoints[4][1] > 230 and pose.keypoints[4][1] < 330) and \ (pose.keypoints[7][0] > 330 and pose.keypoints[7][0] < 430) and (pose.keypoints[7][1] > 230 and pose.keypoints[7][1] < 330): cv2.rectangle( img, (pose.keypoints[4][0] - 10, pose.keypoints[4][1] - 10), (pose.keypoints[4][0] + 10, pose.keypoints[4][1] + 10), (0, 255, 0), 2) cv2.rectangle( img, (pose.keypoints[7][0] - 10, pose.keypoints[7][1] - 10), (pose.keypoints[7][0] + 10, pose.keypoints[7][1] + 10), (0, 255, 0), 2) if bool(previous_poses): prev_pose = previous_poses[0] prep_displacement += ( (pose.keypoints[4][0] - prev_pose.keypoints[4][0])**2 + (pose.keypoints[4][1] - prev_pose.keypoints[4][1])** 2)**0.5 prep_displacement += ( (pose.keypoints[7][0] - prev_pose.keypoints[7][0])**2 + (pose.keypoints[7][1] - prev_pose.keypoints[7][1])** 2)**0.5 prep_time += (cv2.getTickCount() - tik1) / cv2.getTickFrequency() previous_poses = current_poses current_time = (cv2.getTickCount() - tik1) / cv2.getTickFrequency() video_time += current_time cv2.putText(img, "displacement: %d" % int(prep_displacement), (20, 20), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1) cv2.putText(img, "prep time (s): %.1f" % (prep_time / 2), (20, 40), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1) cv2.putText(img, "video time (s): %.1f" % (video_time / 2), (20, 60), cv2.FONT_HERSHEY_COMPLEX, 0.6, (0, 0, 255), 1) if mean_time == 0: mean_time = current_time else: mean_time = mean_time * 0.95 + current_time * 0.05 #cv2.putText(img, 'FPS: {}'.format(int(1 / mean_time * 10) / 10), # (40, 80), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) cv2.imshow('Chicken Prep Test Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 1: delay = 0 else: delay = 1
def run_demo(net, height_size, cpu, track, smooth, image_provider=None, use_realsense_cam=False, openvino=False): if not image_provider and not use_realsense_cam: raise ValueError('Either `image_provider` or `use_realsense_cam` must be provided') if use_realsense_cam: pipeline = init_realsense() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 while True: # get frame from source provided if use_realsense_cam: frames = pipeline.wait_for_frames() color_frame = frames.get_color_frame() img = np.asanyarray(color_frame.get_data()) else: img = next(image_provider) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu, openvino=openvino) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track, smooth, com): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # +1 for Hidden COM previous_poses = [] # original delay 33 # 0 = pause / wait for input indefinetly delay = 0 for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints + 1, 2), dtype=np.int32) * -1 # +1 here for COM found_kpts = [] C_pts = [] BOS = [[-1, -1], [-1, -1]] for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) found_kpts.append(kpt_id) # print(kpt_id, pose_keypoints[kpt_id], Pose.kpt_names[kpt_id]) # ====== HOLY GRAIL ========= if com: COM, C_pts, BOS = compute_com(found_kpts, pose_keypoints) pose_keypoints[-1] = COM pose = Pose(pose_keypoints, pose_entries[n][18], C_pts, BOS) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, output_dir, height_size, cpu, track, smooth, generate_video): net = net.eval() if not cpu: net = net.cuda() if generate_video: video_output_path = os.path.join(output_dir, output_dir.split("/")[-1] +"-annotations" + ".mp4") #(1080, 1920, 3) video_out = cv2.VideoWriter(video_output_path, cv2.VideoWriter_fourcc('m','p','4','v'), 10, (1920,1080)) stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 for idx, img in enumerate(image_provider): orig_img = img.copy() frame_file = os.path.join(output_dir, output_dir.split("/")[-1]+f"_{idx:06}.txt") with open(frame_file, "w") as frame_f: print("Input the model is", orig_img.shape) #print(output_path) heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses annotations_hand = [] annotations_head = [] for pose in current_poses: pose.draw(img) print(pose.hands_bbox, pose.head_bbox) for hand_bbox in pose.hands_bbox: x_center = (hand_bbox[0][0] + hand_bbox[1][0]) / (2*1920) y_center = (hand_bbox[0][1] + hand_bbox[1][1]) / (2*1080) x_offset = (hand_bbox[1][0] - hand_bbox[0][0]) / 1920 y_offset = (hand_bbox[1][1] - hand_bbox[0][1]) / 1080 if x_center < 0.0: x_offset = x_offset - x_center x_center = 0 hand_bbox_scaled = (x_center, y_center, x_offset, y_offset) print(hand_bbox_scaled) frame_f.write("8 " + ' '.join(map(str, hand_bbox_scaled)) + "\n") annotations_hand.append(hand_bbox_scaled) x_center = (pose.head_bbox[0][0] + pose.head_bbox[1][0]) / (2*1920) y_center = (pose.head_bbox[0][1] + pose.head_bbox[1][1]) / (2*1080) x_offset = (pose.head_bbox[1][0] - pose.head_bbox[0][0]) / 1920 y_offset = (pose.head_bbox[1][1] - pose.head_bbox[0][1]) / 1080 if x_center < 0.0: x_offset = x_offset - x_center x_center = 0 head_bbox_scaled = (x_center, y_center, x_offset, y_offset) print(head_bbox_scaled) frame_f.write("9 " + ' '.join(map(str, head_bbox_scaled)) + "\n") annotations_head.append(head_bbox_scaled) img = cv2.addWeighted(orig_img, 0.3, img, 0.7, 0) for pose in current_poses: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) img_filename = os.path.join(output_dir, output_dir.split("/")[-1] + f"_{idx:06}.jpg") print(img_filename) cv2.imwrite(img_filename, orig_img) if generate_video: video_out.write(img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33 if video_out: video_out.release()
def evaluate(labels, output_name, images_folder, net, multiscale=False, visualize=False): net = net.cuda().eval() base_height = 368 scales = [1] if multiscale: scales = [0.5, 1.0, 1.5, 2.0] stride = 8 dataset = CocoValDataset(labels, images_folder) coco_result = [] for sample in dataset: file_name = sample["file_name"] img = sample["img"] avg_heatmaps, avg_pafs = infer(net, img, scales, base_height, stride) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(18): # 19th for bg total_keypoints_num += extract_keypoints( avg_heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, avg_pafs) coco_keypoints, scores = convert_to_coco_format( pose_entries, all_keypoints) image_id = int(file_name[0:file_name.rfind(".")]) for idx in range(len(coco_keypoints)): coco_result.append({ "image_id": image_id, "category_id": 1, # person "keypoints": coco_keypoints[idx], "score": scores[idx], }) if visualize: for keypoints in coco_keypoints: for idx in range(len(keypoints) // 3): cv2.circle( img, (int(keypoints[idx * 3]), int(keypoints[idx * 3 + 1])), 3, (255, 0, 255), -1, ) cv2.imshow("keypoints", img) key = cv2.waitKey() if key == 27: # esc return with open(output_name, "w") as f: json.dump(coco_result, f, indent=4) run_coco_eval(labels, output_name)
def run_demo(net, action_net, image_provider, height_size, cpu): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts i = 0 for img in image_provider: orig_img = img.copy() # print(i) if i % 1 == 0: heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) if len(pose.getKeyPoints()) >= 10: current_poses.append(pose) # current_poses.append(pose) for pose in current_poses: pose.img_pose = pose.draw(img, show_draw=True) crown_proportion = pose.bbox[2] / pose.bbox[3] #宽高比 pose = action_detect(action_net, pose, crown_proportion) if pose.pose_action == 'fall': cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 0, 255), thickness=3) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) else: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'state: {}'.format(pose.pose_action), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0)) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) cv2.waitKey(1) i += 1 cv2.destroyAllWindows()
def run_lightweight(net, img, height_size=256, cpu=False, track=False, smooth=True): ts1 = time.time() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 orig_img = img.copy() ts_netw1 = time.time() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) ts_netw2 = time.time() # print("network time", (ts_netw2 - ts_netw1) * 1000, "ms") total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: # pose.draw(img) pose.draw_only_relevant(img) #img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: #relevant only for demonstration # cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), # (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) ts2 = time.time() cv2.putText(img, '%.2fms' % ((ts2 - ts1) * 1000.0), (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255)) return img
def run_demo(net, image_provider, height_size, cpu): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 color = [255, 0, 0] #To show the bones colorcircle = [0, 0, 255] #To show the joints for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(18): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale global imagename imagepath = imagename if imagepath != "": outputname = (imagepath[0].split('/')[-1]) outputname = "output/" + (outputname.split('.')[0]) + ".skl" skeletonfile = open(outputname, "w") skeletonfile.write(imagepath[0] + "\n") # Write all the keypoints to output for kpt in all_keypoints: skeletonfile.write(str(kpt[0]) + "," + str(kpt[1]) + ",0.0\n") for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue ######### calculate root point (since the prediction will give a set of joints that doesn't have root node) bneck = 0.406394346548266 bleft = 0.295839233078765 bright = 0.297766420372969 neckx, necky = all_keypoints[int(pose_entries[n][1]), 0:2] #neck vertex leftx, lefty = all_keypoints[int(pose_entries[n][11]), 0:2] #Left leg vertex rightx, righty = all_keypoints[int(pose_entries[n][8]), 0:2] #right leg vertex rootx = bneck * neckx + bleft * leftx + bright * rightx rooty = bneck * necky + bleft * lefty + bright * righty # Add root point to keypoints skeletonfile.write(str(rootx) + "," + str(rooty) + ",0.0\n") ########### Plot root point cv2.circle(img, (int(rootx), int(rooty)), 8, colorcircle, -1) ############ Draw lines writeedges = [] for part_id in range(len(BODY_PARTS_PAF_IDS) - 2): kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0] global_kpt_a_id = pose_entries[n][kpt_a_id] if global_kpt_a_id != -1: x_a, y_a = all_keypoints[int(global_kpt_a_id), 0:2] cv2.circle(img, (int(x_a), int(y_a)), 8, colorcircle, -1) kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1] global_kpt_b_id = pose_entries[n][kpt_b_id] if global_kpt_b_id != -1: x_b, y_b = all_keypoints[int(global_kpt_b_id), 0:2] cv2.circle(img, (int(x_b), int(y_b)), 8, colorcircle, -1) keypointsize = len(all_keypoints) if global_kpt_a_id != -1 and global_kpt_b_id != -1: if kpt_a_id == 1 and (kpt_b_id == 8 or kpt_b_id == 11): if (global_kpt_a_id, keypointsize) not in writeedges: writeedges.append((global_kpt_a_id, keypointsize)) if (keypointsize, global_kpt_b_id) not in writeedges: writeedges.append((keypointsize, global_kpt_b_id)) # skeletonfile.write(str(global_kpt_a_id)+","+str(keypointsize)+"\n") # skeletonfile.write(str(keypointsize)+","+str(global_kpt_b_id)+"\n") cv2.line(img, (int(x_a), int(y_a)), (int(rootx), int(rooty)), color, 3) cv2.line(img, (int(rootx), int(rooty)), (int(x_b), int(y_b)), color, 3) else: if (global_kpt_a_id, global_kpt_b_id) not in writeedges: writeedges.append( (global_kpt_a_id, global_kpt_b_id)) # skeletonfile.write(str(global_kpt_a_id)+","+str(global_kpt_b_id)+"\n") cv2.line(img, (int(x_a), int(y_a)), (int(x_b), int(y_b)), color, 3) # Writing the edges to file for eachi in writeedges: skeletonfile.write(str(eachi[0]) + "," + str(eachi[1]) + "\n") skeletonfile.close() if imagename != "": imagename = (imagename[0].split('/')[-1]) imagename = "output/" + imagename # print(imagename) # cv2.imwrite(imagename, img) else: imagename = "output/tempop.png" cv2.imwrite(imagename, img)
def postprocess(self, orig_image, output_dict): """ Reformat output into standard annotations """ heatmaps = output_dict['heatmaps'] pafs = output_dict['pafs'] scale = output_dict['scale'] pad = output_dict['pad'] num_keypoints = Pose.num_kpts total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * self.stride / self.upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * self.stride / self.upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) annotations = [] for pose_idx, pose in enumerate(current_poses): person_label = 'person_{}'.format(pose_idx) if envars.OUTPUT_POSES(): point_arr = [] found_points = set([]) for idx in range(pose.keypoints.shape[0]): x = int(pose.keypoints[idx, 0]) y = int(pose.keypoints[idx, 1]) label = self.kpt_names[idx] if x >= 0 and y >= 0: point_arr.append({'x': x, 'y': y, 'label': label}) found_points.add(label) links = [{ 'from': x[0], 'to': x[1] } for x in self.connections if x[0] in found_points and x[1] in found_points] annotations.append({ 'kind': 'lines', 'label': person_label, 'confidence': pose.confidence, 'points': point_arr, 'links': links }) if envars.OUTPUT_BOXES(): bbox_annotation = { 'kind': 'box', 'x': pose.bbox[0], 'y': pose.bbox[1], 'height': pose.bbox[3], 'width': pose.bbox[2], 'label': person_label, 'confidence': pose.confidence } annotations.append(bbox_annotation) return {'name': self.name, 'annotations': annotations}
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = num_keys previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][num_keypoints]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.3, (0, 0, 255)) allkey = True landmarks = [] if (len(pose_entries) > 0): for g in range(4): if pose_entries[0][g] == -1: allkey = False if allkey: for l in range(4): landmarks.append( int(all_keypoints[int(pose_entries[0][l])][0])) landmarks.append( int(all_keypoints[int(pose_entries[0][l])][1])) if len(landmarks) > 0: imgpts, modelpts, rotate_degree, center = face_orientation( img, landmarks) cv2.line(img, center, tuple(imgpts[1].ravel()), (0, 255, 0), 3) #GREEN cv2.line(img, center, tuple(imgpts[0].ravel()), ( 255, 0, ), 3) #BLUE cv2.line(img, center, tuple(imgpts[2].ravel()), (0, 0, 255), 3) #RED for j in range(len(rotate_degree)): cv2.putText(img, ('{:05.2f}').format(float(rotate_degree[j])), (10, 30 + (50 * j)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), thickness=2, lineType=2) cv2.imwrite('/home/charles/DockerData/Mattia/pose/poseFINAL/out.jpg', img) #cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def anime_frame(rgb, env, size=None, useSigmod=False, useTwice=False): if env is None: # return init_pose("./checkpoint_iter_370000.pth") # return init_pose("./default_checkpoints/R.pth") return init_pose("./refine4_checkpoints/checkpoint_iter_14000.pth") net, previous_poses = env stride = 8 upsample_ratio = 4 heatmaps, pafs, scale, pad = infer_fast(net, rgb, 368, stride, upsample_ratio) num_keypoints = Pose.num_kpts total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[ 1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[ 0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) # if track: track_poses(previous_poses, current_poses, smooth=True) previous_poses = current_poses env[1] = previous_poses # return rgb, env # print(rgb.min(), rgb.max()) # img = rgb.squeeze(0).permute(1, 2, 0).cpu().numpy()[:, :, ::-1]# * 255 # img = img.squeeze(0).permute(1, 2, 0).cpu().numpy()[:, :, ::-1]# * 255 # img += 0.5 # img *= 255 # img = img.astype(np.uint8) img = np.zeros((rgb.shape[2], rgb.shape[3], rgb.shape[1]), dtype=np.uint8) show_info = True for pose in current_poses: pose.draw(img, show_info) show_info = False # break # for pose in current_poses: # cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), # (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) # if track: # cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), # cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) img = img[:, :, ::-1] #/ 255 # print(img.shape, img.dtype, img.min(), img.max()) img = torch.FloatTensor(img.astype(np.float32)) img /= 255 img = img.permute(2, 0, 1).unsqueeze(0).cuda() output = rgb * 0.6 + img * 0.4 # output = img return output, env
def run_demo(net, image_provider, height_size, cpu, track_ids): # , filename): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts # 18개 previous_poses = [] c = 0 ttt = 0 idxx = 0 csv_dict = { 'frame_number': [], 'driver_index': [], 'is_driver_flag': [] } #,'state' : []} driver_find_flag = False weird_state_flag = False extractor = Extractor('default_checkpoints/ckpt.t7', True) find_class = Find_assault(extractor) for img in image_provider: is_driver_flag = False t5 = time.time() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): ##사이즈 변환 all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) # 운전자를 못 찾았으면 find_driver 에 들어감. if driver_find_flag is False: driver_find_flag, find_driver_count, find_state = find_class.find_driver( current_poses, orig_img) cv2.putText(img, "Driver_find_count : " + str(find_driver_count), (0, 20), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) cv2.putText(img, "State : Find_Driver", (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) else: # print("idxx : ", idxx) is_driver_flag, driver_index, weird_state_count, weird_state_flag = find_class.is_driver( current_poses, orig_img) cv2.putText(img, "Weird_State_Count : " + str(weird_state_count), (0, 20), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) if weird_state_flag: cv2.putText(img, 'State : ABNORMAL', (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) else: cv2.putText(img, "State : Driver_Found", (0, 50), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0)) # print("Driver_index :", driver_index) # print("Driver_Flag : ", is_driver_flag) # csv_dict['frame_number'].append(idxx) # csv_dict['driver_index'].append(driver_index) # csv_dict['is_driver_flag'].append(is_driver_flag) # csv_dict['state'].append(state) if track_ids == True: ##Track Poses propagate_ids(previous_poses, current_poses) previous_poses = current_poses index_counter = 0 for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if is_driver_flag and index_counter == driver_index: cv2.putText(img, 'DRIVER', (pose.bbox[0] + 100, pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255)) index_counter += 1 tt = time.time() fps = 1 / (tt - ttt) print('fps=', fps) ttt = time.time() str_ = "FPS : %0.2f" % fps cv2.putText(img, str_, (0, 100), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) # cv2_imshow(img) cv2.imwrite('output/two_2/' + str(idxx) + '.png', img) idxx += 1 key = cv2.waitKey(1) if key == 27: # esc return df = pd.DataFrame(csv_dict)
def callback(self, data): try: cv_image = self.bridge.imgmsg_to_cv2(data, "bgr8") except CvBridgeError as e: print(e) ## Rescale Image size rescale_factor = 1 width = int(cv_image.shape[1] * rescale_factor) height = int(cv_image.shape[0] * rescale_factor) dim = (width, height) resized_img = cv2.resize(cv_image, dim) net = PoseEstimationWithMobileNet() checkpoint = torch.load( "/home/zheng/lightweight-human-pose-estimation.pytorch/checkpoint_iter_370000.pth", map_location='cpu') load_state(net, checkpoint) height_size = 256 net = net.eval() net = net.cuda() net.eval() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 # img = cv2.imread("/home/zheng/lightweight-human-pose-estimation.pytorch/data/image_1400.jpg") img = asarray(cv_image) orig_img = img heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu="store_true") total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) # cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) self.image_pub.publish(self.bridge.cv2_to_imgmsg(img, "bgr8")) # cv2.imwrite('/home/zheng/Bureau/image_1400_key.jpg',img) cv2.waitKey(2)
def run_demo(net, image_provider, height_size, cpu, track, smooth): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 dict_id_color = {} for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int(all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int(all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses dict_id_color_r = {} for id_ in dict_id_color.keys(): flag_track = False for pose in current_poses: if id_ == pose.id: flag_track = True break if flag_track: dict_id_color_r[pose.id] = dict_id_color[pose.id] dict_id_color = dict_id_color_r track_thr = 0 for pose in current_poses: # print('pose.id : ',pose.id) if pose.id not in dict_id_color.keys(): R_ = random.randint(30,255) G_ = random.randint(30,255) B_ = random.randint(30,255) dict_id_color[pose.id] = [[B_,G_,R_],1] else: dict_id_color[pose.id][1] += 1 if dict_id_color[pose.id][1]>track_thr: pose.draw(img,color_x = dict_id_color[pose.id][0]) img = cv2.addWeighted(orig_img, 0.3, img, 0.7, 0) for pose in current_poses: if dict_id_color[pose.id][1]>track_thr: cv2.rectangle(img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 255, 0),4) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.namedWindow('Lightweight Human Pose Estimation Python Demo', 0) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33
def run_demo(net, image_provider, height_size, cpu, track_ids): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = num_keys previous_poses = [] for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][num_keypoints]) current_poses.append(pose) pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) if track_ids == True: propagate_ids(previous_poses, current_poses) previous_poses = current_poses for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) if save: cv2.imwrite( '/Users/Utente/Desktop/POSE-ESTIMATION/custom_pose/poseFINAL/out.jpg', img) else: cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(33) if key == 27: # esc return
def pose2d_detectHuman(net, img, height_size=256, track=1, smooth=1, bVis=True): stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 33 if True: # for img in image_provider: orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu=0) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if bVis: if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 33: delay = 0 else: delay = 33 return current_poses
def run_demo(net, height_size, track, smooth, record_vid, camera_type): net = net.eval() net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] ##Tarit defined slope_threshold = 0.4 ear_slope_threshold = 0.5 eye_ear_slope_threshold = 0.5 not_detected = (-1, -1) sleep_confirmation_time = 60 #in seconds #flags to detect whether the person is sleeping or not sleeping = False timer_started = False time_notified = 0 selected_pose = None while True: #msg, frame = receiver.receive(timeout = 60.0) #img = cv2.imdecode(np.frombuffer(frame, dtype='uint8'), -1) img = cap.read() if camera_type == "jetson": img = img[1300:1780, 1320:1960] #start_time = time.time() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses '''for pose in current_poses: pose.draw(img)''' ##find longest_nect_to_nose_dst and select that pose longest_nect_to_nose_dst = 0 for pose in current_poses: nose = tuple(pose.keypoints[0]) neck = tuple(pose.keypoints[1]) ##pythagoras nect_to_nose_dst = pow((pow(abs(nose[0] - neck[0]), 2)) + (pow(abs(nose[1] - neck[1]), 2)), 1 / 2) if nect_to_nose_dst > longest_nect_to_nose_dst: longest_nect_to_nose_dst = nect_to_nose_dst selected_pose = pose if selected_pose is not None: selected_pose.draw(img) nose = tuple(selected_pose.keypoints[0]) neck = tuple(selected_pose.keypoints[1]) l_ear = tuple(selected_pose.keypoints[16]) r_ear = tuple(selected_pose.keypoints[17]) l_eye = tuple(selected_pose.keypoints[15]) r_eye = tuple(selected_pose.keypoints[14]) #print(cal_slope(l_eye,l_ear),cal_slope(r_eye,r_ear)) ##detect if the person back if facing to the camera if nose == (-1, -1): if l_ear != not_detected and r_ear != not_detected: ear_slope = abs(l_ear[1] - r_ear[1]) / abs(l_ear[0] - r_ear[0]) cv2.circle(img, l_ear, 5, (255, 0, 0), 3) cv2.circle(img, r_ear, 5, (0, 255, 0), 3) if ear_slope > ear_slope_threshold: sleeping = True print("sleeping") else: sleeping = False else: ##out of condition, can't detect sleeping = False else: cv2.circle(img, nose, 5, (255, 0, 0), 3) cv2.circle(img, neck, 5, (0, 255, 0), 3) slope_inverse = (nose[0] - neck[0]) / (nose[1] - neck[1]) l_ear_eye_slope = cal_slope(l_eye, l_ear) r_ear_eye_slope = cal_slope(r_eye, r_ear) #increase the slope_threshold if the person is turning their head #print(pose.keypoints[16],pose.keypoints[17]) #print ear location if l_ear == (-1, -1) or r_ear == (-1, -1): slope_threshold = 1 print("one ear missing , Increasing slope_threshold") else: slope_threshold = 0.4 if abs(slope_inverse) > slope_threshold: #cv2.putText(img,"".join([str(pose.id),"sleeping"]),(20,50),cv2.FONT_HERSHEY_COMPLEX,2,(255,0,0),3) print("Sleeping (neck bend more than threshold)") #cv2.putText(img,"sleeping",(20,50),cv2.FONT_HERSHEY_COMPLEX,2,(255,0,0),3) sleeping = True elif l_eye == not_detected or r_eye == not_detected: sleeping = True print("Sleeping (not seeing both eyes)") elif l_ear_eye_slope < -0.6 or r_ear_eye_slope > 0.6 or l_ear_eye_slope > eye_ear_slope_threshold or r_ear_eye_slope < -eye_ear_slope_threshold: sleeping = True print("Sleeping (ears higher/lower than eyes)") else: print("Not sleeping") sleeping = False if sleeping: if not timer_started: t_start_sleep = time.time() timer_started = True else: if time.time() - t_start_sleep > sleep_confirmation_time: print("sending line message") pic_name = "".join( ["log_data/", str(time_notified), ".jpg"]) cv2.imwrite(pic_name, img) #lineNotify("Elderly sleeping %d"%time_notified) notifyFile("Elderly sleeping %d" % time_notified, pic_name) time_notified += 1 timer_started = False sleeping = False else: timer_started = False #song = AudioSegment.from_mp3("Alarm_Clock_Sound.mp3") #play(song) img = cv2.addWeighted(orig_img, 0.6, img, 0.6, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('Sleep detector', img) if record_vid: out_raw.write(orig_img) out_pose.write(img) #print((1/(time.time()-start_time))) if cv2.waitKey(1) == 27: # esc #receiver.close() cap.stop() if record_vid: out_raw.release() out_pose.release() return
def socketComm(conn, index): global terminate global net width = struct.unpack("<I", conn.recv(4))[0] height = struct.unpack("<I", conn.recv(4))[0] conn.settimeout(60) print("Recieved webcam:", width, height) if (width >= 3200) or (height >= 2400): print("Too large, closing") conn.close() terminate.put(index) return stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts delay = 33 if True: #try: while True: datalen = conn.recv(4) #receives length of data before data if len(datalen) == 0: break datalen = struct.unpack( "<I", datalen)[0] #convert little Endian binary to integer imageData = bytearray( 0) #master bytearray (imageData) to append to while len(imageData) != datalen: imageData.extend(conn.recv(datalen - len(imageData))) imageData = np.frombuffer(imageData, dtype=np.uint8) #decodes binary from jpeg into an image we can work with img = cv2.imdecode(imageData, cv2.IMREAD_COLOR) orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, args.height_size, stride, upsample_ratio, args.cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints( heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints( all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones( (num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) response = struct.pack("<36i", *pose_keypoints.flatten()) conn.send(struct.pack("<I", len(response))) conn.send(response) #except Exception as e: # print(e) conn.close() print("Closing", index) terminate.put(index)
def run_demo(net, image_provider, height_size, cpu): net = net.eval() net = net.cuda() stride = 8 upsample_ratio = 4 color = [0, 0, 255] global socket cv2.namedWindow("Human Pose", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("Human Pose", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) while True: grabbed, img = image_provider.read() orig_img = img.copy() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(18): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale humans = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue human = {} #print(all_keypoints[n]) #all_keypoints[n] = savgol_filter(all_keypoints[n], 9, 3) for part_id in range(17): kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0] global_kpt_a_id = pose_entries[n][kpt_a_id] if global_kpt_a_id != -1: x_a, y_a = all_keypoints[int(global_kpt_a_id), 0:2] cv2.circle(img, (int(x_a), int(y_a)), 8, color, -1) human[BODY_PARTS_IDS[int(kpt_a_id)]] = { 'position': { 'x': x_a, 'y': y_a } } kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1] global_kpt_b_id = pose_entries[n][kpt_b_id] if global_kpt_b_id != -1: x_b, y_b = all_keypoints[int(global_kpt_b_id), 0:2] cv2.circle(img, (int(x_b), int(y_b)), 8, color, -1) human[BODY_PARTS_IDS[int(kpt_b_id)]] = { 'position': { 'x': x_b, 'y': y_b } } if global_kpt_a_id != -1 and global_kpt_b_id != -1: cv2.line(img, (int(x_a), int(y_a)), (int(x_b), int(y_b)), color, 2) if any(human): humans.append(human) #socket.send_string(json.dumps(humans)) img = cv2.addWeighted(orig_img, 0.8, img, 0.8, 0) cv2.imshow('Human Pose', img) key = cv2.waitKey(1) if key == 27: # esc return
def run(net, image_provider, height_size, cpu, track, smooth): ts1 = time.time() net = setup(net, cpu) stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 1 ts2 = time.time() print("setup took ", ts2 - ts1, " s") print(ts2) print(ts1) for img in image_provider: ts1 = time.time() orig_img = img.copy() print('Original Dimensions : ', orig_img.shape) ts_netw1 = time.time() heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) ts_netw2 = time.time() print("network time", (ts_netw2 - ts_netw1) * 1000, "ms") total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): # 19th for bg total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n in range(len(pose_entries)): if len(pose_entries[n]) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entries[n][kpt_id] != -1.0: # keypoint was found pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entries[n][kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entries[n][kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entries[n][18]) current_poses.append(pose) if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) img = cv2.addWeighted(orig_img, 0.6, img, 0.4, 0) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (0, 255, 0)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) ts2 = time.time() cv2.putText(img, '%.2fms' % ((ts2 - ts1) * 1000.0), (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255)) cv2.imshow('Lightweight Human Pose Estimation Python Demo', img) key = cv2.waitKey(delay) if key == 27: # esc return elif key == 112: # 'p' if delay == 1: delay = 0 else: delay = 1
def run_inference(net, image_provider, height_size, cpu, track, smooth, no_display, json_view=False): net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 num_keypoints = Pose.num_kpts previous_poses = [] delay = 100 if isinstance(image_provider, ImageReader): delay = 0 for img in image_provider: heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(num_keypoints): total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs, demo=True) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale current_poses = [] for n, pose_entry in enumerate(pose_entries): if len(pose_entry) == 0: continue pose_keypoints = np.ones((num_keypoints, 2), dtype=np.int32) * -1 for kpt_id in range(num_keypoints): if pose_entry[kpt_id] != -1.0: pose_keypoints[kpt_id, 0] = int( all_keypoints[int(pose_entry[kpt_id]), 0]) pose_keypoints[kpt_id, 1] = int( all_keypoints[int(pose_entry[kpt_id]), 1]) pose = Pose(pose_keypoints, pose_entry[18]) current_poses.append(pose) if json_view == True: return current_poses if not no_display: if track: track_poses(previous_poses, current_poses, smooth=smooth) previous_poses = current_poses for pose in current_poses: pose.draw(img) for pose in current_poses: cv2.rectangle( img, (pose.bbox[0], pose.bbox[1]), (pose.bbox[0] + pose.bbox[2], pose.bbox[1] + pose.bbox[3]), (32, 202, 252)) if track: cv2.putText(img, 'id: {}'.format(pose.id), (pose.bbox[0], pose.bbox[1] - 16), cv2.FONT_HERSHEY_COMPLEX, 0.5, (0, 0, 255)) cv2.imshow('PoseCamera', img) key = cv2.waitKey(delay) if key == 27: return
def convert_to_skelets(in_, out_, cpu=False, height_size=256): # height_size - network input layer height size # cpu - True if we would like to run in CPU print('start convert to skelets') # mask that shows - this is bed mask = cv2.imread(os.path.join('mask', 'mask.jpg'), 0) mask = cv2.normalize(mask, None, alpha=0, beta=1, norm_type=cv2.NORM_MINMAX, dtype=cv2.CV_32F) net = PoseEstimationWithMobileNet() load_state(net, checkpoint) net = net.eval() if not cpu: net = net.cuda() stride = 8 upsample_ratio = 4 max_number = 963 num_img = 0 stream = cv2.VideoCapture("rtsp://*****:*****@62.140.233.76:554") # for num in range(0, max_number + 1): while (True): # frame = 'frame' + str(num) + '.jpg' # img = cv2.imread(os.path.join(in_, frame), cv2.IMREAD_COLOR) r, img = stream.read() # cv2.destroyAllWindows() # find the place of the bed - and add border to it, so we can cut the unnecessary part # apply object detection and find bed # output is an image with black pixels of not bed, and white pixels of bed heatmaps, pafs, scale, pad = infer_fast(net, img, height_size, stride, upsample_ratio, cpu) total_keypoints_num = 0 all_keypoints_by_type = [] for kpt_idx in range(18): total_keypoints_num += extract_keypoints(heatmaps[:, :, kpt_idx], all_keypoints_by_type, total_keypoints_num) pose_entries, all_keypoints = group_keypoints(all_keypoints_by_type, pafs) for kpt_id in range(all_keypoints.shape[0]): all_keypoints[kpt_id, 0] = (all_keypoints[kpt_id, 0] * stride / upsample_ratio - pad[1]) / scale all_keypoints[kpt_id, 1] = (all_keypoints[kpt_id, 1] * stride / upsample_ratio - pad[0]) / scale # how many persons in image num_persons = len(pose_entries) # num_img more than time_period - we delete first second and add the last second bones_detected = np.zeros(len(bones_to_detect)) bones_xa = np.zeros(len(bones_to_detect)) bones_ya = np.zeros(len(bones_to_detect)) bones_xb = np.zeros(len(bones_to_detect)) bones_yb = np.zeros(len(bones_to_detect)) bones_in_bed = np.zeros(len(bones_to_detect)) for n in range(num_persons): count_person_not_in_bed = 1 for id_x in range(len(bones_to_detect)): bones_detected[id_x] = 0 bones_xa[id_x] = 0 bones_ya[id_x] = 0 bones_xb[id_x] = 0 bones_yb[id_x] = 0 bones_in_bed[id_x] = 0 if len(pose_entries[n]) == 0: continue for id_, part_id in enumerate(bones_to_detect): kpt_a_id = BODY_PARTS_KPT_IDS[part_id][0] global_kpt_a_id = pose_entries[n][kpt_a_id] kpt_b_id = BODY_PARTS_KPT_IDS[part_id][1] global_kpt_b_id = pose_entries[n][kpt_b_id] # if both points are detected if global_kpt_a_id != -1 and global_kpt_b_id != -1: bones_xa[id_], bones_ya[id_] = all_keypoints[ int(global_kpt_a_id), 0:2] bones_xb[id_], bones_yb[id_] = all_keypoints[ int(global_kpt_b_id), 0:2] if mask[int(bones_ya[id_])][int( bones_xa[id_])] == 1 and mask[int( bones_yb[id_])][int(bones_xb[id_])] == 1: bones_in_bed[id_] = 1 bones_detected[id_] = 1 sum_bones = 0 for id_, val in enumerate(bones_in_bed): sum_bones += val if sum_bones == len(bones_in_bed): # anomaly # we take mean vector of 2 vectors of bones 6 and 9 bone_xa = (bones_xa[0] + bones_xa[2]) / 2 bone_ya = (bones_ya[0] + bones_ya[2]) / 2 bone_xb = (bones_xb[0] + bones_xb[2]) / 2 bone_yb = (bones_yb[0] + bones_yb[2]) / 2 x1 = bone_xb - bone_xa y1 = bone_yb - bone_ya x2 = 100 y2 = 0 global anomaly_checker alfa = math.acos( (x1 * x2 + y1 * y2) / (math.sqrt(x1**2 + y1**2) * math.sqrt(x2**2 + y2**2))) # if alfa is close to 90 degree - anomaly if min(abs(alfa - rad_90), abs(alfa - rad_270)) <= threshold: print('num_persons', num_persons) if num_persons == 1: anomaly_checker = np.delete(anomaly_checker, 0) anomaly_checker = np.append(anomaly_checker, 1) cv2.imwrite(os.path.join('out_out', frame), img) if np.sum(anomaly_checker) >= SEC_WITHOUT_HELP: print('ALARM!') num_img += 1 if not os.path.exists(out_): os.mkdir(out_) if cv2.waitKey(1) & 0xFF == ord('q'): break print('done convert to skelets')